From 3962bb4a3e2cddbb5b6c2530a012371774da28bb Mon Sep 17 00:00:00 2001 From: Yinyi Chen Date: Tue, 22 Oct 2024 10:33:39 -0700 Subject: [PATCH] set firelens mem_buf_limit --- agent/api/container/container.go | 19 +++++ agent/api/container/container_test.go | 42 +++++++++ agent/api/task/task.go | 14 ++- .../firelens/firelens_unimplemented.go | 2 +- agent/taskresource/firelens/firelens_unix.go | 4 +- .../firelens/firelens_unix_test.go | 46 +++++----- .../firelens/firelensconfig_unix.go | 32 +++++-- .../firelens/firelensconfig_unix_test.go | 85 +++++++++++++++++-- 8 files changed, 203 insertions(+), 41 deletions(-) diff --git a/agent/api/container/container.go b/agent/api/container/container.go index 6faed8197cf..5306491467c 100644 --- a/agent/api/container/container.go +++ b/agent/api/container/container.go @@ -1272,6 +1272,25 @@ func (c *Container) GetNetworkModeFromHostConfig() string { return hostConfig.NetworkMode.NetworkName() } +// GetMemoryReservationFromHostConfig returns the container memory reservation +func (c *Container) GetMemoryReservationFromHostConfig() int64 { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.DockerConfig.HostConfig == nil { + return 0 + } + + hostConfig := &dockercontainer.HostConfig{} + err := json.Unmarshal([]byte(*c.DockerConfig.HostConfig), hostConfig) + if err != nil { + seelog.Warnf("Encountered error when trying to get memory reservation for container %s: %v", c.RuntimeID, err) + return 0 + } + + return hostConfig.MemoryReservation +} + // GetHostConfig returns the container's host config. func (c *Container) GetHostConfig() *string { c.lock.RLock() diff --git a/agent/api/container/container_test.go b/agent/api/container/container_test.go index 8bc151fd847..ab3ffcab24e 100644 --- a/agent/api/container/container_test.go +++ b/agent/api/container/container_test.go @@ -751,6 +751,48 @@ func TestGetNetworkModeFromHostConfig(t *testing.T) { } } +func TestGetMemoryReservationFromHostConfig(t *testing.T) { + getContainer := func(hostConfig *string) *Container { + c := &Container{ + Name: "c", + } + c.DockerConfig.HostConfig = hostConfig + return c + } + + getStrPtr := func(s string) *string { + return &s + } + + testCases := []struct { + name string + container *Container + expectedOutput int64 + }{ + { + name: "happy case", + container: getContainer(getStrPtr("{\"MemoryReservation\":50}")), + expectedOutput: 50, + }, + { + name: "invalid case", + container: getContainer(getStrPtr("invalid")), + expectedOutput: 0, + }, + { + name: "nil case", + container: getContainer(nil), + expectedOutput: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expectedOutput, tc.container.GetMemoryReservationFromHostConfig()) + }) + } +} + func TestShouldCreateWithEnvfiles(t *testing.T) { cases := []struct { in Container diff --git a/agent/api/task/task.go b/agent/api/task/task.go index d7be2be28ce..812810e6247 100644 --- a/agent/api/task/task.go +++ b/agent/api/task/task.go @@ -1246,7 +1246,7 @@ func (task *Task) initializeFirelensResource(config *config.Config, resourceFiel for _, container := range task.Containers { firelensConfig := container.GetFirelensConfig() - if firelensConfig != nil { + if firelensConfig != nil { // is Firelens container var ec2InstanceID string if container.Environment != nil && container.Environment[awsExecutionEnvKey] == ec2ExecutionEnv { ec2InstanceID = resourceFields.EC2InstanceID @@ -1260,9 +1260,19 @@ func (task *Task) initializeFirelensResource(config *config.Config, resourceFiel } else { networkMode = container.GetNetworkModeFromHostConfig() } + // Resolve Firelens container memory limit to be used for calculating Firelens memory buffer limit + // We will use the container 'memoryReservation' (soft limit) if present, otherwise use container 'memory' (hard limit) + var containerMemoryLimit int64 + if memoryReservation := container.GetMemoryReservationFromHostConfig(); memoryReservation > 0 { + containerMemoryLimit = memoryReservation + } else if container.Memory > 0 { + containerMemoryLimit = int64(container.Memory) + } else { + containerMemoryLimit = 0 + } firelensResource, err := firelens.NewFirelensResource(config.Cluster, task.Arn, task.Family+":"+task.Version, ec2InstanceID, config.DataDir, firelensConfig.Type, config.AWSRegion, networkMode, firelensConfig.Options, containerToLogOptions, - credentialsManager, task.ExecutionCredentialsID) + credentialsManager, task.ExecutionCredentialsID, containerMemoryLimit) if err != nil { return errors.Wrap(err, "unable to initialize firelens resource") } diff --git a/agent/taskresource/firelens/firelens_unimplemented.go b/agent/taskresource/firelens/firelens_unimplemented.go index 63b59048c2a..fa5386bbdbb 100644 --- a/agent/taskresource/firelens/firelens_unimplemented.go +++ b/agent/taskresource/firelens/firelens_unimplemented.go @@ -53,7 +53,7 @@ type FirelensResource struct{} // NewFirelensResource returns a new FirelensResource. func NewFirelensResource(cluster, taskARN, taskDefinition, ec2InstanceID, dataDir, firelensConfigType, region, networkMode string, firelensOptions map[string]string, containerToLogOptions map[string]map[string]string, credentialsManager credentials.Manager, - executionCredentialsID string) (*FirelensResource, error) { + executionCredentialsID string, containerMemoryLimit int64) (*FirelensResource, error) { return nil, errors.New("not implemented") } diff --git a/agent/taskresource/firelens/firelens_unix.go b/agent/taskresource/firelens/firelens_unix.go index a9c02370e0c..c708e0ed193 100644 --- a/agent/taskresource/firelens/firelens_unix.go +++ b/agent/taskresource/firelens/firelens_unix.go @@ -83,6 +83,7 @@ type FirelensResource struct { networkMode string ioutil ioutilwrapper.IOUtil s3ClientCreator factory.S3ClientCreator + containerMemoryLimit int64 // Fields for the common functionality of task resource. Access to these fields are protected by lock. createdAtUnsafe time.Time @@ -98,7 +99,7 @@ type FirelensResource struct { // NewFirelensResource returns a new FirelensResource. func NewFirelensResource(cluster, taskARN, taskDefinition, ec2InstanceID, dataDir, firelensConfigType, region, networkMode string, firelensOptions map[string]string, containerToLogOptions map[string]map[string]string, credentialsManager credentials.Manager, - executionCredentialsID string) (*FirelensResource, error) { + executionCredentialsID string, containerMemoryLimit int64) (*FirelensResource, error) { firelensResource := &FirelensResource{ cluster: cluster, taskARN: taskARN, @@ -112,6 +113,7 @@ func NewFirelensResource(cluster, taskARN, taskDefinition, ec2InstanceID, dataDi s3ClientCreator: factory.NewS3ClientCreator(), executionCredentialsID: executionCredentialsID, credentialsManager: credentialsManager, + containerMemoryLimit: containerMemoryLimit, } fields := strings.Split(taskARN, "/") diff --git a/agent/taskresource/firelens/firelens_unix_test.go b/agent/taskresource/firelens/firelens_unix_test.go index 9070591fb17..ec8d2bbc68e 100644 --- a/agent/taskresource/firelens/firelens_unix_test.go +++ b/agent/taskresource/firelens/firelens_unix_test.go @@ -54,6 +54,7 @@ const ( testExecutionCredentialsID = "testexecutioncredentialsid" testExternalConfigType = "testexternalconfigtype" testExternalConfigValue = "testexternalconfigvalue" + testContainerMemoryLimit = 100 ) var ( @@ -105,7 +106,7 @@ func mockMkdirAllError() func() { func newMockFirelensResource(firelensConfigType, networkMode string, lopOptions map[string]string, mockIOUtil *mock_ioutilwrapper.MockIOUtil, mockCredentialsManager *mock_credentials.MockManager, - mockS3ClientCreator *mock_factory.MockS3ClientCreator) *FirelensResource { + mockS3ClientCreator *mock_factory.MockS3ClientCreator, containerMemoryReservation int64) *FirelensResource { return &FirelensResource{ cluster: testCluster, taskARN: testTaskARN, @@ -122,6 +123,7 @@ func newMockFirelensResource(firelensConfigType, networkMode string, lopOptions credentialsManager: mockCredentialsManager, ioutil: mockIOUtil, s3ClientCreator: mockS3ClientCreator, + containerMemoryLimit: containerMemoryReservation, } } @@ -158,7 +160,7 @@ func TestCreateFirelensResourceFluentdBridgeMode(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) defer mockRename()() gomock.InOrder( @@ -173,7 +175,7 @@ func TestCreateFirelensResourceFluentdAWSVPCMode(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, awsvpcNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) defer mockRename()() gomock.InOrder( @@ -188,7 +190,7 @@ func TestCreateFirelensResourceFluentdDefaultMode(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, "", testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) defer mockRename()() gomock.InOrder( @@ -203,7 +205,7 @@ func TestCreateFirelensResourceFluentbit(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentbit, bridgeNetworkMode, testFluentbitOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) defer mockRename()() gomock.InOrder( @@ -218,7 +220,7 @@ func TestCreateFirelensResourceInvalidType(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) firelensResource.firelensConfigType = "invalid" assert.Error(t, firelensResource.Create()) @@ -230,7 +232,7 @@ func TestCreateFirelensResourceCreateConfigDirError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) defer mockMkdirAllError()() @@ -243,7 +245,7 @@ func TestCreateFirelensResourceCreateSocketDirError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) defer mockMkdirAllError()() @@ -256,7 +258,7 @@ func TestCreateFirelensResourceGenerateConfigError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) firelensResource.containerToLogOptions = map[string]map[string]string{ "container": { "invalid": "invalid", @@ -272,7 +274,7 @@ func TestCreateFirelensResourceCreateTempFileError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) gomock.InOrder( mockIOUtil.EXPECT().TempFile(testResourceDir, tempFile).Return(nil, errors.New("test error")), @@ -287,7 +289,7 @@ func TestCreateFirelensResourceWriteConfigFileError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) mockFile.(*mock_oswrapper.MockFile).WriteImpl = func(bytes []byte) (i int, e error) { return 0, errors.New("test error") @@ -306,7 +308,7 @@ func TestCreateFirelensResourceChmodError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) mockFile.(*mock_oswrapper.MockFile).ChmodImpl = func(mode os.FileMode) error { return errors.New("test error") @@ -325,7 +327,7 @@ func TestCreateFirelensResourceRenameError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) gomock.InOrder( mockIOUtil.EXPECT().TempFile(testResourceDir, tempFile).Return(mockFile, nil), @@ -347,7 +349,7 @@ func TestCreateFirelensResourceWithS3Config(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) err := firelensResource.parseOptions(testFirelensOptionsS3) require.NoError(t, err) @@ -385,7 +387,7 @@ func TestCreateFirelensResourceWithS3ConfigMissingCredentials(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) err := firelensResource.parseOptions(testFirelensOptionsS3) require.NoError(t, err) @@ -403,7 +405,7 @@ func TestCreateFirelensResourceWithS3ConfigInvalidS3ARN(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) err := firelensResource.parseOptions(testFirelensOptionsS3) require.NoError(t, err) @@ -422,7 +424,7 @@ func TestCreateFirelensResourceWithS3ConfigDownloadFailure(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) err := firelensResource.parseOptions(testFirelensOptionsS3) require.NoError(t, err) @@ -450,7 +452,7 @@ func TestCleanupFirelensResource(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) assert.NoError(t, firelensResource.Cleanup()) } @@ -460,7 +462,7 @@ func TestCleanupFirelensResourceError(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, mockIOUtil, - mockCredentialsManager, mockS3ClientCreator) + mockCredentialsManager, mockS3ClientCreator, testContainerMemoryLimit) removeAll = func(path string) error { return errors.New("test error") @@ -478,7 +480,7 @@ func TestInitializeFirelensResource(t *testing.T) { defer done() firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, nil, nil, - nil) + nil, testContainerMemoryLimit) firelensResource.Initialize(&taskresource.ResourceFields{ ResourceFieldsCommon: &taskresource.ResourceFieldsCommon{ CredentialsManager: mockCredentialsManager, @@ -494,7 +496,7 @@ func TestInitializeFirelensResource(t *testing.T) { func TestSetKnownStatus(t *testing.T) { firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, nil, nil, - nil) + nil, testContainerMemoryLimit) firelensResource.appliedStatusUnsafe = resourcestatus.ResourceStatus(FirelensCreated) firelensResource.SetKnownStatus(resourcestatus.ResourceStatus(FirelensCreated)) @@ -504,7 +506,7 @@ func TestSetKnownStatus(t *testing.T) { func TestSetKnownStatusNoAppliedStatusUpdate(t *testing.T) { firelensResource := newMockFirelensResource(FirelensConfigTypeFluentd, bridgeNetworkMode, testFluentdOptions, nil, nil, - nil) + nil, testContainerMemoryLimit) firelensResource.appliedStatusUnsafe = resourcestatus.ResourceStatus(FirelensCreated) firelensResource.SetKnownStatus(resourcestatus.ResourceStatus(FirelensStatusNone)) diff --git a/agent/taskresource/firelens/firelensconfig_unix.go b/agent/taskresource/firelens/firelensconfig_unix.go index 30bd5c03214..a7ebe66b88d 100644 --- a/agent/taskresource/firelens/firelensconfig_unix.go +++ b/agent/taskresource/firelens/firelensconfig_unix.go @@ -113,28 +113,36 @@ const ( // specifies awsvpc type mode for a task awsvpcNetworkMode = "awsvpc" + + memBufLimitOptionFluentBit = "Mem_Buf_Limit" + + // the default mem buf limit (in MB) in case we cannot infer from the container memory reservation + defaultMemBufLimit = 25 ) // generateConfig generates a FluentConfig object that contains all necessary information to construct // a fluentd or fluentbit config file for a firelens container. func (firelens *FirelensResource) generateConfig() (generator.FluentConfig, error) { config := generator.New() - + var defaultInputMap map[string]string // Specify log stream input, which is a unix socket that will be used for communication between the Firelens // container and other containers. - var inputName, inputPathOption, matchAnyWildcard string + var inputName, matchAnyWildcard string if firelens.firelensConfigType == FirelensConfigTypeFluentd { inputName = socketInputNameFluentd - inputPathOption = socketInputPathOptionFluentd matchAnyWildcard = matchAnyWildcardFluentd + defaultInputMap = map[string]string{ + socketInputPathOptionFluentd: socketPath, + } } else { inputName = inputNameForward - inputPathOption = socketInputPathOptionFluentbit matchAnyWildcard = matchAnyWildcardFluentbit + defaultInputMap = map[string]string{ + socketInputPathOptionFluentbit: socketPath, + memBufLimitOptionFluentBit: firelens.resolveDefaultMemBufLimit(), + } } - config.AddInput(inputName, "", map[string]string{ - inputPathOption: socketPath, - }) + config.AddInput(inputName, "", defaultInputMap) // Specify log stream input of tcp socket kind that can be used for communication between the Firelens // container and other containers if the network is bridge or awsvpc mode. Also add health check sections to support // doing container health check on firlens container for these two modes. @@ -202,6 +210,16 @@ func (firelens *FirelensResource) generateConfig() (generator.FluentConfig, erro return config, nil } +func (firelens *FirelensResource) resolveDefaultMemBufLimit() string { + memBufLimit := firelens.containerMemoryLimit / 2 + if memBufLimit <= 0 { + memBufLimit = defaultMemBufLimit + } + memBufLimitString := fmt.Sprintf("%dMB", memBufLimit) + seelog.Infof("Resolved Firelens default mem buf limit for task %s: %s", firelens.taskARN, memBufLimitString) + return memBufLimitString +} + // addHealthcheckSections adds a health check input section and a health check output section to the config. func (firelens *FirelensResource) addHealthcheckSections(config generator.FluentConfig) { // Health check supported is only added for fluentbit. diff --git a/agent/taskresource/firelens/firelensconfig_unix_test.go b/agent/taskresource/firelens/firelensconfig_unix_test.go index 1a18f0cf49c..2791b560019 100644 --- a/agent/taskresource/firelens/firelensconfig_unix_test.go +++ b/agent/taskresource/firelens/firelensconfig_unix_test.go @@ -176,6 +176,55 @@ var ( expectedFluentbitConfig = ` [INPUT] Name forward + Mem_Buf_Limit 50MB + unix_path /var/run/fluent.sock + +[INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + +[INPUT] + Name tcp + Tag firelens-healthcheck + Listen 127.0.0.1 + Port 8877 + +[FILTER] + Name grep + Match container-firelens* + Regex log *failure* + +[FILTER] + Name grep + Match container-firelens* + Exclude log *success* + +[FILTER] + Name record_modifier + Match * + Record ec2_instance_id i-123456789a + Record ecs_cluster mycluster + Record ecs_task_arn arn:aws:ecs:us-east-2:01234567891011:task/mycluster/3de392df-6bfa-470b-97ed-aa6f482cd7a + Record ecs_task_definition taskdefinition:1 + +@INCLUDE /fluent-bit/etc/external.conf + +[OUTPUT] + Name null + Match firelens-healthcheck + +[OUTPUT] + Name kinesis_firehose + Match container-firelens* + deliver_stream_name my-stream + region us-west-2 +` + + expectedFluentbitConfigDefaultMemBufLimit = ` +[INPUT] + Name forward + Mem_Buf_Limit 25MB unix_path /var/run/fluent.sock [INPUT] @@ -259,6 +308,7 @@ var ( expectedFluentbitConfigWithoutOutputSection = ` [INPUT] Name forward + Mem_Buf_Limit 50MB unix_path /var/run/fluent.sock [INPUT] @@ -305,7 +355,7 @@ func TestGenerateFluentdBridgeModeConfig(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentd, testRegion, bridgeNetworkMode, testFirelensOptionsFile, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) config, err := firelensResource.generateConfig() @@ -324,7 +374,7 @@ func TestGenerateFluentdAWSVPCModeConfig(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentd, testRegion, awsvpcNetworkMode, testFirelensOptionsFile, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) config, err := firelensResource.generateConfig() @@ -343,7 +393,7 @@ func TestGenerateFluentdDefaultModeConfig(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentd, testRegion, "", testFirelensOptionsFile, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) config, err := firelensResource.generateConfig() @@ -362,7 +412,7 @@ func TestGenerateFluentbitConfig(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentbit, testRegion, bridgeNetworkMode, testFirelensOptionsS3, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) config, err := firelensResource.generateConfig() @@ -374,6 +424,25 @@ func TestGenerateFluentbitConfig(t *testing.T) { assert.Equal(t, expectedFluentbitConfig, configBytes.String()) } +func TestGenerateFluentbitConfigWithDefaultMemBufLimit(t *testing.T) { + containerToLogOptions := map[string]map[string]string{ + "container": testFluentbitOptions, + } + + firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, + testDataDir, FirelensConfigTypeFluentbit, testRegion, bridgeNetworkMode, testFirelensOptionsS3, containerToLogOptions, + nil, testExecutionCredentialsID, 0) + require.NoError(t, err) + + config, err := firelensResource.generateConfig() + assert.NoError(t, err) + + configBytes := new(bytes.Buffer) + err = config.WriteFluentBitConfig(configBytes) + assert.NoError(t, err) + assert.Equal(t, expectedFluentbitConfigDefaultMemBufLimit, configBytes.String()) +} + func TestGenerateFluentdConfigMissingOutputName(t *testing.T) { containerToLogOptions := map[string]map[string]string{ "container": { @@ -383,7 +452,7 @@ func TestGenerateFluentdConfigMissingOutputName(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentd, testRegion, bridgeNetworkMode, testFirelensOptionsFile, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) _, err = firelensResource.generateConfig() @@ -399,7 +468,7 @@ func TestGenerateFLuentbitConfigMissingOutputName(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentbit, testRegion, bridgeNetworkMode, testFirelensOptionsFile, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) _, err = firelensResource.generateConfig() @@ -418,7 +487,7 @@ func TestGenerateConfigWithECSMetadataDisabled(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentd, testRegion, bridgeNetworkMode, testFirelensOptions, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) config, err := firelensResource.generateConfig() @@ -440,7 +509,7 @@ func TestGenerateConfigWithoutOutputSection(t *testing.T) { firelensResource, err := NewFirelensResource(testCluster, testTaskARN, testTaskDefinition, testEC2InstanceID, testDataDir, FirelensConfigTypeFluentbit, testRegion, bridgeNetworkMode, testFirelensOptionsS3, containerToLogOptions, - nil, testExecutionCredentialsID) + nil, testExecutionCredentialsID, testContainerMemoryReservation) require.NoError(t, err) config, err := firelensResource.generateConfig()