diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/README.md b/ecs-task-definition-templates/deployment-mode/daemon-service/README.md index 10e08a5..18c72bd 100644 --- a/ecs-task-definition-templates/deployment-mode/daemon-service/README.md +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/README.md @@ -5,4 +5,10 @@ This folder contains the example Amazon ECS task definitions for DaemonService d Check the sub folders for different functionality: ### [cwagent-ecs-instance-metric](cwagent-ecs-instance-metric) -This folder provides the functionality that enables you to deploy the CloudWatch Agent to collect ECS Instance Metrics. \ No newline at end of file +This folder provides the functionality that enables you to deploy the CloudWatch Agent to collect ECS Instance Metrics. + +### [fluent-bit-ecs-instance-logs]( fluent-bit-ecs-instance-logs) +This folder provides the functionality that enables you to deploy Fluent-Bit to collect ECS Container Instance. + +### [fluent-bit-cwagent-combined-cfn]( fluent-bit-cwagent-combined-cfn) +This folder provides the functionality that enables you to deploy both the CloudWatch Agent and Fluent-Bit in a single CloudFormation stack. \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/README.md b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/README.md new file mode 100644 index 0000000..32ce841 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/README.md @@ -0,0 +1,19 @@ +## CloudWatch Agent and Fluent-Bit for ECS Instance Metrics and Logs Collection Quick Start + +The cloudformation template in this folder helps you to quickly deploy both the CloudWatch agent and Fluent-Bit as daemon-services using a single CloudFormation stack to collect ECS Container Instance Metrics and Logs. + +* [fluent-bit-cwagent-combined-cfn.yaml](fluent-bit-cwagent-combined-cfn.yaml): sample cloudformation template + + +Run following aws cloudformation command with the cloudformation template file to deploy the Fluent-Bit and CloudWatch agent Daemonsets with required IAM roles. ***Please assign the actual ECS cluster name and the cluster region in the first two lines of the command separately.*** + +``` +ClusterName= +Region= +aws cloudformation create-stack --stack-name CloudWatch-Fluent-Bit-${ClusterName}-${Region} \ + --template-body file://fluent-bit-cwagent-combined-cfn.yaml \ + --parameters ParameterKey=ClusterName,ParameterValue=${ClusterName} \ + ParameterKey=CreateIAMRoles,ParameterValue=True \ + --capabilities CAPABILITY_NAMED_IAM \ + --region ${Region} +``` \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/fluent-bit-cwagent-combined-cfn.yaml b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/fluent-bit-cwagent-combined-cfn.yaml new file mode 100644 index 0000000..80d34a7 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/fluent-bit-cwagent-combined-cfn.yaml @@ -0,0 +1,328 @@ +AWSTemplateFormatVersion: '2010-09-09' +Parameters: + ClusterName: + Type: String + Description: Enter the name of your ECS cluster from which you want to collect metrics and/or logs + CreateIAMRoles: + Type: String + Default: 'False' + AllowedValues: + - 'True' + - 'False' + Description: Whether to create default IAM roles + ConstraintDescription: must specify True or False. + CWAgentTaskRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent ecs task role + CWAgentExecutionRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent execution role + FluentBitTaskRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent ecs task role + FluentBitExecutionRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent execution role + FluentBitConfigFile: + Type: String + Default: Default + Description: Enter the S3 URI for the fluent-bit config file (fluent bit task role must have permissions to download the file) + FluentBitParsersFile: + Type: String + Default: Default + Description: Enter the S3 URI for the fluent-bit parsers file (fluent bit task role must have permissions to download the file) +Conditions: + CreateRoles: !Equals + - !Ref CreateIAMRoles + - 'True' + DefaultCWAgentTaskRole: !Equals + - !Ref CWAgentTaskRoleArn + - Default + DefaultCWAgentExecutionRole: !Equals + - !Ref CWAgentExecutionRoleArn + - Default + DefaultFluentBitTaskRole: !Equals + - !Ref FluentBitTaskRoleArn + - Default + DefaultFluentBitExecutionRole: !Equals + - !Ref FluentBitExecutionRoleArn + - Default + DefaultFluentBitConfigFile: !Equals + - !Ref FluentBitConfigFile + - Default + DefaultFluentBitParsersFile: !Equals + - !Ref FluentBitParsersFile + - Default +Resources: + CWAgentECSTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + TaskRoleArn: !If + - CreateRoles + - !GetAtt CWAgentECSTaskRole.Arn + - !If + - DefaultCWAgentTaskRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/CWAgentECSTaskRole + - !Ref CWAgentTaskRoleArn + ExecutionRoleArn: !If + - CreateRoles + - !GetAtt CWAgentECSExecutionRole.Arn + - !If + - DefaultCWAgentExecutionRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/CWAgentECSExecutionRole + - !Ref CWAgentExecutionRoleArn + NetworkMode: bridge + ContainerDefinitions: + - Name: cloudwatch-agent + Image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300042.1b746 + MountPoints: + - ReadOnly: true + ContainerPath: /rootfs/proc + SourceVolume: proc + - ReadOnly: true + ContainerPath: /rootfs/dev + SourceVolume: dev + - ReadOnly: true + ContainerPath: /sys/fs/cgroup + SourceVolume: al2_cgroup + - ReadOnly: true + ContainerPath: /cgroup + SourceVolume: al1_cgroup + - ReadOnly: true + ContainerPath: /rootfs/sys/fs/cgroup + SourceVolume: al2_cgroup + - ReadOnly: true + ContainerPath: /rootfs/cgroup + SourceVolume: al1_cgroup + Environment: + - Name: USE_DEFAULT_CONFIG + Value: 'True' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-cwagent-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + RequiresCompatibilities: + - EC2 + Volumes: + - Name: proc + Host: + SourcePath: /proc + - Name: dev + Host: + SourcePath: /dev + - Name: al1_cgroup + Host: + SourcePath: /cgroup + - Name: al2_cgroup + Host: + SourcePath: /sys/fs/cgroup + Cpu: '128' + Memory: '64' + CWAgentECSDaemonService: + Type: AWS::ECS::Service + Properties: + TaskDefinition: !Ref CWAgentECSTaskDefinition + Cluster: !Ref ClusterName + LaunchType: EC2 + SchedulingStrategy: DAEMON + ServiceName: cwagent-daemon-service + CWAgentECSTaskRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS tasks to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + RoleName: CWAgentECSTaskRole + CWAgentECSExecutionRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS container agent makes calls to the Amazon ECS API on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + RoleName: CWAgentECSExecutionRole + FluentBitTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + TaskRoleArn: !If + - CreateRoles + - !GetAtt FluentBitTaskRole.Arn + - !If + - DefaultFluentBitTaskRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/FluentBitECSTaskRole + - !Ref FluentBitTaskRoleArn + ExecutionRoleArn: !If + - CreateRoles + - !GetAtt FluentBitExecutionRole.Arn + - !If + - DefaultFluentBitExecutionRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/FluentBitECSExecutionRole + - !Ref FluentBitExecutionRoleArn + NetworkMode: bridge + ContainerDefinitions: + - Name: fluent-bit + Image: fluent/fluent-bit:3.1.4 + MountPoints: + - ContainerPath: /fluent-bit/etc/ + SourceVolume: fluent-bit-config + - ContainerPath: /var/fluent-bit/state + SourceVolume: fluent-bit-state + - ReadOnly: true + ContainerPath: /var/log + SourceVolume: instance-logs + - ReadOnly: true + ContainerPath: /run/log/journal + SourceVolume: systemd-journal + - ReadOnly: true + ContainerPath: /etc/machine-id + SourceVolume: machine-id + Environment: + - Name: AWS_REGION + Value: !Ref AWS::Region + - Name: CLUSTER_NAME + Value: !Ref ClusterName + DependsOn: + - ContainerName: fluent-bit-config-init + Condition: COMPLETE + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-fluent-bit-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + - Name: fluent-bit-config-init + Image: public.ecr.aws/aws-cli/aws-cli:2.18.14 + Essential: False + EntryPoint: + - '/bin/sh' + - '-c' + Command: + - | + set -ex && if [ -z $( ls -A '/run/log/journal' ) ]; then echo @SET SYSTEMD_PATH=/var/log/journal >> /fluent-bit/etc/fluent-bit.conf; else echo @SET SYSTEMD_PATH=/run/log/journal >> /fluent-bit/etc/fluent-bit.conf; fi && echo @SET INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id) >> /fluent-bit/etc/fluent-bit.conf && if [ -z "${FLUENT_BIT_CONFIG_FILE}" ]; then echo 'W1NFUlZJQ0VdCiAgICBGbHVzaCAgICAgICAgICAgICAgICAgICAgIDUKICAgIEdyYWNlICAgICAgICAgICAgICAgICAgICAgMzAKICAgIExvZ19MZXZlbCAgICAgICAgICAgICAgICAgZXJyb3IKICAgIERhZW1vbiAgICAgICAgICAgICAgICAgICAgb2ZmCiAgICBQYXJzZXJzX0ZpbGUgICAgICAgICAgICAgIHBhcnNlcnMuY29uZgogICAgc3RvcmFnZS5wYXRoICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiLXN0b3JhZ2UvCiAgICBzdG9yYWdlLnN5bmMgICAgICAgICAgICAgIG5vcm1hbAogICAgc3RvcmFnZS5jaGVja3N1bSAgICAgICAgICBvZmYKICAgIHN0b3JhZ2UuYmFja2xvZy5tZW1fbGltaXQgNU0KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzdGVtZAogICAgVGFnICAgICAgICAgICAgICAgICBzeXN0ZW1kLSoKICAgIFN5c3RlbWRfRmlsdGVyICAgICAgX1NZU1RFTURfVU5JVD1kb2NrZXIuc2VydmljZQogICAgU3lzdGVtZF9GaWx0ZXIgICAgICBfU1lTVEVNRF9VTklUPWNvbnRhaW5lcmQuc2VydmljZQogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvc3lzdGVtZC5kYgogICAgUGF0aCAgICAgICAgICAgICAgICAke1NZU1RFTURfUEFUSH0KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzdGVtZAogICAgVGFnICAgICAgICAgICAgICAgICBzeXN0ZW1kLXN5c2xvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvc3lzdGVtZC1zeXNsb2cuZGIKICAgIFBhdGggICAgICAgICAgICAgICAgJHtTWVNURU1EX1BBVEh9CiAgICAKW0lOUFVUXQogICAgTmFtZSAgICAgICAgICAgICAgICBzeXN0ZW1kCiAgICBUYWcgICAgICAgICAgICAgICAgIHN5c3RlbWQtZG1lc2cuKgogICAgU3lzdGVtZF9GaWx0ZXIgICAgICBfVFJBTlNQT1JUPWtlcm5lbAogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZG1lc2cuZGIKICAgIFBhdGggICAgICAgICAgICAgICAgJHtTWVNURU1EX1BBVEh9CgpbSU5QVVRdCiAgICBOYW1lICAgICAgICAgICAgICAgIHRhaWwKICAgIFRhZyAgICAgICAgICAgICAgICAgc2VjdXJlCiAgICBQYXRoICAgICAgICAgICAgICAgIC92YXIvbG9nL3NlY3VyZQogICAgUGFyc2VyICAgICAgICAgICAgICBzeXNsb2cKICAgIERCICAgICAgICAgICAgICAgICAgL3Zhci9mbHVlbnQtYml0L3N0YXRlL2ZsYl9zZWN1cmUuZGIKICAgIE1lbV9CdWZfTGltaXQgICAgICAgNU1CCiAgICBTa2lwX0xvbmdfTGluZXMgICAgIE9uCiAgICBSZWZyZXNoX0ludGVydmFsICAgIDEwCiAgICBSZWFkX2Zyb21fSGVhZCAgICAgIE9uCgpbSU5QVVRdCiAgICBOYW1lICAgICAgICAgICAgICAgIHRhaWwKICAgIFRhZyAgICAgICAgICAgICAgICAgZWNzLWFnZW50LmxvZwogICAgUGF0aCAgICAgICAgICAgICAgICAvdmFyL2xvZy9lY3MvZWNzLWFnZW50LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19hZ2VudC5kYgogICAgTWVtX0J1Zl9MaW1pdCAgICAgICA1TUIKICAgIFNraXBfTG9uZ19MaW5lcyAgICAgT24KICAgIFJlZnJlc2hfSW50ZXJ2YWwgICAgMTAKICAgIFJlYWRfZnJvbV9IZWFkICAgICAgT24KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgdGFpbAogICAgVGFnICAgICAgICAgICAgICAgICBlY3MtaW5pdC5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2Vjcy1pbml0LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19pbml0LmRiCiAgICBNZW1fQnVmX0xpbWl0ICAgICAgIDVNQgogICAgU2tpcF9Mb25nX0xpbmVzICAgICBPbgogICAgUmVmcmVzaF9JbnRlcnZhbCAgICAxMAogICAgUmVhZF9mcm9tX0hlYWQgICAgICBPbgoKW0lOUFVUXQogICAgTmFtZSAgICAgICAgICAgICAgICB0YWlsCiAgICBUYWcgICAgICAgICAgICAgICAgIGVjcy1hdWRpdC5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2F1ZGl0LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19hdWRpdC5kYgogICAgTWVtX0J1Zl9MaW1pdCAgICAgICA1TUIKICAgIFNraXBfTG9uZ19MaW5lcyAgICAgT24KICAgIFJlZnJlc2hfSW50ZXJ2YWwgICAgMTAKICAgIFJlYWRfZnJvbV9IZWFkICAgICAgT24KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgdGFpbAogICAgVGFnICAgICAgICAgICAgICAgICBlY3Mtdm9sdW1lLXBsdWdpbi5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2Vjcy12b2x1bWUtcGx1Z2luLmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc192b2x1bWVfcGx1Z2luLmRiCiAgICBNZW1fQnVmX0xpbWl0ICAgICAgIDVNQgogICAgU2tpcF9Mb25nX0xpbmVzICAgICBPbgogICAgUmVmcmVzaF9JbnRlcnZhbCAgICAxMAogICAgUmVhZF9mcm9tX0hlYWQgICAgICBPbiAKCltGSUxURVJdCiAgICBOYW1lICAgICAgICAgICAgICAgIG1vZGlmeQogICAgTWF0Y2ggICAgICAgICAgICAgICBzeXN0ZW1kLSoKICAgIFJlbmFtZSAgICAgICAgICAgICAgX0hPU1ROQU1FICAgICAgICAgICAgICAgICAgIGhvc3RuYW1lCiAgICBSZW5hbWUgICAgICAgICAgICAgIF9TWVNURU1EX1VOSVQgICAgICAgICAgICAgICBzeXN0ZW1kX3VuaXQKICAgIFJlbmFtZSAgICAgICAgICAgICAgTUVTU0FHRSAgICAgICAgICAgICAgICAgICAgIG1lc3NhZ2UKICAgIFJlbW92ZV9yZWdleCAgICAgICAgXigoPyFob3N0bmFtZXxzeXN0ZW1kX3VuaXR8bWVzc2FnZSkuKSokCgpbRklMVEVSXQogICAgTmFtZSAgICAgICAgICAgICAgICBhd3MKICAgIE1hdGNoICAgICAgICAgICAgICAgKgogICAgaW1kc192ZXJzaW9uICAgICAgICB2MgoKW09VVFBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgY2xvdWR3YXRjaF9sb2dzCiAgICBNYXRjaCAgICAgICAgICAgICAgICoKICAgIHJlZ2lvbiAgICAgICAgICAgICAgJHtBV1NfUkVHSU9OfQogICAgbG9nX2dyb3VwX25hbWUgICAgICAvYXdzL2Vjcy9jb250YWluZXJpbnNpZ2h0cy8ke0NMVVNURVJfTkFNRX0vaW5zdGFuY2UtbG9ncwogICAgbG9nX3N0cmVhbV9wcmVmaXggICAke0lOU1RBTkNFX0lEfS0KICAgIGF1dG9fY3JlYXRlX2dyb3VwICAgdHJ1ZQ==' | base64 -d >> /fluent-bit/etc/fluent-bit.conf; else aws s3 cp $FLUENT_BIT_CONFIG_FILE /tmp/fluent-bit-temp.conf && cat /tmp/fluent-bit-temp.conf >> /fluent-bit/etc/fluent-bit.conf; fi && if [ -z "${FLUENT_BIT_PARSERS_FILE}" ]; then echo 'W1BBUlNFUl0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzbG9nCiAgICBGb3JtYXQgICAgICAgICAgICAgIHJlZ2V4CiAgICBSZWdleCAgICAgICAgICAgICAgIF4oPzx0aW1lPlteIF0qIHsxLDJ9W14gXSogW14gXSopICg/PGhvc3Q+W14gXSopICg/PGlkZW50PlthLXpBLVowLTlfXC9cLlwtXSopKD86XFsoPzxwaWQ+WzAtOV0rKVxdKT8oPzpbXlw6XSpcOik/ICooPzxtZXNzYWdlPi4qKSQKICAgIFRpbWVfS2V5ICAgICAgICAgICAgdGltZQogICAgVGltZV9Gb3JtYXQgICAgICAgICAlYiAlZCAlSDolTTolUw==' | base64 -d >> /fluent-bit/etc/parsers.conf; else aws s3 cp $FLUENT_BIT_PARSERS_FILE /tmp/parsers-temp.conf && cat /tmp/parsers-temp.conf >> /fluent-bit/etc/parsers.conf; fi + Environment: + - !If + - DefaultFluentBitConfigFile + - !Ref AWS::NoValue + - Name: FLUENT_BIT_CONFIG_FILE + Value: !Ref FluentBitConfigFile + - !If + - DefaultFluentBitParsersFile + - !Ref AWS::NoValue + - Name: FLUENT_BIT_PARSERS_FILE + Value: !Ref FluentBitParsersFile + MountPoints: + - ContainerPath: /fluent-bit/etc/ + SourceVolume: fluent-bit-config + - ReadOnly: true + ContainerPath: /var/lib/cloud/data/instance-id + SourceVolume: instance-id + - ReadOnly: true + ContainerPath: /run/log/journal + SourceVolume: systemd-journal + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-fluent-bit-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + RequiresCompatibilities: + - EC2 + Volumes: + - Name: fluent-bit-config + DockerVolumeConfiguration: + Scope: 'task' + - Name: instance-id + Host: + SourcePath: /var/lib/cloud/data/instance-id + - Name: fluent-bit-state + Host: + SourcePath: /var/fluent-bit/state + - Name: instance-logs + Host: + SourcePath: /var/log + - Name: systemd-journal + Host: + SourcePath: /run/log/journal + - Name: machine-id + Host: + SourcePath: /etc/machine-id + Cpu: '512' + Memory: '256' + FluentBitECSDaemonService: + Type: AWS::ECS::Service + Properties: + TaskDefinition: !Ref FluentBitTaskDefinition + Cluster: !Ref ClusterName + LaunchType: EC2 + SchedulingStrategy: DAEMON + ServiceName: fluent-bit-daemon-service + FluentBitTaskRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS tasks to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: fluent-bit-task-role-policy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: FluentBitLogs + Effect: Allow + Action: + - 'logs:CreateLogStream' + - 'logs:CreateLogGroup' + - 'logs:PutLogEvents' + Resource: '*' + RoleName: FluentBitECSTaskRole + FluentBitExecutionRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS container agent makes calls to the Amazon ECS API on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + RoleName: FluentBitECSExecutionRole \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/README.md b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/README.md new file mode 100644 index 0000000..28fbd2c --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/README.md @@ -0,0 +1,41 @@ +## Fluent-Bit for ECS Instance Logs Collection + +The sample ECS task definition in this folder deploys Fluent-Bit as a DaemonService. + +* [fluent-bit-ecs-instance-logs.json](fluent-bit-ecs-instance-logs.json): sample ECS task definition +* [fluent-bit.conf](fluent-bit.conf): default fluent-bit configuration +* [parsers.conf](parsers.conf): default fluent-bit parsers file + +You must replace all the placeholders (with ```{{ }}```) in the above task definitions with your information: +* ```{{task-role-arn}}```: ECS task role ARN. + * This is role that you application containers will use. The permission should be whatever your applications need. + * Ensure that your Task role has the below permissions attached in order to access CloudWatch (for log ingestion). + ``` + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents" + ], + "Resource": "*", + "Effect": "Allow", + "Sid": "FluentBitLogs" + } + ] + } + ``` + +* ```{{execution-role-arn}}```: ECS task execution role ARN. + * This is the role that Amazon ECS agent requires to launch/execute your containers. + * Ensure that the ```AmazonSSMReadOnlyAccess```, ```AmazonECSTaskExecutionRolePolicy``` and ```CloudWatchAgentServerPolicy``` policies are attached to your ECS Task execution role. + * If you would like to store more sensitive data for ECS to use, refer to https://docs.aws.amazon.com/AmazonECS/latest/developerguide/specifying-sensitive-data.html. + +* ```{{aws-region}}```: The AWS region where the ECS Daemonset will be deployed: e.g. ```us-west-2``` + +* ```{{ecs-cluster-name}}```: The name of the ECS cluster in which you are deploying the Fluent-Bit Daemonset. + + +You can also adjust the resource limit (e.g. cpu and memory) based on your particular use cases. \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/README.md b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/README.md new file mode 100644 index 0000000..2036c74 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/README.md @@ -0,0 +1,19 @@ +## CloudWatch Agent for ECS Instance Logs Collection Quick Start + +The cloudformation template in this folder helps you to quickly deploy Fluent-Bit as a daemon-service to collect ECS Container Instance Logs. + +* [fluent-bit-ecs-instance-logs-cfn.yaml](fluent-bit-ecs-instance-logs-cfn.yaml): sample cloudformation template + + +Run following aws cloudformation command with the cloudformation template file to deploy the Fluent-Bit Daemonset with required IAM roles. ***Please assign the actual ECS cluster name and the cluster region in the first two lines of the command separately.*** + +``` +ClusterName= +Region= +aws cloudformation create-stack --stack-name Fluent-Bit-${ClusterName}-${Region} \ + --template-body file://fluent-bit-ecs-instance-logs-cfn.yaml \ + --parameters ParameterKey=ClusterName,ParameterValue=${ClusterName} \ + ParameterKey=CreateIAMRoles,ParameterValue=True \ + --capabilities CAPABILITY_NAMED_IAM \ + --region ${Region} +``` \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/fluent-bit-ecs-instance-logs-cfn.yaml b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/fluent-bit-ecs-instance-logs-cfn.yaml new file mode 100644 index 0000000..c1cdac9 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/fluent-bit-ecs-instance-logs-cfn.yaml @@ -0,0 +1,206 @@ +AWSTemplateFormatVersion: '2010-09-09' +Parameters: + ClusterName: + Type: String + Description: Enter the name of your ECS cluster from which you want to collect metrics and/or logs + CreateIAMRoles: + Type: String + Default: 'False' + AllowedValues: + - 'True' + - 'False' + Description: Whether to create default IAM roles + ConstraintDescription: must specify True or False. + FluentBitTaskRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent ecs task role + FluentBitExecutionRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent execution role + FluentBitConfigFile: + Type: String + Default: Default + Description: Enter the S3 URI for the fluent-bit config file (fluent bit task role must have permissions to download the file) + FluentBitParsersFile: + Type: String + Default: Default + Description: Enter the S3 URI for the fluent-bit parsers file (fluent bit task role must have permissions to download the file) +Conditions: + CreateRoles: !Equals + - !Ref CreateIAMRoles + - 'True' + DefaultFluentBitTaskRole: !Equals + - !Ref FluentBitTaskRoleArn + - Default + DefaultFluentBitExecutionRole: !Equals + - !Ref FluentBitExecutionRoleArn + - Default + DefaultFluentBitConfigFile: !Equals + - !Ref FluentBitConfigFile + - Default + DefaultFluentBitParsersFile: !Equals + - !Ref FluentBitParsersFile + - Default +Resources: + FluentBitTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + TaskRoleArn: !If + - CreateRoles + - !GetAtt FluentBitTaskRole.Arn + - !If + - DefaultFluentBitTaskRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/FluentBitECSTaskRole + - !Ref FluentBitTaskRoleArn + ExecutionRoleArn: !If + - CreateRoles + - !GetAtt FluentBitExecutionRole.Arn + - !If + - DefaultFluentBitExecutionRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/FluentBitECSExecutionRole + - !Ref FluentBitExecutionRoleArn + NetworkMode: bridge + ContainerDefinitions: + - Name: fluent-bit + Image: fluent/fluent-bit:3.1.4 + MountPoints: + - ContainerPath: /fluent-bit/etc/ + SourceVolume: fluent-bit-config + - ContainerPath: /var/fluent-bit/state + SourceVolume: fluent-bit-state + - ReadOnly: true + ContainerPath: /var/log + SourceVolume: instance-logs + - ReadOnly: true + ContainerPath: /run/log/journal + SourceVolume: systemd-journal + - ReadOnly: true + ContainerPath: /etc/machine-id + SourceVolume: machine-id + Environment: + - Name: AWS_REGION + Value: !Ref AWS::Region + - Name: CLUSTER_NAME + Value: !Ref ClusterName + DependsOn: + - ContainerName: fluent-bit-config-init + Condition: COMPLETE + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-fluent-bit-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + - Name: fluent-bit-config-init + Image: public.ecr.aws/aws-cli/aws-cli:2.18.14 + Essential: False + EntryPoint: + - '/bin/sh' + - '-c' + Command: + - | + set -ex && if [ -z $( ls -A '/run/log/journal' ) ]; then echo @SET SYSTEMD_PATH=/var/log/journal >> /fluent-bit/etc/fluent-bit.conf; else echo @SET SYSTEMD_PATH=/run/log/journal >> /fluent-bit/etc/fluent-bit.conf; fi && echo @SET INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id) >> /fluent-bit/etc/fluent-bit.conf && if [ -z "${FLUENT_BIT_CONFIG_FILE}" ]; then echo 'W1NFUlZJQ0VdCiAgICBGbHVzaCAgICAgICAgICAgICAgICAgICAgIDUKICAgIEdyYWNlICAgICAgICAgICAgICAgICAgICAgMzAKICAgIExvZ19MZXZlbCAgICAgICAgICAgICAgICAgZXJyb3IKICAgIERhZW1vbiAgICAgICAgICAgICAgICAgICAgb2ZmCiAgICBQYXJzZXJzX0ZpbGUgICAgICAgICAgICAgIHBhcnNlcnMuY29uZgogICAgc3RvcmFnZS5wYXRoICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiLXN0b3JhZ2UvCiAgICBzdG9yYWdlLnN5bmMgICAgICAgICAgICAgIG5vcm1hbAogICAgc3RvcmFnZS5jaGVja3N1bSAgICAgICAgICBvZmYKICAgIHN0b3JhZ2UuYmFja2xvZy5tZW1fbGltaXQgNU0KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzdGVtZAogICAgVGFnICAgICAgICAgICAgICAgICBzeXN0ZW1kLSoKICAgIFN5c3RlbWRfRmlsdGVyICAgICAgX1NZU1RFTURfVU5JVD1kb2NrZXIuc2VydmljZQogICAgU3lzdGVtZF9GaWx0ZXIgICAgICBfU1lTVEVNRF9VTklUPWNvbnRhaW5lcmQuc2VydmljZQogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvc3lzdGVtZC5kYgogICAgUGF0aCAgICAgICAgICAgICAgICAke1NZU1RFTURfUEFUSH0KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzdGVtZAogICAgVGFnICAgICAgICAgICAgICAgICBzeXN0ZW1kLXN5c2xvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvc3lzdGVtZC1zeXNsb2cuZGIKICAgIFBhdGggICAgICAgICAgICAgICAgJHtTWVNURU1EX1BBVEh9CiAgICAKW0lOUFVUXQogICAgTmFtZSAgICAgICAgICAgICAgICBzeXN0ZW1kCiAgICBUYWcgICAgICAgICAgICAgICAgIHN5c3RlbWQtZG1lc2cuKgogICAgU3lzdGVtZF9GaWx0ZXIgICAgICBfVFJBTlNQT1JUPWtlcm5lbAogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZG1lc2cuZGIKICAgIFBhdGggICAgICAgICAgICAgICAgJHtTWVNURU1EX1BBVEh9CgpbSU5QVVRdCiAgICBOYW1lICAgICAgICAgICAgICAgIHRhaWwKICAgIFRhZyAgICAgICAgICAgICAgICAgc2VjdXJlCiAgICBQYXRoICAgICAgICAgICAgICAgIC92YXIvbG9nL3NlY3VyZQogICAgUGFyc2VyICAgICAgICAgICAgICBzeXNsb2cKICAgIERCICAgICAgICAgICAgICAgICAgL3Zhci9mbHVlbnQtYml0L3N0YXRlL2ZsYl9zZWN1cmUuZGIKICAgIE1lbV9CdWZfTGltaXQgICAgICAgNU1CCiAgICBTa2lwX0xvbmdfTGluZXMgICAgIE9uCiAgICBSZWZyZXNoX0ludGVydmFsICAgIDEwCiAgICBSZWFkX2Zyb21fSGVhZCAgICAgIE9uCgpbSU5QVVRdCiAgICBOYW1lICAgICAgICAgICAgICAgIHRhaWwKICAgIFRhZyAgICAgICAgICAgICAgICAgZWNzLWFnZW50LmxvZwogICAgUGF0aCAgICAgICAgICAgICAgICAvdmFyL2xvZy9lY3MvZWNzLWFnZW50LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19hZ2VudC5kYgogICAgTWVtX0J1Zl9MaW1pdCAgICAgICA1TUIKICAgIFNraXBfTG9uZ19MaW5lcyAgICAgT24KICAgIFJlZnJlc2hfSW50ZXJ2YWwgICAgMTAKICAgIFJlYWRfZnJvbV9IZWFkICAgICAgT24KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgdGFpbAogICAgVGFnICAgICAgICAgICAgICAgICBlY3MtaW5pdC5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2Vjcy1pbml0LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19pbml0LmRiCiAgICBNZW1fQnVmX0xpbWl0ICAgICAgIDVNQgogICAgU2tpcF9Mb25nX0xpbmVzICAgICBPbgogICAgUmVmcmVzaF9JbnRlcnZhbCAgICAxMAogICAgUmVhZF9mcm9tX0hlYWQgICAgICBPbgoKW0lOUFVUXQogICAgTmFtZSAgICAgICAgICAgICAgICB0YWlsCiAgICBUYWcgICAgICAgICAgICAgICAgIGVjcy1hdWRpdC5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2F1ZGl0LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19hdWRpdC5kYgogICAgTWVtX0J1Zl9MaW1pdCAgICAgICA1TUIKICAgIFNraXBfTG9uZ19MaW5lcyAgICAgT24KICAgIFJlZnJlc2hfSW50ZXJ2YWwgICAgMTAKICAgIFJlYWRfZnJvbV9IZWFkICAgICAgT24KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgdGFpbAogICAgVGFnICAgICAgICAgICAgICAgICBlY3Mtdm9sdW1lLXBsdWdpbi5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2Vjcy12b2x1bWUtcGx1Z2luLmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc192b2x1bWVfcGx1Z2luLmRiCiAgICBNZW1fQnVmX0xpbWl0ICAgICAgIDVNQgogICAgU2tpcF9Mb25nX0xpbmVzICAgICBPbgogICAgUmVmcmVzaF9JbnRlcnZhbCAgICAxMAogICAgUmVhZF9mcm9tX0hlYWQgICAgICBPbiAKCltGSUxURVJdCiAgICBOYW1lICAgICAgICAgICAgICAgIG1vZGlmeQogICAgTWF0Y2ggICAgICAgICAgICAgICBzeXN0ZW1kLSoKICAgIFJlbmFtZSAgICAgICAgICAgICAgX0hPU1ROQU1FICAgICAgICAgICAgICAgICAgIGhvc3RuYW1lCiAgICBSZW5hbWUgICAgICAgICAgICAgIF9TWVNURU1EX1VOSVQgICAgICAgICAgICAgICBzeXN0ZW1kX3VuaXQKICAgIFJlbmFtZSAgICAgICAgICAgICAgTUVTU0FHRSAgICAgICAgICAgICAgICAgICAgIG1lc3NhZ2UKICAgIFJlbW92ZV9yZWdleCAgICAgICAgXigoPyFob3N0bmFtZXxzeXN0ZW1kX3VuaXR8bWVzc2FnZSkuKSokCgpbRklMVEVSXQogICAgTmFtZSAgICAgICAgICAgICAgICBhd3MKICAgIE1hdGNoICAgICAgICAgICAgICAgKgogICAgaW1kc192ZXJzaW9uICAgICAgICB2MgoKW09VVFBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgY2xvdWR3YXRjaF9sb2dzCiAgICBNYXRjaCAgICAgICAgICAgICAgICoKICAgIHJlZ2lvbiAgICAgICAgICAgICAgJHtBV1NfUkVHSU9OfQogICAgbG9nX2dyb3VwX25hbWUgICAgICAvYXdzL2Vjcy9jb250YWluZXJpbnNpZ2h0cy8ke0NMVVNURVJfTkFNRX0vaW5zdGFuY2UtbG9ncwogICAgbG9nX3N0cmVhbV9wcmVmaXggICAke0lOU1RBTkNFX0lEfS0KICAgIGF1dG9fY3JlYXRlX2dyb3VwICAgdHJ1ZQ==' | base64 -d >> /fluent-bit/etc/fluent-bit.conf; else aws s3 cp $FLUENT_BIT_CONFIG_FILE /tmp/fluent-bit-temp.conf && cat /tmp/fluent-bit-temp.conf >> /fluent-bit/etc/fluent-bit.conf; fi && if [ -z "${FLUENT_BIT_PARSERS_FILE}" ]; then echo 'W1BBUlNFUl0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzbG9nCiAgICBGb3JtYXQgICAgICAgICAgICAgIHJlZ2V4CiAgICBSZWdleCAgICAgICAgICAgICAgIF4oPzx0aW1lPlteIF0qIHsxLDJ9W14gXSogW14gXSopICg/PGhvc3Q+W14gXSopICg/PGlkZW50PlthLXpBLVowLTlfXC9cLlwtXSopKD86XFsoPzxwaWQ+WzAtOV0rKVxdKT8oPzpbXlw6XSpcOik/ICooPzxtZXNzYWdlPi4qKSQKICAgIFRpbWVfS2V5ICAgICAgICAgICAgdGltZQogICAgVGltZV9Gb3JtYXQgICAgICAgICAlYiAlZCAlSDolTTolUw==' | base64 -d >> /fluent-bit/etc/parsers.conf; else aws s3 cp $FLUENT_BIT_PARSERS_FILE /tmp/parsers-temp.conf && cat /tmp/parsers-temp.conf >> /fluent-bit/etc/parsers.conf; fi + Environment: + - !If + - DefaultFluentBitConfigFile + - !Ref AWS::NoValue + - Name: FLUENT_BIT_CONFIG_FILE + Value: !Ref FluentBitConfigFile + - !If + - DefaultFluentBitParsersFile + - !Ref AWS::NoValue + - Name: FLUENT_BIT_PARSERS_FILE + Value: !Ref FluentBitParsersFile + MountPoints: + - ContainerPath: /fluent-bit/etc/ + SourceVolume: fluent-bit-config + - ReadOnly: true + ContainerPath: /var/lib/cloud/data/instance-id + SourceVolume: instance-id + - ReadOnly: true + ContainerPath: /run/log/journal + SourceVolume: systemd-journal + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-fluent-bit-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + RequiresCompatibilities: + - EC2 + Volumes: + - Name: fluent-bit-config + DockerVolumeConfiguration: + Scope: 'task' + - Name: instance-id + Host: + SourcePath: /var/lib/cloud/data/instance-id + - Name: fluent-bit-state + Host: + SourcePath: /var/fluent-bit/state + - Name: instance-logs + Host: + SourcePath: /var/log + - Name: systemd-journal + Host: + SourcePath: /run/log/journal + - Name: machine-id + Host: + SourcePath: /etc/machine-id + Cpu: '512' + Memory: '256' + FluentBitECSDaemonService: + Type: AWS::ECS::Service + Properties: + TaskDefinition: !Ref FluentBitTaskDefinition + Cluster: !Ref ClusterName + LaunchType: EC2 + SchedulingStrategy: DAEMON + ServiceName: fluent-bit-daemon-service + FluentBitTaskRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS tasks to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: fluent-bit-task-role-policy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: FluentBitLogs + Effect: Allow + Action: + - 'logs:CreateLogStream' + - 'logs:CreateLogGroup' + - 'logs:PutLogEvents' + Resource: '*' + RoleName: FluentBitECSTaskRole + FluentBitExecutionRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS container agent makes calls to the Amazon ECS API on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + RoleName: FluentBitECSExecutionRole \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit-ecs-instance-logs.json b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit-ecs-instance-logs.json new file mode 100644 index 0000000..54fc015 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit-ecs-instance-logs.json @@ -0,0 +1,146 @@ +{ + "family": "fluent-bit-daemon-service", + "taskRoleArn": "{{task-role-arn}}", + "executionRoleArn": "{{execution-role-arn}}", + "containerDefinitions": [ + { + "name": "fluent-bit-config-init", + "image": "public.ecr.aws/aws-cli/aws-cli:2.18.14", + "cpu": 0, + "essential": false, + "entryPoint": [ + "/bin/sh", + "-c" + ], + "command": [ + "set -ex && if [ -z $( ls -A '/run/log/journal' ) ]; then echo @SET SYSTEMD_PATH=/var/log/journal >> /fluent-bit/etc/fluent-bit.conf; else echo @SET SYSTEMD_PATH=/run/log/journal >> /fluent-bit/etc/fluent-bit.conf; fi && echo @SET INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id) >> /fluent-bit/etc/fluent-bit.conf && if [ -z \"${FLUENT_BIT_CONFIG_FILE}\" ]; then echo 'W1NFUlZJQ0VdCiAgICBGbHVzaCAgICAgICAgICAgICAgICAgICAgIDUKICAgIEdyYWNlICAgICAgICAgICAgICAgICAgICAgMzAKICAgIExvZ19MZXZlbCAgICAgICAgICAgICAgICAgZXJyb3IKICAgIERhZW1vbiAgICAgICAgICAgICAgICAgICAgb2ZmCiAgICBQYXJzZXJzX0ZpbGUgICAgICAgICAgICAgIHBhcnNlcnMuY29uZgogICAgc3RvcmFnZS5wYXRoICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiLXN0b3JhZ2UvCiAgICBzdG9yYWdlLnN5bmMgICAgICAgICAgICAgIG5vcm1hbAogICAgc3RvcmFnZS5jaGVja3N1bSAgICAgICAgICBvZmYKICAgIHN0b3JhZ2UuYmFja2xvZy5tZW1fbGltaXQgNU0KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzdGVtZAogICAgVGFnICAgICAgICAgICAgICAgICBzeXN0ZW1kLSoKICAgIFN5c3RlbWRfRmlsdGVyICAgICAgX1NZU1RFTURfVU5JVD1kb2NrZXIuc2VydmljZQogICAgU3lzdGVtZF9GaWx0ZXIgICAgICBfU1lTVEVNRF9VTklUPWNvbnRhaW5lcmQuc2VydmljZQogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvc3lzdGVtZC5kYgogICAgUGF0aCAgICAgICAgICAgICAgICAke1NZU1RFTURfUEFUSH0KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzdGVtZAogICAgVGFnICAgICAgICAgICAgICAgICBzeXN0ZW1kLXN5c2xvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvc3lzdGVtZC1zeXNsb2cuZGIKICAgIFBhdGggICAgICAgICAgICAgICAgJHtTWVNURU1EX1BBVEh9CiAgICAKW0lOUFVUXQogICAgTmFtZSAgICAgICAgICAgICAgICBzeXN0ZW1kCiAgICBUYWcgICAgICAgICAgICAgICAgIHN5c3RlbWQtZG1lc2cuKgogICAgU3lzdGVtZF9GaWx0ZXIgICAgICBfVFJBTlNQT1JUPWtlcm5lbAogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZG1lc2cuZGIKICAgIFBhdGggICAgICAgICAgICAgICAgJHtTWVNURU1EX1BBVEh9CgpbSU5QVVRdCiAgICBOYW1lICAgICAgICAgICAgICAgIHRhaWwKICAgIFRhZyAgICAgICAgICAgICAgICAgc2VjdXJlCiAgICBQYXRoICAgICAgICAgICAgICAgIC92YXIvbG9nL3NlY3VyZQogICAgUGFyc2VyICAgICAgICAgICAgICBzeXNsb2cKICAgIERCICAgICAgICAgICAgICAgICAgL3Zhci9mbHVlbnQtYml0L3N0YXRlL2ZsYl9zZWN1cmUuZGIKICAgIE1lbV9CdWZfTGltaXQgICAgICAgNU1CCiAgICBTa2lwX0xvbmdfTGluZXMgICAgIE9uCiAgICBSZWZyZXNoX0ludGVydmFsICAgIDEwCiAgICBSZWFkX2Zyb21fSGVhZCAgICAgIE9uCgpbSU5QVVRdCiAgICBOYW1lICAgICAgICAgICAgICAgIHRhaWwKICAgIFRhZyAgICAgICAgICAgICAgICAgZWNzLWFnZW50LmxvZwogICAgUGF0aCAgICAgICAgICAgICAgICAvdmFyL2xvZy9lY3MvZWNzLWFnZW50LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19hZ2VudC5kYgogICAgTWVtX0J1Zl9MaW1pdCAgICAgICA1TUIKICAgIFNraXBfTG9uZ19MaW5lcyAgICAgT24KICAgIFJlZnJlc2hfSW50ZXJ2YWwgICAgMTAKICAgIFJlYWRfZnJvbV9IZWFkICAgICAgT24KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgdGFpbAogICAgVGFnICAgICAgICAgICAgICAgICBlY3MtaW5pdC5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2Vjcy1pbml0LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19pbml0LmRiCiAgICBNZW1fQnVmX0xpbWl0ICAgICAgIDVNQgogICAgU2tpcF9Mb25nX0xpbmVzICAgICBPbgogICAgUmVmcmVzaF9JbnRlcnZhbCAgICAxMAogICAgUmVhZF9mcm9tX0hlYWQgICAgICBPbgoKW0lOUFVUXQogICAgTmFtZSAgICAgICAgICAgICAgICB0YWlsCiAgICBUYWcgICAgICAgICAgICAgICAgIGVjcy1hdWRpdC5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2F1ZGl0LmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc19hdWRpdC5kYgogICAgTWVtX0J1Zl9MaW1pdCAgICAgICA1TUIKICAgIFNraXBfTG9uZ19MaW5lcyAgICAgT24KICAgIFJlZnJlc2hfSW50ZXJ2YWwgICAgMTAKICAgIFJlYWRfZnJvbV9IZWFkICAgICAgT24KCltJTlBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgdGFpbAogICAgVGFnICAgICAgICAgICAgICAgICBlY3Mtdm9sdW1lLXBsdWdpbi5sb2cKICAgIFBhdGggICAgICAgICAgICAgICAgL3Zhci9sb2cvZWNzL2Vjcy12b2x1bWUtcGx1Z2luLmxvZwogICAgREIgICAgICAgICAgICAgICAgICAvdmFyL2ZsdWVudC1iaXQvc3RhdGUvZmxiX2Vjc192b2x1bWVfcGx1Z2luLmRiCiAgICBNZW1fQnVmX0xpbWl0ICAgICAgIDVNQgogICAgU2tpcF9Mb25nX0xpbmVzICAgICBPbgogICAgUmVmcmVzaF9JbnRlcnZhbCAgICAxMAogICAgUmVhZF9mcm9tX0hlYWQgICAgICBPbiAKCltGSUxURVJdCiAgICBOYW1lICAgICAgICAgICAgICAgIG1vZGlmeQogICAgTWF0Y2ggICAgICAgICAgICAgICBzeXN0ZW1kLSoKICAgIFJlbmFtZSAgICAgICAgICAgICAgX0hPU1ROQU1FICAgICAgICAgICAgICAgICAgIGhvc3RuYW1lCiAgICBSZW5hbWUgICAgICAgICAgICAgIF9TWVNURU1EX1VOSVQgICAgICAgICAgICAgICBzeXN0ZW1kX3VuaXQKICAgIFJlbmFtZSAgICAgICAgICAgICAgTUVTU0FHRSAgICAgICAgICAgICAgICAgICAgIG1lc3NhZ2UKICAgIFJlbW92ZV9yZWdleCAgICAgICAgXigoPyFob3N0bmFtZXxzeXN0ZW1kX3VuaXR8bWVzc2FnZSkuKSokCgpbRklMVEVSXQogICAgTmFtZSAgICAgICAgICAgICAgICBhd3MKICAgIE1hdGNoICAgICAgICAgICAgICAgKgogICAgaW1kc192ZXJzaW9uICAgICAgICB2MgoKW09VVFBVVF0KICAgIE5hbWUgICAgICAgICAgICAgICAgY2xvdWR3YXRjaF9sb2dzCiAgICBNYXRjaCAgICAgICAgICAgICAgICoKICAgIHJlZ2lvbiAgICAgICAgICAgICAgJHtBV1NfUkVHSU9OfQogICAgbG9nX2dyb3VwX25hbWUgICAgICAvYXdzL2Vjcy9jb250YWluZXJpbnNpZ2h0cy8ke0NMVVNURVJfTkFNRX0vaW5zdGFuY2UtbG9ncwogICAgbG9nX3N0cmVhbV9wcmVmaXggICAke0lOU1RBTkNFX0lEfS0KICAgIGF1dG9fY3JlYXRlX2dyb3VwICAgdHJ1ZQ==' | base64 -d >> /fluent-bit/etc/fluent-bit.conf; else aws s3 cp $FLUENT_BIT_CONFIG_FILE /tmp/fluent-bit-temp.conf && cat /tmp/fluent-bit-temp.conf >> /fluent-bit/etc/fluent-bit.conf; fi && if [ -z \"${FLUENT_BIT_PARSERS_FILE}\" ]; then echo 'W1BBUlNFUl0KICAgIE5hbWUgICAgICAgICAgICAgICAgc3lzbG9nCiAgICBGb3JtYXQgICAgICAgICAgICAgIHJlZ2V4CiAgICBSZWdleCAgICAgICAgICAgICAgIF4oPzx0aW1lPlteIF0qIHsxLDJ9W14gXSogW14gXSopICg/PGhvc3Q+W14gXSopICg/PGlkZW50PlthLXpBLVowLTlfXC9cLlwtXSopKD86XFsoPzxwaWQ+WzAtOV0rKVxdKT8oPzpbXlw6XSpcOik/ICooPzxtZXNzYWdlPi4qKSQKICAgIFRpbWVfS2V5ICAgICAgICAgICAgdGltZQogICAgVGltZV9Gb3JtYXQgICAgICAgICAlYiAlZCAlSDolTTolUw==' | base64 -d >> /fluent-bit/etc/parsers.conf; else aws s3 cp $FLUENT_BIT_PARSERS_FILE /tmp/parsers-temp.conf && cat /tmp/parsers-temp.conf >> /fluent-bit/etc/parsers.conf; fi\n" + ], + "mountPoints": [ + { + "sourceVolume": "fluent-bit-config", + "containerPath": "/fluent-bit/etc/" + }, + { + "sourceVolume": "instance-id", + "containerPath": "/var/lib/cloud/data/instance-id", + "readOnly": true + }, + { + "sourceVolume": "systemd-journal", + "containerPath": "/run/log/journal", + "readOnly": true + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/ecs-fluent-bit-daemon-service", + "awslogs-create-group": "True", + "awslogs-region": "{{aws-region}}", + "awslogs-stream-prefix": "ecs" + } + } + }, + { + "name": "fluent-bit", + "image": "fluent/fluent-bit:3.1.4", + "cpu": 0, + "essential": true, + "environment": [ + { + "name": "AWS_REGION", + "value": "{{aws-region}}" + }, + { + "name": "CLUSTER_NAME", + "value": "{{ecs-cluster-name}}" + } + ], + "mountPoints": [ + { + "sourceVolume": "fluent-bit-config", + "containerPath": "/fluent-bit/etc/" + }, + { + "sourceVolume": "fluent-bit-state", + "containerPath": "/var/fluent-bit/state" + }, + { + "sourceVolume": "instance-logs", + "containerPath": "/var/log", + "readOnly": true + }, + { + "sourceVolume": "systemd-journal", + "containerPath": "/run/log/journal", + "readOnly": true + }, + { + "sourceVolume": "machine-id", + "containerPath": "/etc/machine-id", + "readOnly": true + } + ], + "dependsOn": [ + { + "containerName": "fluent-bit-config-init", + "condition": "COMPLETE" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/ecs-fluent-bit-daemon-service", + "awslogs-create-group": "True", + "awslogs-region": "{{aws-region}}", + "awslogs-stream-prefix": "ecs" + } + } + } + ], + "networkMode": "bridge", + "volumes": [ + { + "name": "fluent-bit-state", + "host": { + "sourcePath": "/var/fluent-bit/state" + } + }, + { + "name": "instance-logs", + "host": { + "sourcePath": "/var/log" + } + }, + { + "name": "instance-id", + "host": { + "sourcePath": "/var/lib/cloud/data/instance-id" + } + }, + { + "name": "systemd-journal", + "host": { + "sourcePath": "/run/log/journal" + } + }, + { + "name": "machine-id", + "host": { + "sourcePath": "/etc/machine-id" + } + }, + { + "name": "fluent-bit-config", + "dockerVolumeConfiguration": { + "scope": "task", + "driver": "local" + } + } + ], + "requiresCompatibilities": [ + "EC2" + ], + "cpu": "512", + "memory": "256" +} \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit.conf b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit.conf new file mode 100644 index 0000000..b8895c9 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit.conf @@ -0,0 +1,103 @@ +[SERVICE] + Flush 5 + Grace 30 + Log_Level error + Daemon off + Parsers_File parsers.conf + storage.path /var/fluent-bit/state/flb-storage/ + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 5M + +[INPUT] + Name systemd + Tag systemd-* + Systemd_Filter _SYSTEMD_UNIT=docker.service + Systemd_Filter _SYSTEMD_UNIT=containerd.service + DB /var/fluent-bit/state/systemd.db + Path ${SYSTEMD_PATH} + +[INPUT] + Name systemd + Tag systemd-syslog + DB /var/fluent-bit/state/systemd-syslog.db + Path ${SYSTEMD_PATH} + +[INPUT] + Name systemd + Tag systemd-dmesg.* + Systemd_Filter _TRANSPORT=kernel + DB /var/fluent-bit/state/dmesg.db + Path ${SYSTEMD_PATH} + +[INPUT] + Name tail + Tag secure + Path /var/log/secure + Parser syslog + DB /var/fluent-bit/state/flb_secure.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + +[INPUT] + Name tail + Tag ecs-agent.log + Path /var/log/ecs/ecs-agent.log + DB /var/fluent-bit/state/flb_ecs_agent.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + +[INPUT] + Name tail + Tag ecs-init.log + Path /var/log/ecs/ecs-init.log + DB /var/fluent-bit/state/flb_ecs_init.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + +[INPUT] + Name tail + Tag ecs-audit.log + Path /var/log/ecs/audit.log + DB /var/fluent-bit/state/flb_ecs_audit.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + +[INPUT] + Name tail + Tag ecs-volume-plugin.log + Path /var/log/ecs/ecs-volume-plugin.log + DB /var/fluent-bit/state/flb_ecs_volume_plugin.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + +[FILTER] + Name modify + Match systemd-* + Rename _HOSTNAME hostname + Rename _SYSTEMD_UNIT systemd_unit + Rename MESSAGE message + Remove_regex ^((?!hostname|systemd_unit|message).)*$ + +[FILTER] + Name aws + Match * + imds_version v2 + +[OUTPUT] + Name cloudwatch_logs + Match * + region ${AWS_REGION} + log_group_name /aws/ecs/containerinsights/${CLUSTER_NAME}/instance-logs + log_stream_prefix ${INSTANCE_ID}- + auto_create_group true diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/parsers.conf b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/parsers.conf new file mode 100644 index 0000000..99d3b06 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/parsers.conf @@ -0,0 +1,6 @@ +[PARSER] + Name syslog + Format regex + Regex ^(?