From 8bd9501abc3fee45a550a9f301e369f16a8a99ac Mon Sep 17 00:00:00 2001 From: Jenade Moodley Date: Wed, 21 Aug 2024 15:46:09 +0200 Subject: [PATCH 1/2] Added fluent-bit logging capabilities --- .../deployment-mode/daemon-service/README.md | 8 +- .../fluent-bit-cwagent-combined-cfn/README.md | 19 + .../fluent-bit-cwagent-combined-cfn.yaml | 448 ++++++++++++++++++ .../fluent-bit-ecs-instance-logs/README.md | 56 +++ .../cloudformation-quickstart/README.md | 19 + .../fluent-bit-ecs-instance-logs.cfn.yaml | 326 +++++++++++++ .../fluent-bit-ecs-instance-logs.json | 157 ++++++ .../fluent-bit.conf | 103 ++++ .../fluent-bit-ecs-instance-logs/parsers.conf | 6 + 9 files changed, 1141 insertions(+), 1 deletion(-) create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/README.md create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/fluent-bit-cwagent-combined-cfn.yaml create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/README.md create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/README.md create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/cloudformation-quickstart/fluent-bit-ecs-instance-logs.cfn.yaml create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit-ecs-instance-logs.json create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/fluent-bit.conf create mode 100644 ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-ecs-instance-logs/parsers.conf diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/README.md b/ecs-task-definition-templates/deployment-mode/daemon-service/README.md index 10e08a5e..18c72bd5 100644 --- a/ecs-task-definition-templates/deployment-mode/daemon-service/README.md +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/README.md @@ -5,4 +5,10 @@ This folder contains the example Amazon ECS task definitions for DaemonService d Check the sub folders for different functionality: ### [cwagent-ecs-instance-metric](cwagent-ecs-instance-metric) -This folder provides the functionality that enables you to deploy the CloudWatch Agent to collect ECS Instance Metrics. \ No newline at end of file +This folder provides the functionality that enables you to deploy the CloudWatch Agent to collect ECS Instance Metrics. + +### [fluent-bit-ecs-instance-logs]( fluent-bit-ecs-instance-logs) +This folder provides the functionality that enables you to deploy Fluent-Bit to collect ECS Container Instance. + +### [fluent-bit-cwagent-combined-cfn]( fluent-bit-cwagent-combined-cfn) +This folder provides the functionality that enables you to deploy both the CloudWatch Agent and Fluent-Bit in a single CloudFormation stack. \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/README.md b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/README.md new file mode 100644 index 00000000..e12c99ee --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/README.md @@ -0,0 +1,19 @@ +## CloudWatch Agent and Fluent-Bit for ECS Instance Metrics and Logs Collection Quick Start + +The cloudformation template in this folder helps you to quickly deploy both the CloudWatch agent and Fluent-Bit as daemon-services using a single CloudFormation stack to collect ECS Container Instance Metrics and Logs. + +* [fluent-bit-cwagent-combined-cfn.yaml](fluent-bit-cwagent-combined-cfn.yaml): sample cloudformation template + + +Run following aws cloudformation command with the cloudformation template file to deploy the Fluent-Bit and CloudWatch agent Daemonsets with required IAM roles. ***Please assign the actual ECS cluster name and the cluster region in the first two lines of the command separately.*** + +``` +ClusterName= +Region= +aws cloudformation create-stack --stack-name Fluent-Bit-${ClusterName}-${Region} \ + --template-body file://fluent-bit-cwagent-combined-cfn.yaml \ + --parameters ParameterKey=ClusterName,ParameterValue=${ClusterName} \ + ParameterKey=CreateIAMRoles,ParameterValue=True \ + --capabilities CAPABILITY_NAMED_IAM \ + --region ${Region} +``` \ No newline at end of file diff --git a/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/fluent-bit-cwagent-combined-cfn.yaml b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/fluent-bit-cwagent-combined-cfn.yaml new file mode 100644 index 00000000..a0c63c71 --- /dev/null +++ b/ecs-task-definition-templates/deployment-mode/daemon-service/fluent-bit-cwagent-combined-cfn/fluent-bit-cwagent-combined-cfn.yaml @@ -0,0 +1,448 @@ +AWSTemplateFormatVersion: '2010-09-09' +Parameters: + ClusterName: + Type: String + Description: Enter the name of your ECS cluster from which you want to collect metrics and/or logs + CreateIAMRoles: + Type: String + Default: 'False' + AllowedValues: + - 'True' + - 'False' + Description: Whether to create default IAM roles + ConstraintDescription: must specify True or False. + CWAgentTaskRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent ecs task role + CWAgentExecutionRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent execution role + FluentBitTaskRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent ecs task role + FluentBitExecutionRoleArn: + Type: String + Default: Default + Description: Enter the role arn you want to use for the CloudWatch Agent execution role +Conditions: + CreateRoles: !Equals + - !Ref CreateIAMRoles + - 'True' + DefaultCWAgentTaskRole: !Equals + - !Ref CWAgentTaskRoleArn + - Default + DefaultCWAgentExecutionRole: !Equals + - !Ref CWAgentExecutionRoleArn + - Default + DefaultFluentBitTaskRole: !Equals + - !Ref FluentBitTaskRoleArn + - Default + DefaultFluentBitExecutionRole: !Equals + - !Ref FluentBitExecutionRoleArn + - Default +Resources: + CWAgentECSTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + TaskRoleArn: !If + - CreateRoles + - !GetAtt CWAgentECSTaskRole.Arn + - !If + - DefaultCWAgentTaskRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/CWAgentECSTaskRole + - !Ref CWAgentTaskRoleArn + ExecutionRoleArn: !If + - CreateRoles + - !GetAtt CWAgentECSExecutionRole.Arn + - !If + - DefaultCWAgentExecutionRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/CWAgentECSExecutionRole + - !Ref CWAgentExecutionRoleArn + NetworkMode: bridge + ContainerDefinitions: + - Name: cloudwatch-agent + Image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300042.1b746 + MountPoints: + - ReadOnly: true + ContainerPath: /rootfs/proc + SourceVolume: proc + - ReadOnly: true + ContainerPath: /rootfs/dev + SourceVolume: dev + - ReadOnly: true + ContainerPath: /sys/fs/cgroup + SourceVolume: al2_cgroup + - ReadOnly: true + ContainerPath: /cgroup + SourceVolume: al1_cgroup + - ReadOnly: true + ContainerPath: /rootfs/sys/fs/cgroup + SourceVolume: al2_cgroup + - ReadOnly: true + ContainerPath: /rootfs/cgroup + SourceVolume: al1_cgroup + Environment: + - Name: USE_DEFAULT_CONFIG + Value: 'True' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-cwagent-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + RequiresCompatibilities: + - EC2 + Volumes: + - Name: proc + Host: + SourcePath: /proc + - Name: dev + Host: + SourcePath: /dev + - Name: al1_cgroup + Host: + SourcePath: /cgroup + - Name: al2_cgroup + Host: + SourcePath: /sys/fs/cgroup + Cpu: '128' + Memory: '64' + CWAgentECSDaemonService: + Type: AWS::ECS::Service + Properties: + TaskDefinition: !Ref CWAgentECSTaskDefinition + Cluster: !Ref ClusterName + LaunchType: EC2 + SchedulingStrategy: DAEMON + ServiceName: cwagent-daemon-service + CWAgentECSTaskRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS tasks to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + RoleName: CWAgentECSTaskRole + CWAgentECSExecutionRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS container agent makes calls to the Amazon ECS API on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + RoleName: CWAgentECSExecutionRole + FluentBitTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + TaskRoleArn: !If + - CreateRoles + - !GetAtt FluentBitTaskRole.Arn + - !If + - DefaultFluentBitTaskRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/FluentBitECSTaskRole + - !Ref FluentBitTaskRoleArn + ExecutionRoleArn: !If + - CreateRoles + - !GetAtt FluentBitExecutionRole.Arn + - !If + - DefaultFluentBitExecutionRole + - !Sub arn:aws:iam::${AWS::AccountId}:role/FluentBitECSExecutionRole + - !Ref FluentBitExecutionRoleArn + NetworkMode: bridge + ContainerDefinitions: + - Name: fluent-bit + Image: fluent/fluent-bit:3.1.4 + MountPoints: + - ContainerPath: /fluent-bit/etc/ + SourceVolume: fluent-bit-config + - ContainerPath: /var/fluent-bit/state + SourceVolume: fluent-bit-state + - ReadOnly: true + ContainerPath: /var/log + SourceVolume: instance-logs + - ReadOnly: true + ContainerPath: /run/log/journal + SourceVolume: systemd-journal + - ReadOnly: true + ContainerPath: /etc/machine-id + SourceVolume: machine-id + Environment: + - Name: AWS_REGION + Value: !Ref AWS::Region + - Name: CLUSTER_NAME + Value: !Ref ClusterName + DependsOn: + - ContainerName: fluent-bit-config-init + Condition: COMPLETE + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-fluent-bit-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + - Name: fluent-bit-config-init + Image: amazon/aws-cli:2.17.24 + Essential: False + EntryPoint: + - '/bin/sh' + - '-c' + Command: + - | + set -ex && if [ -z $( ls -A '/run/log/journal' ) ]; then echo @SET SYSTEMD_PATH=/var/log/journal >> /fluent-bit/etc/fluent-bit.conf; else echo @SET SYSTEMD_PATH=/run/log/journal >> /fluent-bit/etc/fluent-bit.conf; fi && echo @SET INSTANCE_ID=$(cat /var/lib/cloud/data/instance-id) >> /fluent-bit/etc/fluent-bit.conf && aws ssm get-parameter --name $FB_CONFIG_SSM_PARAM --query Parameter.Value --output text >> /fluent-bit/etc/fluent-bit.conf && aws ssm get-parameter --name $FB_PARSER_SSM_PARAM --query Parameter.Value --output text >> /fluent-bit/etc/parsers.conf && cat /fluent-bit/etc/fluent-bit.conf + Environment: + - Name: FB_CONFIG_SSM_PARAM + Value: !Ref FluentBitConfigParam + - Name: FB_PARSER_SSM_PARAM + Value: !Ref FluentBitParserParam + MountPoints: + - ContainerPath: /fluent-bit/etc/ + SourceVolume: fluent-bit-config + - ReadOnly: true + ContainerPath: /var/lib/cloud/data/instance-id + SourceVolume: instance-id + - ReadOnly: true + ContainerPath: /run/log/journal + SourceVolume: systemd-journal + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/ecs-fluent-bit-daemon-service + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + RequiresCompatibilities: + - EC2 + Volumes: + - Name: fluent-bit-config + DockerVolumeConfiguration: + Scope: 'task' + - Name: instance-id + Host: + SourcePath: /var/lib/cloud/data/instance-id + - Name: fluent-bit-state + Host: + SourcePath: /var/fluent-bit/state + - Name: instance-logs + Host: + SourcePath: /var/log + - Name: systemd-journal + Host: + SourcePath: /run/log/journal + - Name: machine-id + Host: + SourcePath: /etc/machine-id + Cpu: '512' + Memory: '256' + FluentBitECSDaemonService: + Type: AWS::ECS::Service + Properties: + TaskDefinition: !Ref FluentBitTaskDefinition + Cluster: !Ref ClusterName + LaunchType: EC2 + SchedulingStrategy: DAEMON + ServiceName: fluent-bit-daemon-service + FluentBitTaskRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS tasks to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: fluent-bit-task-role-policy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: FluentBitLogs + Effect: Allow + Action: + - 'logs:CreateLogStream' + - 'logs:CreateLogGroup' + - 'logs:PutLogEvents' + Resource: '*' + - Sid: FluentBitSSMParameterAccess + Effect: Allow + Action: + - 'ssm:GetParameters' + - 'ssm:GetParameter' + Resource: + - !Sub + - 'arn:${Partition}:ssm:${Region}:${AccountID}:parameter/*' + - Partition: !Ref AWS::Partition + Region: !Ref AWS::Region + AccountID: !Ref AWS::AccountId + RoleName: FluentBitECSTaskRole + FluentBitExecutionRole: + Type: AWS::IAM::Role + Condition: CreateRoles + Properties: + Description: Allows ECS container agent makes calls to the Amazon ECS API on your behalf. + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + RoleName: FluentBitECSExecutionRole + FluentBitConfigParam: + Type: AWS::SSM::Parameter + Properties: + Name: !Join + - '-' + - - 'fluent-bit-config' + - !Ref ClusterName + Type: String + Value: | + [SERVICE] + Flush 5 + Grace 30 + Log_Level error + Daemon off + Parsers_File parsers.conf + storage.path /var/fluent-bit/state/flb-storage/ + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 5M + + [INPUT] + Name systemd + Tag systemd-* + Systemd_Filter _SYSTEMD_UNIT=docker.service + Systemd_Filter _SYSTEMD_UNIT=containerd.service + DB /var/fluent-bit/state/systemd.db + Path ${SYSTEMD_PATH} + + [INPUT] + Name systemd + Tag systemd-syslog + DB /var/fluent-bit/state/systemd-syslog.db + Path ${SYSTEMD_PATH} + + [INPUT] + Name systemd + Tag systemd-dmesg.* + Systemd_Filter _TRANSPORT=kernel + DB /var/fluent-bit/state/dmesg.db + Path ${SYSTEMD_PATH} + + [INPUT] + Name tail + Tag secure + Path /var/log/secure + Parser syslog + DB /var/fluent-bit/state/flb_secure.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + + [INPUT] + Name tail + Tag ecs-agent.log + Path /var/log/ecs/ecs-agent.log + DB /var/fluent-bit/state/flb_ecs_agent.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + + [INPUT] + Name tail + Tag ecs-init.log + Path /var/log/ecs/ecs-init.log + DB /var/fluent-bit/state/flb_ecs_init.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + + [INPUT] + Name tail + Tag ecs-audit.log + Path /var/log/ecs/audit.log + DB /var/fluent-bit/state/flb_ecs_audit.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + + [INPUT] + Name tail + Tag ecs-volume-plugin.log + Path /var/log/ecs/ecs-volume-plugin.log + DB /var/fluent-bit/state/flb_ecs_volume_plugin.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head On + + [FILTER] + Name modify + Match systemd-* + Rename _HOSTNAME hostname + Rename _SYSTEMD_UNIT systemd_unit + Rename MESSAGE message + Remove_regex ^((?!hostname|systemd_unit|message).)*$ + + [FILTER] + Name aws + Match * + imds_version v2 + + [OUTPUT] + Name cloudwatch_logs + Match * + region ${AWS_REGION} + log_group_name /aws/ecs/containerinsights/${CLUSTER_NAME}/instance-logs + log_stream_prefix ${INSTANCE_ID}- + auto_create_group true + Description: SSM Parameter for Fluent Bit configuration in ECS + FluentBitParserParam: + Type: AWS::SSM::Parameter + Properties: + Name: !Join + - '-' + - - 'fluent-bit-parser' + - !Ref ClusterName + Type: String + Value: !Sub | + [PARSER] + Name syslog + Format regex + Regex ^(?