From ab34011c9b19718d36bafbb012db426691928fb7 Mon Sep 17 00:00:00 2001 From: Illia Popov Date: Wed, 14 Apr 2021 20:11:40 -0400 Subject: [PATCH] Performance optimizations for kubernetes_metadata filter kubernetes_metadata filter enriches messages with kubernetes context like pod_id, namespace_name, labels etc. It reads all pods from api server and subscribed for the updates. With relatively large clusters it generates enormous load on the api server as it list ALL pods for the whole cluster instead of just for the machine where agent is running. Issue is fixed by filter developers: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/170 However this fix is not included into the image used by AWS. With this pull request I upgrade image version to latest with the filter plugin fix and pass K8S_NODE_NAME env variable which enables the optimization. --- container-insights-manifest-update.sh | 2 +- .../deployment-mode/daemonset/combination/combination.yaml | 7 ++++++- .../container-insights-monitoring/fluentd/fluentd.yaml | 7 ++++++- .../quickstart/cwagent-fluentd-quickstart.yaml | 7 ++++++- .../cwagent-fluentd-xray-quickstart.yaml | 7 ++++++- k8s-yaml-templates/fluentd/fluentd.yaml | 7 ++++++- .../quickstart/cwagent-fluentd-quickstart.yaml | 7 ++++++- 7 files changed, 37 insertions(+), 7 deletions(-) diff --git a/container-insights-manifest-update.sh b/container-insights-manifest-update.sh index ec6b758..121d839 100755 --- a/container-insights-manifest-update.sh +++ b/container-insights-manifest-update.sh @@ -6,7 +6,7 @@ ecsDirPrefix="./ecs-task-definition-templates/deployment-mode/daemon-service/cwa newK8sVersion="k8s/1.3.5" agentVersion="amazon/cloudwatch-agent:1.247347.5b250583" -fluentdVersion="fluent/fluentd-kubernetes-daemonset:v1.7.3-debian-cloudwatch-1.0" +fluentdVersion="fluent/fluentd-kubernetes-daemonset:v1.12.2-debian-cloudwatch-1.3" fluentBitVersion="amazon/aws-for-fluent-bit:2.10.0" k8sPrometheusDirPrefix="./k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus" diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml index 0d598cd..7c1eef4 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/combination/combination.yaml @@ -580,7 +580,7 @@ spec: command: ['sh','-c',''] containers: - name: fluentd-cloudwatch - image: fluent/fluentd-kubernetes-daemonset:v1.7.3-debian-cloudwatch-1.0 + image: fluent/fluentd-kubernetes-daemonset:v1.12.2-debian-cloudwatch-1.3 env: - name: REGION valueFrom: @@ -592,6 +592,11 @@ spec: configMapKeyRef: name: cluster-info key: cluster.name + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName resources: limits: memory: 400Mi diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml index ed36763..0bd14ee 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml @@ -378,7 +378,7 @@ spec: command: ['sh','-c',''] containers: - name: fluentd-cloudwatch - image: fluent/fluentd-kubernetes-daemonset:v1.7.3-debian-cloudwatch-1.0 + image: fluent/fluentd-kubernetes-daemonset:v1.12.2-debian-cloudwatch-1.3 env: - name: AWS_REGION valueFrom: @@ -392,6 +392,11 @@ spec: key: cluster.name - name: CI_VERSION value: "k8s/1.3.5" + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName resources: limits: memory: 400Mi diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml index 15da8c2..80ae650 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/quickstart/cwagent-fluentd-quickstart.yaml @@ -561,7 +561,7 @@ spec: command: ['sh','-c',''] containers: - name: fluentd-cloudwatch - image: fluent/fluentd-kubernetes-daemonset:v1.7.3-debian-cloudwatch-1.0 + image: fluent/fluentd-kubernetes-daemonset:v1.12.2-debian-cloudwatch-1.3 env: - name: AWS_REGION valueFrom: @@ -575,6 +575,11 @@ spec: key: cluster.name - name: CI_VERSION value: "k8s/1.3.5" + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName resources: limits: memory: 400Mi diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/cwagent-fluentd-xray/cwagent-fluentd-xray-quickstart.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/cwagent-fluentd-xray/cwagent-fluentd-xray-quickstart.yaml index 9349010..d9b0a51 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/cwagent-fluentd-xray/cwagent-fluentd-xray-quickstart.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/cwagent-fluentd-xray/cwagent-fluentd-xray-quickstart.yaml @@ -471,7 +471,7 @@ spec: command: ['sh','-c',''] containers: - name: fluentd-cloudwatch - image: fluent/fluentd-kubernetes-daemonset:v1.7.3-debian-cloudwatch-1.0 + image: fluent/fluentd-kubernetes-daemonset:v1.12.2-debian-cloudwatch-1.3 env: - name: REGION valueFrom: @@ -483,6 +483,11 @@ spec: configMapKeyRef: name: cluster-info key: cluster.name + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName resources: limits: memory: 400Mi diff --git a/k8s-yaml-templates/fluentd/fluentd.yaml b/k8s-yaml-templates/fluentd/fluentd.yaml index 44ef6e1..14edc38 100644 --- a/k8s-yaml-templates/fluentd/fluentd.yaml +++ b/k8s-yaml-templates/fluentd/fluentd.yaml @@ -378,7 +378,7 @@ spec: command: ['sh','-c',''] containers: - name: fluentd-cloudwatch - image: fluent/fluentd-kubernetes-daemonset:v1.7.3-debian-cloudwatch-1.0 + image: fluent/fluentd-kubernetes-daemonset:v1.12.2-debian-cloudwatch-1.3 env: - name: REGION valueFrom: @@ -392,6 +392,11 @@ spec: key: cluster.name - name: CI_VERSION value: "k8s/1.0.1" + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName resources: limits: memory: 400Mi diff --git a/k8s-yaml-templates/quickstart/cwagent-fluentd-quickstart.yaml b/k8s-yaml-templates/quickstart/cwagent-fluentd-quickstart.yaml index 3cb62a9..0e31a07 100644 --- a/k8s-yaml-templates/quickstart/cwagent-fluentd-quickstart.yaml +++ b/k8s-yaml-templates/quickstart/cwagent-fluentd-quickstart.yaml @@ -562,7 +562,7 @@ spec: command: ['sh','-c',''] containers: - name: fluentd-cloudwatch - image: fluent/fluentd-kubernetes-daemonset:v1.7.3-debian-cloudwatch-1.0 + image: fluent/fluentd-kubernetes-daemonset:v1.12.2-debian-cloudwatch-1.3 env: - name: REGION valueFrom: @@ -576,6 +576,11 @@ spec: key: cluster.name - name: CI_VERSION value: "k8s/1.0.1" + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName resources: limits: memory: 400Mi