From 24e74b3aa29b1f85fe8d3a1602a8b7f1bda696f8 Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:23:47 -0400 Subject: [PATCH 01/10] Add Prometheus scraper configuration for Karpenter Based on sample found in the Karpenter Getting Started Guide (https://karpenter.sh/v0.27.3/getting-started/getting-started-with-karpenter/prometheus-values.yaml) --- .../service/cwagent-prometheus/prometheus-eks.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml index 81e5fb2..17fa2f7 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml @@ -381,6 +381,17 @@ data: regex: 'jvm_gc_collection_seconds.*' action: drop + - job_name: karpenter + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - karpenter + relabel_configs: + - source_labels: [__meta_kubernetes_endpoint_port_name] + regex: http-metrics + action: keep + kind: ConfigMap metadata: name: prometheus-config From 21004dde2a5db9d157851b19edc0fbc94899314a Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:30:07 -0400 Subject: [PATCH 02/10] Add CWAgent prometheus metrics collection for Karpenter Based on the metrics in v0.27.3 as documented here: https://karpenter.sh/v0.27.3/concepts/metrics/ --- .../cwagent-prometheus/prometheus-eks.yaml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml index 17fa2f7..bbeaaae 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml @@ -168,6 +168,56 @@ data: "metric_selectors": [ "^jvm_memory_pool_bytes_used$" ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName"] + ], + "metric_selectors": [ + "^karpenter_consistency_errors$", + "^karpenter_deprovisioning_evaluation_duration_seconds$", + "^karpenter_deprovisioning_replacement_node_initialized_seconds$", + "^karpenter_interruption_deleted_messages$", + "^karpenter_interruption_message_latency_time_seconds$", + "^karpenter_interruption_received_messages$", + "^karpenter_nodes_allocatable$", + "^karpenter_nodes_system_overhead$", + "^karpenter_nodes_termination_time_seconds$", + "^karpenter_nodes_total_daemon_limits$", + "^karpenter_nodes_total_daemon_requests$", + "^karpenter_nodes_total_pod_limits$", + "^karpenter_nodes_total_pod_requests$", + "^karpenter_pods_startup_time_seconds$", + "^karpenter_cloudprovider_batcher_batch_size$", + "^karpenter_cloudprovider_batcher_batch_time_seconds$", + "^karpenter_cloudprovider_duration_seconds$", + "^karpenter_pods_state$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","action"] + ], + "metric_selectors": [ + "^karpenter_deprovisioning_actions_performed$", + "^karpenter_interruption_actions_performed$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","provisioner"] + ], + "metric_selectors": [ + "^karpenter_provisioner_scheduling_duration_seconds$", + "^karpenter_nodes_created$", + "^karpenter_nodes_terminated$" + ] } ] } From 66416a4817ed4089635f4ed814ca59746c7f456f Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:31:10 -0400 Subject: [PATCH 03/10] Update prometheus-eks.yaml Add missed provisioner*resource_type section --- .../service/cwagent-prometheus/prometheus-eks.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml index bbeaaae..6f31ad0 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml @@ -207,7 +207,7 @@ data: "^karpenter_interruption_actions_performed$" ] }, - { + { "source_labels": ["job"], "label_matcher": "^karpenter$", "dimensions": [ @@ -219,6 +219,18 @@ data: "^karpenter_nodes_terminated$" ] } + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","provisioner","resource_type"] + ], + "metric_selectors": [ + "^karpenter_provisioner_limit$", + "^karpenter_provisioner_usage$", + "^karpenter_provisioner_usage_pct$" + ] + }, ] } } From d91585907a5e9f040785de14612502d4f938c0f7 Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:31:59 -0400 Subject: [PATCH 04/10] Fixed missing comma and whitespace --- .../service/cwagent-prometheus/prometheus-eks.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml index 6f31ad0..2cbebde 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml @@ -207,7 +207,7 @@ data: "^karpenter_interruption_actions_performed$" ] }, - { + { "source_labels": ["job"], "label_matcher": "^karpenter$", "dimensions": [ @@ -218,8 +218,8 @@ data: "^karpenter_nodes_created$", "^karpenter_nodes_terminated$" ] - } - { + }, + { "source_labels": ["job"], "label_matcher": "^karpenter$", "dimensions": [ From 3f9b2e2393d3375a16fca6c9c01690a8ec215d26 Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:33:21 -0400 Subject: [PATCH 05/10] Remove excess comma --- .../service/cwagent-prometheus/prometheus-eks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml index 2cbebde..05595d5 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks.yaml @@ -230,7 +230,7 @@ data: "^karpenter_provisioner_usage$", "^karpenter_provisioner_usage_pct$" ] - }, + } ] } } From bc0afc04e64bca7d987f1c7f64e7dd1ee2bb1af8 Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:34:58 -0400 Subject: [PATCH 06/10] Add prometheus scraper for Karpenter to k8s template also --- .../service/cwagent-prometheus/prometheus-k8s.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml index b238881..774b413 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml @@ -328,6 +328,17 @@ data: regex: 'jvm_gc_collection_seconds.*' action: drop + - job_name: karpenter + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - karpenter + relabel_configs: + - source_labels: [__meta_kubernetes_endpoint_port_name] + regex: http-metrics + action: keep + kind: ConfigMap metadata: name: prometheus-config From c1a33ee600267dcd74f3e69ae267d4fc89abb01b Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:36:25 -0400 Subject: [PATCH 07/10] Add CWAgent Prometheus metric collection for k8s deployment template --- .../cwagent-prometheus/prometheus-k8s.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml index 774b413..2d9aa3a 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-k8s.yaml @@ -160,6 +160,68 @@ data: "metric_selectors": [ "^jvm_memory_pool_bytes_used$" ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName"] + ], + "metric_selectors": [ + "^karpenter_consistency_errors$", + "^karpenter_deprovisioning_evaluation_duration_seconds$", + "^karpenter_deprovisioning_replacement_node_initialized_seconds$", + "^karpenter_interruption_deleted_messages$", + "^karpenter_interruption_message_latency_time_seconds$", + "^karpenter_interruption_received_messages$", + "^karpenter_nodes_allocatable$", + "^karpenter_nodes_system_overhead$", + "^karpenter_nodes_termination_time_seconds$", + "^karpenter_nodes_total_daemon_limits$", + "^karpenter_nodes_total_daemon_requests$", + "^karpenter_nodes_total_pod_limits$", + "^karpenter_nodes_total_pod_requests$", + "^karpenter_pods_startup_time_seconds$", + "^karpenter_cloudprovider_batcher_batch_size$", + "^karpenter_cloudprovider_batcher_batch_time_seconds$", + "^karpenter_cloudprovider_duration_seconds$", + "^karpenter_pods_state$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","action"] + ], + "metric_selectors": [ + "^karpenter_deprovisioning_actions_performed$", + "^karpenter_interruption_actions_performed$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","provisioner"] + ], + "metric_selectors": [ + "^karpenter_provisioner_scheduling_duration_seconds$", + "^karpenter_nodes_created$", + "^karpenter_nodes_terminated$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","provisioner","resource_type"] + ], + "metric_selectors": [ + "^karpenter_provisioner_limit$", + "^karpenter_provisioner_usage$", + "^karpenter_provisioner_usage_pct$" + ] } ] } From 3e357287797587a31a35ebc8e0eaeafd068c9e16 Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:37:15 -0400 Subject: [PATCH 08/10] Add Prometheus scraper for Fargate sample --- .../cwagent-prometheus/prometheus-eks-fargate.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml index 90b42ff..38c482e 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml @@ -338,6 +338,17 @@ data: regex: 'jvm_gc_collection_seconds.*' action: drop + - job_name: karpenter + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - karpenter + relabel_configs: + - source_labels: [__meta_kubernetes_endpoint_port_name] + regex: http-metrics + action: keep + kind: ConfigMap metadata: name: prometheus-config From df1576d94ba892db8f99b5ae1b9029f37f6860e6 Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:37:53 -0400 Subject: [PATCH 09/10] Add CWAgent Prometheus metrics for Fargate sample --- .../prometheus-eks-fargate.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml index 38c482e..11915cb 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/prometheus-eks-fargate.yaml @@ -170,6 +170,68 @@ data: "metric_selectors": [ "^jvm_memory_pool_bytes_used$" ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName"] + ], + "metric_selectors": [ + "^karpenter_consistency_errors$", + "^karpenter_deprovisioning_evaluation_duration_seconds$", + "^karpenter_deprovisioning_replacement_node_initialized_seconds$", + "^karpenter_interruption_deleted_messages$", + "^karpenter_interruption_message_latency_time_seconds$", + "^karpenter_interruption_received_messages$", + "^karpenter_nodes_allocatable$", + "^karpenter_nodes_system_overhead$", + "^karpenter_nodes_termination_time_seconds$", + "^karpenter_nodes_total_daemon_limits$", + "^karpenter_nodes_total_daemon_requests$", + "^karpenter_nodes_total_pod_limits$", + "^karpenter_nodes_total_pod_requests$", + "^karpenter_pods_startup_time_seconds$", + "^karpenter_cloudprovider_batcher_batch_size$", + "^karpenter_cloudprovider_batcher_batch_time_seconds$", + "^karpenter_cloudprovider_duration_seconds$", + "^karpenter_pods_state$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","action"] + ], + "metric_selectors": [ + "^karpenter_deprovisioning_actions_performed$", + "^karpenter_interruption_actions_performed$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","provisioner"] + ], + "metric_selectors": [ + "^karpenter_provisioner_scheduling_duration_seconds$", + "^karpenter_nodes_created$", + "^karpenter_nodes_terminated$" + ] + }, + { + "source_labels": ["job"], + "label_matcher": "^karpenter$", + "dimensions": [ + ["ClusterName","provisioner","resource_type"] + ], + "metric_selectors": [ + "^karpenter_provisioner_limit$", + "^karpenter_provisioner_usage$", + "^karpenter_provisioner_usage_pct$" + ] } ] } From 8a33ea2f8a13a221bb33d5ad87ac584728fa8e89 Mon Sep 17 00:00:00 2001 From: Mike Reinhold Date: Thu, 11 May 2023 17:39:23 -0400 Subject: [PATCH 10/10] Update README.md with Karpenter configuration --- .../deployment-mode/service/cwagent-prometheus/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/README.md b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/README.md index 5ac0c90..555dd69 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/README.md +++ b/k8s-deployment-manifest-templates/deployment-mode/service/cwagent-prometheus/README.md @@ -33,3 +33,4 @@ Both yaml files contain the default settings for the following containerized app |HAPROXY_INGRESS |Exposed by Helm Chart: [incubator/haproxy-ingress](https://github.com/helm/charts/tree/master/incubator/haproxy-ingress) | |AWS APP MESH |Exposed by Helm Chart: [EKS Charts](https://github.com/aws/eks-charts/blob/master/README.md) | |JAVA/JMX |Exposed by JMX_Exporter: [JMX_Exporter](https://github.com/prometheus/jmx_exporter) | +|KARPENTER |Exposed by Karpenter controller: [Karpenter metrics](https://karpenter.sh/v0.27.3/concepts/metrics/) |