From 5fb2b0a1f5702446513e8fac980a92d030dec35f Mon Sep 17 00:00:00 2001
From: Aniket Rao <anik3tra0@gmail.com>
Date: Wed, 6 Oct 2021 19:32:31 +0530
Subject: [PATCH 1/2] update metrics for kube state: wip

---
 modules/prometheus/istio.hcl              | 44 +++++------
 modules/prometheus/k8s_via_kube_state.hcl | 92 +++++++++--------------
 2 files changed, 56 insertions(+), 80 deletions(-)

diff --git a/modules/prometheus/istio.hcl b/modules/prometheus/istio.hcl
index c8341db..e77bf30 100644
--- a/modules/prometheus/istio.hcl
+++ b/modules/prometheus/istio.hcl
@@ -41,8 +41,8 @@ ingester prometheus_istio_workload module {
     unit = "count"
 
     source prometheus "throughput" {
-      //query = "sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='source', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m]))"
-      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='source', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
+      //query = "sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='destination', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m]))"
+      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='destination', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -54,7 +54,7 @@ ingester prometheus_istio_workload module {
     unit = "count"
 
     source prometheus "status_2xx" {
-      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='source', source_canonical_service!='unknown', response_code=~'^2.*', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='destination', source_canonical_service!='unknown', response_code=~'^2.*', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -66,7 +66,7 @@ ingester prometheus_istio_workload module {
     unit = "count"
 
     source prometheus "status_3xx" {
-      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='source', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster', response_code=~'^3.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='destination', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster', response_code=~'^3.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -78,7 +78,7 @@ ingester prometheus_istio_workload module {
     unit = "count"
 
     source prometheus "status_4xx" {
-      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='source', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster', response_code=~'^4.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='destination', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster', response_code=~'^4.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -90,7 +90,7 @@ ingester prometheus_istio_workload module {
     unit = "count"
 
     source prometheus "status_5xx" {
-      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='source', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster', response_code=~'^5.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_requests_total{reporter='destination', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster', response_code=~'^5.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -140,7 +140,7 @@ ingester prometheus_istio_workload module {
     unit = "bytes"
 
     source prometheus "bytes_in" {
-      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_request_bytes_sum{reporter='source', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace, destination_version, pod_name) (increase(istio_request_bytes_sum{reporter='destination', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -152,7 +152,7 @@ ingester prometheus_istio_workload module {
     unit = "bytes"
 
     source prometheus "bytes_out" {
-      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace,  destination_version, pod_name) (increase(istio_response_bytes_sum{reporter='source', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, destination_canonical_service,  destination_workload, destination_workload_namespace,  destination_version, pod_name) (increase(istio_response_bytes_sum{reporter='destination', source_canonical_service!='unknown', destination_service_name!='PassthroughCluster'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -199,7 +199,7 @@ ingester prometheus_istio_cluster module {
     unit = "count"
 
     source prometheus "throughput" {
-      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='source'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='destination'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -211,7 +211,7 @@ ingester prometheus_istio_cluster module {
     unit = "count"
 
     source prometheus "status_2xx" {
-      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='source', response_code=~'^2.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='destination', response_code=~'^2.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -223,7 +223,7 @@ ingester prometheus_istio_cluster module {
     unit = "count"
 
     source prometheus "status_3xx" {
-      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='source', response_code=~'^3.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='destination', response_code=~'^3.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -235,7 +235,7 @@ ingester prometheus_istio_cluster module {
     unit = "count"
 
     source prometheus "status_4xx" {
-      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='source', response_code=~'^4.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='destination', response_code=~'^4.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -247,7 +247,7 @@ ingester prometheus_istio_cluster module {
     unit = "count"
 
     source prometheus "status_5xx" {
-      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='source', response_code=~'^5.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster) (increase(istio_requests_total{reporter='destination', response_code=~'^5.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -259,7 +259,7 @@ ingester prometheus_istio_cluster module {
     unit = "bytes"
 
     source prometheus "bytes_in" {
-      query = "label_set(sum by (cluster) (increase(istio_request_bytes_sum{reporter='source'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster) (increase(istio_request_bytes_sum{reporter='destination'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -271,7 +271,7 @@ ingester prometheus_istio_cluster module {
     unit = "bytes"
 
     source prometheus "bytes_out" {
-      query = "label_set(sum by (cluster) (increase(istio_response_bytes_sum{reporter='source'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster) (increase(istio_response_bytes_sum{reporter='destination'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -338,7 +338,7 @@ ingester prometheus_istio_k8s_pod module {
     unit = "count"
 
     source prometheus "throughput" {
-      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='source', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='destination', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -350,7 +350,7 @@ ingester prometheus_istio_k8s_pod module {
     unit = "count"
 
     source prometheus "status_2xx" {
-      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='source', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^2.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='destination', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^2.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -362,7 +362,7 @@ ingester prometheus_istio_k8s_pod module {
     unit = "count"
 
     source prometheus "status_3xx" {
-      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='source', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^3.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='destination', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^3.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -374,7 +374,7 @@ ingester prometheus_istio_k8s_pod module {
     unit = "count"
 
     source prometheus "status_4xx" {
-      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='source', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^4.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='destination', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^4.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -386,7 +386,7 @@ ingester prometheus_istio_k8s_pod module {
     unit = "count"
 
     source prometheus "status_5xx" {
-      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='source', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^5.*'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_requests_total{reporter='destination', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown', response_code=~'^5.*'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -398,7 +398,7 @@ ingester prometheus_istio_k8s_pod module {
     unit = "bytes"
 
     source prometheus "bytes_in" {
-      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_request_bytes_sum{reporter='source', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_request_bytes_sum{reporter='destination', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -410,7 +410,7 @@ ingester prometheus_istio_k8s_pod module {
     unit = "bytes"
 
     source prometheus "bytes_out" {
-      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_response_bytes_sum{reporter='source', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown'}[1m])), 'cluster', '$input{cluster}')"
+      query = "label_set(sum by (cluster, pod_name, destination_canonical_service, destination_workload_namespace) (increase(istio_response_bytes_sum{reporter='destination', destination_service_name!='PassthroughCluster', source_canonical_service!='unknown'}[1m])), 'cluster', '$input{cluster}')"
 
       join_on = {
         "$output{cluster}" = "$input{cluster}"
diff --git a/modules/prometheus/k8s_via_kube_state.hcl b/modules/prometheus/k8s_via_kube_state.hcl
index 5b9a81c..d4f5616 100644
--- a/modules/prometheus/k8s_via_kube_state.hcl
+++ b/modules/prometheus/k8s_via_kube_state.hcl
@@ -121,7 +121,7 @@ ingester prometheus_kube_cluster module {
   }
 }
 
-
+# Done
 ingester prometheus_kube_cluster_with_namespace module {
   frequency  = 600
   lookback   = 600
@@ -156,55 +156,44 @@ ingester prometheus_kube_cluster_with_namespace module {
     "default" : "$input{using}"
   }
 
-  gauge "total_memory_requested" {
-    unit = "bytes"
-
-    source prometheus "total_memory_requested" {
-      query = "label_set(sum by (cluster, namespace)(kube_pod_container_resource_requests{resource='memory', unit='byte'}), 'cluster', '$input{cluster}')"
-      join_on = {
-        "$output{cluster}" = "$input{cluster}"
-      }
-    }
-  }
-
-  gauge "total_cpu_requested" {
+  gauge "unscheduled_pods" {
     unit = "count"
 
-    source prometheus "total_cpu_requested" {
-      query = "label_set(sum by (cluster, namespace) (kube_pod_container_resource_requests{resource='cpu', unit='core'}), 'cluster', '$input{cluster}')"
+    source prometheus "unscheduled_pods" {
+      query = "sum by (cluster, namespace) (increase(kube_pod_status_unschedulable{}[1m]))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "total_unscheduled_pods" {
+  gauge "desired_pods" {
     unit = "count"
 
-    source prometheus "total_unscheduled_pods" {
-      query = "label_set(sum by (cluster, namespace) (kube_pod_status_unschedulable{}), 'cluster', '$input{cluster}')"
+    source prometheus "desired_pods" {
+      query = "sum by (cluster, namespace) (increase(kube_pod_status_phase{}[1m]))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "total_failed_and_unknown_pods" {
+  gauge "failed_and_unknown_pods" {
     unit = "count"
 
-    source prometheus "total_failed_and_unknown_pods" {
-      query = "label_set(sum by (cluster, namespace) (kube_pod_status_phase{phase=~'Failed|Unknown'}), 'cluster', '$input{cluster}')"
+    source prometheus "failed_and_unknown_pods" {
+      query = "sum by (cluster, namespace) (kube_pod_status_phase{phase=~'Failed|Unknown'})"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "total_container_restarts" {
+  gauge "container_restarts" {
     unit = "count"
 
-    source prometheus "total_container_restarts" {
-      query = "label_set(sum by (cluster, namespace) (kube_pod_container_status_restarts_total{}), 'cluster', '$input{cluster}')"
+    source prometheus "container_restarts" {
+      query = "sum by (cluster, namespace) (kube_pod_container_status_restarts_total{})"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
@@ -252,66 +241,53 @@ ingester prometheus_kube_node module {
     "default" : "$input{using}"
   }
 
-  gauge "total_cpu_for_scheduling" {
+  gauge "disk_pressure" {
     unit = "count"
 
-    source prometheus "total_cpu_for_scheduling" {
-      query = "label_set(sum by (cluster, node) (kube_node_status_allocatable{resource='cpu', unit='core'}) - sum by (cluster, node) (kube_pod_container_resource_limits{resource='cpu', unit='core'}), 'cluster', '$input{cluster}')"
-      join_on = {
-        "$output{cluster}" = "$input{cluster}"
-      }
-    }
-  }
-
-  gauge "total_memory_for_scheduling" {
-    unit = "bytes"
-
-    source prometheus "total_memory_for_scheduling" {
-      query = "label_set(sum by (cluster, node) (kube_node_status_allocatable{resource='memory', unit='byte'}) - sum by (cluster, node) (kube_pod_container_resource_limits{resource='memory', unit='byte'}), 'cluster', '$input{cluster}')"
-      join_on = {
-        "$output{cluster}" = "$input{cluster}"
-      }
-    }
-  }
-
-  gauge "out_of_pods" {
-    unit = "count"
-
-    source prometheus "out_of_pods" {
-      query = "label_set(sum by (cluster, node) (kube_node_spec_unschedulable{}), 'cluster', '$input{cluster}')"
+    source prometheus "disk_pressure" {
+      query = <<EOF
+      sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure', status='true'})/
+      sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure'})
+      EOF
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "high_disk_pressure" {
+  gauge "pid_pressure" {
     unit = "count"
 
-    source prometheus "high_disk_pressure" {
-      query = "label_set(sum by (cluster, node) (kube_node_status_condition{condition='DiskPressure', status='true'}), 'cluster', '$input{cluster}')"
+    source prometheus "pid_pressure" {
+      query = <<EOF
+      sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure', status='true'})/
+      sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure'})
+      EOF
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "high_memory_pressure" {
+  gauge "memory_pressure" {
     unit = "count"
 
-    source prometheus "high_memory_pressure" {
-      query = "label_set(sum by (cluster, node) (kube_node_status_condition{condition='MemoryPressure', status='true'}), 'cluster', '$input{cluster}')"
+    source prometheus "memory_pressure" {
+      query = <<EOF
+      sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure', status='true'})/
+      sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure'})
+      EOF
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "high_pid_pressure" {
+  gauge "saturated" {
     unit = "count"
 
-    source prometheus "high_pid_pressure" {
-      query = "label_set(sum by (cluster, node) (kube_node_status_condition{condition='PIDPressure', status='true'}), 'cluster', '$input{cluster}')"
+    source prometheus "saturated" {
+      query = "sum by (cluster, node) (kube_node_spec_unschedulable{})"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }

From 1a250d6f7222dbbf8e6bb55defed1158a4ef2d1d Mon Sep 17 00:00:00 2001
From: Aniket Rao <anik3tra0@gmail.com>
Date: Thu, 7 Oct 2021 10:53:17 +0530
Subject: [PATCH 2/2] update metrics for kube state ingesters

---
 modules/prometheus/k8s_via_kube_state.hcl | 157 ++++++++++++++++------
 1 file changed, 114 insertions(+), 43 deletions(-)

diff --git a/modules/prometheus/k8s_via_kube_state.hcl b/modules/prometheus/k8s_via_kube_state.hcl
index d4f5616..d54f201 100644
--- a/modules/prometheus/k8s_via_kube_state.hcl
+++ b/modules/prometheus/k8s_via_kube_state.hcl
@@ -121,7 +121,6 @@ ingester prometheus_kube_cluster module {
   }
 }
 
-# Done
 ingester prometheus_kube_cluster_with_namespace module {
   frequency  = 600
   lookback   = 600
@@ -160,7 +159,7 @@ ingester prometheus_kube_cluster_with_namespace module {
     unit = "count"
 
     source prometheus "unscheduled_pods" {
-      query = "sum by (cluster, namespace) (increase(kube_pod_status_unschedulable{}[1m]))"
+      query = "label_set(sum by (cluster, namespace) (increase(kube_pod_status_unschedulable{}[1m])))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
@@ -171,7 +170,7 @@ ingester prometheus_kube_cluster_with_namespace module {
     unit = "count"
 
     source prometheus "desired_pods" {
-      query = "sum by (cluster, namespace) (increase(kube_pod_status_phase{}[1m]))"
+      query = "label_set(sum by (cluster, namespace) (increase(kube_pod_status_phase{}[1m])))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
@@ -182,7 +181,7 @@ ingester prometheus_kube_cluster_with_namespace module {
     unit = "count"
 
     source prometheus "failed_and_unknown_pods" {
-      query = "sum by (cluster, namespace) (kube_pod_status_phase{phase=~'Failed|Unknown'})"
+      query = "label_set(sum by (cluster, namespace) (kube_pod_status_phase{phase=~'Failed|Unknown'}))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
@@ -193,7 +192,7 @@ ingester prometheus_kube_cluster_with_namespace module {
     unit = "count"
 
     source prometheus "container_restarts" {
-      query = "sum by (cluster, namespace) (kube_pod_container_status_restarts_total{})"
+      query = "label_set(sum by (cluster, namespace) (kube_pod_container_status_restarts_total{}))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
@@ -201,7 +200,6 @@ ingester prometheus_kube_cluster_with_namespace module {
   }
 }
 
-
 ingester prometheus_kube_node module {
   frequency  = 600
   lookback   = 600
@@ -246,8 +244,10 @@ ingester prometheus_kube_node module {
 
     source prometheus "disk_pressure" {
       query = <<EOF
-      sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure', status='true'})/
-      sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure'})
+      label_set(
+        sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure', status='true'})/
+        sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure'})
+      )
       EOF
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -260,8 +260,10 @@ ingester prometheus_kube_node module {
 
     source prometheus "pid_pressure" {
       query = <<EOF
-      sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure', status='true'})/
-      sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure'})
+      label_set(
+        sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure', status='true'})/
+        sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure'})
+      )
       EOF
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -274,8 +276,9 @@ ingester prometheus_kube_node module {
 
     source prometheus "memory_pressure" {
       query = <<EOF
-      sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure', status='true'})/
-      sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure'})
+        label_set(sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure', status='true'})) /
+        label_set((sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure'}))
+      )
       EOF
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -287,7 +290,7 @@ ingester prometheus_kube_node module {
     unit = "count"
 
     source prometheus "saturated" {
-      query = "sum by (cluster, node) (kube_node_spec_unschedulable{})"
+      query = "label_set(sum by (cluster, node) (kube_node_spec_unschedulable{}))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
@@ -302,7 +305,6 @@ ingester prometheus_kube_pod_grp module {
   resolution = 60
   lag        = 60
 
-
   inputs = "$input{inputs}"
 
   label {
@@ -340,12 +342,44 @@ ingester prometheus_kube_pod_grp module {
     "default" : "$input{using}"
   }
 
-  gauge "total_container_restarts" {
+  gauge "container_restarted" {
     unit = "count"
 
-    source prometheus "total_container_restarts" {
+    source prometheus "container_restarted" {
+      query = <<EOT
+      label_set(
+        sum without (pod) (label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_restarts_total{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'))
+      )
+      EOT
+      join_on = {
+        "$output{cluster}" = "$input{cluster}"
+      }
+    }
+  }
+
+  gauge "containers_failed" {
+    unit = "count"
+
+    source prometheus "containers_failed" {
+      query = <<EOT
+      label_set(
+        sum without (pod) (label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_terminated_reason{reason!='Completed'})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'))
+      )
+      EOT
+      join_on = {
+        "$output{cluster}" = "$input{cluster}"
+      }
+    }
+  }
+
+  gauge "containers_running" {
+    unit = "count"
+
+    source prometheus "containers_running" {
       query = <<EOT
-      label_set(sum without (pod) (label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_restarts_total{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')), 'cluster', '$input{cluster}')
+      label_set(
+        sum without (pod) (label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_running{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'))
+      )
       EOT
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -353,12 +387,14 @@ ingester prometheus_kube_pod_grp module {
     }
   }
 
-  gauge "total_containers_in_error" {
+  gauge "containers_desired" {
     unit = "count"
 
-    source prometheus "total_containers_in_error" {
+    source prometheus "containers_desired" {
       query = <<EOT
-      label_set(sum without (pod) (label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_terminated_reason{reason!='Completed'})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')), 'cluster', '$input{cluster}')
+        label_set(sum without (pod) (label_replace((sum by (cluster, namespace, pod) (kube_pod_container_status_terminated_reason{})) + \
+        label_set(sum by (cluster, namespace, pod) (kube_pod_container_status_running{})) + \
+        label_set(sum by (cluster, namespace, pod) (kube_pod_container_status_waiting{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')))
       EOT
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -366,12 +402,13 @@ ingester prometheus_kube_pod_grp module {
     }
   }
 
-  gauge "running_vs_waiting_containers" {
+  gauge "containers_cold" {
     unit = "count"
 
-    source prometheus "running_vs_waiting_containers" {
+    source prometheus "containers_cold" {
       query = <<EOT
-      label_set(sum without (pod) (label_replace((sum by (cluster, namespace, pod) (kube_pod_container_status_running{}) - sum by (cluster, namespace, pod) (kube_pod_container_status_waiting{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')), 'cluster', '$input{cluster}')
+        label_set(sum without (pod) (label_replace((sum by (cluster, namespace, pod) (kube_pod_container_status_running{})) - \
+        label_set(sum by (cluster, namespace, pod) (kube_pod_container_status_ready{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')))
       EOT
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -429,12 +466,14 @@ ingester prometheus_kube_pod module {
     "default" : "$input{using}"
   }
 
-  gauge "total_container_restarts" {
+  gauge "container_restarted" {
     unit = "count"
 
-    source prometheus "total_container_restarts" {
+    source prometheus "container_restarted" {
       query = <<EOT
-      label_set(label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_restarts_total{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'), 'cluster', '$input{cluster}')
+      label_set(
+        label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_restarts_total{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')
+      )
       EOT
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -442,12 +481,14 @@ ingester prometheus_kube_pod module {
     }
   }
 
-  gauge "total_containers_in_error" {
+  gauge "containers_failed" {
     unit = "count"
 
-    source prometheus "total_containers_in_error" {
+    source prometheus "containers_failed" {
       query = <<EOT
-      label_set(label_replace((sum by (cluster, namespace, pod) (kube_pod_container_status_terminated_reason{reason!='Completed'})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'), 'cluster', '$input{cluster}')
+      label_set(
+        label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_terminated_reason{reason!='Completed'})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')
+      )
       EOT
       join_on = {
         "$output{cluster}" = "$input{cluster}"
@@ -455,20 +496,50 @@ ingester prometheus_kube_pod module {
     }
   }
 
-  gauge "running_vs_waiting_containers" {
+  gauge "containers_running" {
     unit = "count"
 
-    source prometheus "running_vs_waiting_containers" {
+    source prometheus "containers_failed" {
       query = <<EOT
-      label_set(label_replace((sum by (cluster, namespace, pod) (kube_pod_container_status_running{}) - sum by (cluster, namespace, pod) (kube_pod_container_status_waiting{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'), 'cluster', '$input{cluster}')
+      label_set(
+        label_replace((sum by (cluster, namespace, pod_group, pod) (kube_pod_container_status_running{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)')
+      )
       EOT
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
-}
 
+  gauge "containers_desired" {
+    unit = "count"
+
+    source prometheus "containers_desired" {
+      query = <<EOT
+        label_set(label_replace((sum by (cluster, namespace, pod) (kube_pod_container_status_terminated_reason{})) + \
+        label_set(sum by (cluster, namespace, pod) (kube_pod_container_status_running{})) + \
+        label_set(sum by (cluster, namespace, pod) (kube_pod_container_status_waiting{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'))
+      EOT
+      join_on = {
+        "$output{cluster}" = "$input{cluster}"
+      }
+    }
+  }
+
+  gauge "containers_cold" {
+    unit = "count"
+
+    source prometheus "containers_cold" {
+      query = <<EOT
+        label_set(label_replace((sum by (cluster, namespace, pod) (kube_pod_container_status_running{})) - \
+        label_set(sum by (cluster, namespace, pod) (kube_pod_container_status_ready{})), 'pod_group', '$1', 'pod', '(\\D+)-(.*)'))
+      EOT
+      join_on = {
+        "$output{cluster}" = "$input{cluster}"
+      }
+    }
+  }
+}
 
 ingester prometheus_kube_container module {
   frequency  = 600
@@ -589,7 +660,7 @@ ingester prometheus_kube_container module {
   }
 }
 
-ingester prometheus_kube_deployment module {
+ingester prometheus_kube_deployment module  {
   frequency  = 600
   lookback   = 600
   timeout    = 180
@@ -634,32 +705,32 @@ ingester prometheus_kube_deployment module {
     "default" : "$input{using}"
   }
 
-  gauge "total_unavailable_replicas" {
+  gauge "unavailable_replicas" {
     unit = "count"
 
-    source prometheus "total_unavailable_replicas" {
-      query = "label_set(sum by (cluster, namespace, deployment) (kube_deployment_status_replicas_unavailable{}), 'cluster', '$input{cluster}')"
+    source prometheus "unavailable_replicas" {
+      query = "label_set(sum by (cluster, namespace, deployment) (kube_deployment_status_replicas_unavailable{}))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "total_updated_replicas" {
+  gauge "desired_replicas" {
     unit = "count"
 
-    source prometheus "total_updated_replicas" {
-      query = "label_set(sum by (cluster, namespace, deployment) (kube_deployment_status_replicas_updated{}), 'cluster', '$input{cluster}')"
+    source prometheus "desired_replicas" {
+      query = "label_set(sum by (cluster, namespace, deployment) (kube_deployment_spec_replicas{}))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }
     }
   }
 
-  gauge "desired_vs_available_replicas" {
+  gauge "cold_replicas" {
     unit = "count"
-    source prometheus "desired_vs_available_replicas" {
-      query = "label_set(sum by (cluster, namespace, deployment)(kube_deployment_status_replicas{}) - sum by (cluster, namespace, deployment) (kube_deployment_status_replicas_available{}), 'cluster', '$input{cluster}')"
+    source prometheus "cold_replicas" {
+      query = "label_set(sum by (cluster, namespace, deployment) (kube_deployment_status_replicas{}) - sum by (cluster, namespace, deployment)(kube_deployment_status_replicas_available{}))"
       join_on = {
         "$output{cluster}" = "$input{cluster}"
       }