From c8d4e859aa4931c612f90edae7d4ddd2694b5c4a Mon Sep 17 00:00:00 2001
From: John McGrath <john.mcgrath207@gmail.com>
Date: Sat, 23 Nov 2024 13:33:40 -0600
Subject: [PATCH] evict pods metrics by dead node

---
 pkg/node/metrics.go | 15 +++++++++------
 pkg/pod/k8s.go      |  2 +-
 pkg/pod/metrics.go  | 14 ++++++++++++--
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/pkg/node/metrics.go b/pkg/node/metrics.go
index 16b26a6..c9e3a77 100644
--- a/pkg/node/metrics.go
+++ b/pkg/node/metrics.go
@@ -2,10 +2,10 @@ package node
 
 import (
 	"fmt"
-	"math"
-
+	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/pod"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog/log"
+	"math"
 )
 
 var (
@@ -93,11 +93,14 @@ func (n *Node) SetMetrics(nodeName string, availableBytes float64, capacityBytes
 
 func (n *Node) evict(node string) {
 	n.Set.Remove(node)
-	nodeAvailableGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
-	nodeCapacityGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
-	nodePercentageGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
+	deleteLabel := prometheus.Labels{"node_name": node}
+
+	nodeAvailableGaugeVec.DeletePartialMatch(deleteLabel)
+	nodeCapacityGaugeVec.DeletePartialMatch(deleteLabel)
+	nodePercentageGaugeVec.DeletePartialMatch(deleteLabel)
 	if n.AdjustedPollingRate {
-		AdjustedPollingRateGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
+		AdjustedPollingRateGaugeVec.DeletePartialMatch(deleteLabel)
 	}
+	pod.EvictPodByNode(&deleteLabel)
 	log.Info().Msgf("Node %s does not exist or is unresponsive. Removed from monitoring", node)
 }
diff --git a/pkg/pod/k8s.go b/pkg/pod/k8s.go
index 284a7a0..ded0083 100644
--- a/pkg/pod/k8s.go
+++ b/pkg/pod/k8s.go
@@ -76,7 +76,7 @@ func (cr Collector) podWatch() {
 			cr.lookupMutex.Lock()
 			delete(*cr.lookup, p.Name)
 			cr.lookupMutex.Unlock()
-			evictPodFromMetrics(*p)
+			evictPodByName(*p)
 		},
 	}
 
diff --git a/pkg/pod/metrics.go b/pkg/pod/metrics.go
index 77038dd..91324a8 100644
--- a/pkg/pod/metrics.go
+++ b/pkg/pod/metrics.go
@@ -200,12 +200,22 @@ func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName str
 	}
 }
 
-func evictPodFromMetrics(p v1.Pod) {
-
+// Evicts exporter metrics by pod and container name
+func evictPodByName(p v1.Pod) {
 	podGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
+	// TODO: Look into removing this for loop and delete by pod_name
+	// e.g. containerVolumeUsageVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
 	for _, c := range p.Spec.Containers {
 		containerVolumeUsageVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 		containerPercentageLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 		containerPercentageVolumeLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 	}
 }
+
+// EvictPodByNode Evicts exporter metrics by Node
+func EvictPodByNode(deleteLabel *prometheus.Labels) {
+	podGaugeVec.DeletePartialMatch(*deleteLabel)
+	containerVolumeUsageVec.DeletePartialMatch(*deleteLabel)
+	containerPercentageLimitsVec.DeletePartialMatch(*deleteLabel)
+	containerPercentageVolumeLimitsVec.DeletePartialMatch(*deleteLabel)
+}