Evict pods metrics if node is dead (#134)

jmcgrath207 · Nov 23, 2024 · c19635f · c19635f
1 parent 61b8248
commit c19635f
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 9 deletions.
diff --git a/pkg/node/metrics.go b/pkg/node/metrics.go
@@ -2,10 +2,10 @@ package node
 
 import (
 	"fmt"
-	"math"
-
+	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/pod"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog/log"
+	"math"
 )
 
 var (
@@ -93,11 +93,14 @@ func (n *Node) SetMetrics(nodeName string, availableBytes float64, capacityBytes
 
 func (n *Node) evict(node string) {
 	n.Set.Remove(node)
-	nodeAvailableGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
-	nodeCapacityGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
-	nodePercentageGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
+	deleteLabel := prometheus.Labels{"node_name": node}
+
+	nodeAvailableGaugeVec.DeletePartialMatch(deleteLabel)
+	nodeCapacityGaugeVec.DeletePartialMatch(deleteLabel)
+	nodePercentageGaugeVec.DeletePartialMatch(deleteLabel)
 	if n.AdjustedPollingRate {
-		AdjustedPollingRateGaugeVec.DeletePartialMatch(prometheus.Labels{"node_name": node})
+		AdjustedPollingRateGaugeVec.DeletePartialMatch(deleteLabel)
 	}
+	pod.EvictPodByNode(&deleteLabel)
 	log.Info().Msgf("Node %s does not exist or is unresponsive. Removed from monitoring", node)
 }
diff --git a/pkg/pod/k8s.go b/pkg/pod/k8s.go
@@ -76,7 +76,7 @@ func (cr Collector) podWatch() {
 			cr.lookupMutex.Lock()
 			delete(*cr.lookup, p.Name)
 			cr.lookupMutex.Unlock()
-			evictPodFromMetrics(*p)
+			evictPodByName(*p)
 		},
 	}
 

diff --git a/pkg/pod/metrics.go b/pkg/pod/metrics.go
@@ -200,12 +200,22 @@ func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName str
 	}
 }
 
-func evictPodFromMetrics(p v1.Pod) {
-
+// Evicts exporter metrics by pod and container name
+func evictPodByName(p v1.Pod) {
 	podGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
+	// TODO: Look into removing this for loop and delete by pod_name
+	// e.g. containerVolumeUsageVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
 	for _, c := range p.Spec.Containers {
 		containerVolumeUsageVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 		containerPercentageLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 		containerPercentageVolumeLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 	}
 }
+
+// EvictPodByNode Evicts exporter metrics by Node
+func EvictPodByNode(deleteLabel *prometheus.Labels) {
+	podGaugeVec.DeletePartialMatch(*deleteLabel)
+	containerVolumeUsageVec.DeletePartialMatch(*deleteLabel)
+	containerPercentageLimitsVec.DeletePartialMatch(*deleteLabel)
+	containerPercentageVolumeLimitsVec.DeletePartialMatch(*deleteLabel)
+}