add ephemeral_storage_container_volume_usage metrics (#87)

Co-authored-by: zhihuanzhu <zhihuanzhu@deeproute.ai>
jmcgrath207 · Apr 26, 2024 · a1215e8 · a1215e8
1 parent 4d5249e
commit a1215e8
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 5 deletions.
diff --git a/pkg/pod/factory.go b/pkg/pod/factory.go
@@ -12,6 +12,7 @@ var (
 )
 
 type Collector struct {
+	containerVolumeUsage            bool
 	containerLimitsPercentage       bool
 	containerVolumeLimitsPercentage bool
 	lookup                          *map[string]pod
@@ -23,11 +24,13 @@ type Collector struct {
 
 func NewCollector(sampleInterval int64) Collector {
 	podUsage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_POD_USAGE", "false"))
+	containerVolumeUsage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_VOLUME_USAGE", "false"))
 	containerLimitsPercentage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_LIMIT_PERCENTAGE", "false"))
 	containerVolumeLimitsPercentage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE", "false"))
 	lookup := make(map[string]pod)
 
 	var c = Collector{
+		containerVolumeUsage:            containerVolumeUsage,
 		containerLimitsPercentage:       containerLimitsPercentage,
 		containerVolumeLimitsPercentage: containerVolumeLimitsPercentage,
 		lookup:                          &lookup,

diff --git a/pkg/pod/k8s.go b/pkg/pod/k8s.go
@@ -108,23 +108,23 @@ func (cr Collector) getContainerData(c v1.Container, p v1.Pod) container {
 	setContainer.name = c.Name
 	matchKey := v1.ResourceName("ephemeral-storage")
 
-	if cr.containerVolumeLimitsPercentage && p.Spec.Volumes != nil {
+	if (cr.containerVolumeUsage || cr.containerVolumeLimitsPercentage) && p.Spec.Volumes != nil {
 		collectMounts := false
 
 		podMountsMap := make(map[string]float64)
 		for _, v := range p.Spec.Volumes {
 			if v.VolumeSource.EmptyDir != nil {
+				podMountsMap[v.Name] = 0
+				collectMounts = true
 				if v.VolumeSource.EmptyDir.SizeLimit != nil {
 					podMountsMap[v.Name] = v.VolumeSource.EmptyDir.SizeLimit.AsApproximateFloat64()
 					collectMounts = true
 				}
 			}
-
 		}
 
 		if collectMounts {
 			var collectVolumes []emptyDirVolumes
-
 			for _, volumeMount := range c.VolumeMounts {
 				size, ok := podMountsMap[volumeMount.Name]
 				if ok {

diff --git a/pkg/pod/metrics.go b/pkg/pod/metrics.go
@@ -9,6 +9,7 @@ import (
 
 var (
 	podGaugeVec                        *prometheus.GaugeVec
+	containerVolumeUsageVec            *prometheus.GaugeVec
 	containerPercentageLimitsVec       *prometheus.GaugeVec
 	containerPercentageVolumeLimitsVec *prometheus.GaugeVec
 )
@@ -38,6 +39,28 @@ func (cr Collector) createMetrics() {
 
 	prometheus.MustRegister(podGaugeVec)
 
+	containerVolumeUsageVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "ephemeral_storage_container_volume_usage",
+		Help: "Current ephemeral storage used by a container's volume in a pod",
+	},
+		[]string{
+			// name of pod for Ephemeral Storage
+			"pod_name",
+			// namespace of pod for Ephemeral Storage
+			"pod_namespace",
+			// Name of Node where pod is placed.
+			"node_name",
+			// Name of container
+			"container",
+			// Name of Volume
+			"volume_name",
+			// Name of Mount Path
+			"mount_path",
+		},
+	)
+
+	prometheus.MustRegister(containerVolumeUsageVec)
+
 	containerPercentageLimitsVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "ephemeral_storage_container_limit_percentage",
 		Help: "Percentage of ephemeral storage used by a container in a pod",
@@ -92,7 +115,7 @@ func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName str
 	// 		need to source it from the pod spec
 	// 		make issue upstream with CA advisor
 	// TODO: need to do a grow and shrink test for this.
-	if cr.containerVolumeLimitsPercentage {
+	if cr.containerVolumeUsage {
 		// TODO: what a mess...need to figure out a better way.
 		if okPodResult {
 			for _, c := range podResult.containers {
@@ -103,7 +126,30 @@ func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName str
 								labels := prometheus.Labels{"pod_namespace": podNamespace,
 									"pod_name": podName, "node_name": nodeName, "container": c.name, "volume_name": v.Name,
 									"mount_path": edv.mountPath}
-								containerPercentageVolumeLimitsVec.With(labels).Set((float64(v.UsedBytes) / edv.sizeLimit) * 100.0)
+								containerVolumeUsageVec.With(labels).Set(float64(v.UsedBytes))
+								log.Debug().Msg(fmt.Sprintf("pod %s/%s/%s  on %s with usedBytes: %f", podNamespace, podName, c.name, nodeName, usedBytes))
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	if cr.containerVolumeLimitsPercentage {
+		// TODO: what a mess...need to figure out a better way.
+		if okPodResult {
+			for _, c := range podResult.containers {
+				if c.emptyDirVolumes != nil {
+					for _, edv := range c.emptyDirVolumes {
+						if edv.sizeLimit != 0 {
+							for _, v := range volumes {
+								if edv.name == v.Name {
+									labels := prometheus.Labels{"pod_namespace": podNamespace,
+										"pod_name": podName, "node_name": nodeName, "container": c.name, "volume_name": v.Name,
+										"mount_path": edv.mountPath}
+									containerPercentageVolumeLimitsVec.With(labels).Set((float64(v.UsedBytes) / edv.sizeLimit) * 100.0)
+								}
 							}
 						}
 					}
@@ -141,6 +187,7 @@ func evictPodFromMetrics(p v1.Pod) {
 
 	podGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
 	for _, c := range p.Spec.Containers {
+		containerVolumeUsageVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 		containerPercentageLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 		containerPercentageVolumeLimitsVec.DeletePartialMatch(prometheus.Labels{"container": c.Name})
 	}