From bbd4e1ce63d9c4dbfb92b1612b842ec5aed989fd Mon Sep 17 00:00:00 2001 From: mattcontinisio Date: Mon, 16 Dec 2024 17:28:49 -0500 Subject: [PATCH] Add inode metrics (#136) --- README.md | 1 + chart/README.md | 1 + chart/templates/DeployType.yaml | 4 ++ chart/test-values.yaml | 2 + chart/values.yaml | 2 + cmd/app/main.go | 17 ++++++--- pkg/node/metrics.go | 5 ++- pkg/pod/factory.go | 6 ++- pkg/pod/metrics.go | 66 ++++++++++++++++++++++++++++++++- tests/e2e/deployment_test.go | 24 ++++++++++++ 10 files changed, 119 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index cd09b12..259712c 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node | | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node | | metrics.ephemeral_storage_pod_usage | bool | `true` | Current ephemeral byte usage of pod | +| metrics.ephemeral_storage_inodes | bool | `true` | Current ephemeral inode usage of pod | | metrics.port | int | `9100` | Adjust the metric port as needed (default 9100) | | nodeSelector | object | `{}` | | | podAnnotations | object | `{}` | | diff --git a/chart/README.md b/chart/README.md index 6d7e23d..55e860f 100644 --- a/chart/README.md +++ b/chart/README.md @@ -35,6 +35,7 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node | | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node | | metrics.ephemeral_storage_pod_usage | bool | `true` | Current ephemeral byte usage of pod | +| metrics.ephemeral_storage_inodes | bool | `true` | Current ephemeral inode usage of pod | | metrics.port | int | `9100` | Adjust the metric port as needed (default 9100) | | nodeSelector | object | `{}` | | | podAnnotations | object | `{}` | | diff --git a/chart/templates/DeployType.yaml b/chart/templates/DeployType.yaml index 2adec17..54e6864 100644 --- a/chart/templates/DeployType.yaml +++ b/chart/templates/DeployType.yaml @@ -121,6 +121,10 @@ spec: - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE value: "{{ .Values.metrics.ephemeral_storage_container_volume_limit_percentage }}" {{- end }} + {{- if .Values.metrics.ephemeral_storage_inodes }} + - name: EPHEMERAL_STORAGE_INODES + value: "{{ .Values.metrics.ephemeral_storage_inodes }}" + {{- end }} {{- if .Values.kubelet.scrape }} - name: SCRAPE_FROM_KUBELET value: "{{ .Values.kubelet.scrape }}" diff --git a/chart/test-values.yaml b/chart/test-values.yaml index 220f91f..b825142 100644 --- a/chart/test-values.yaml +++ b/chart/test-values.yaml @@ -27,6 +27,8 @@ metrics: ephemeral_storage_container_volume_limit_percentage: true # -- Current ephemeral byte usage of pod ephemeral_storage_pod_usage: true + # -- Current ephemeral inode usage of pod + ephemeral_storage_inodes: true # -- Available ephemeral storage for a node ephemeral_storage_node_available: true # -- Capacity of ephemeral storage for a node diff --git a/chart/values.yaml b/chart/values.yaml index 3411b1b..550cc10 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -36,6 +36,8 @@ metrics: ephemeral_storage_container_volume_limit_percentage: true # -- Current ephemeral byte usage of pod ephemeral_storage_pod_usage: true + # -- Current ephemeral inode usage of pod + ephemeral_storage_inodes: true # -- Available ephemeral storage for a node ephemeral_storage_node_available: true # -- Capacity of ephemeral storage for a node diff --git a/cmd/app/main.go b/cmd/app/main.go index 26e4e2f..c7d260f 100644 --- a/cmd/app/main.go +++ b/cmd/app/main.go @@ -4,6 +4,10 @@ import ( "encoding/json" "flag" "fmt" + "net/http" + "strconv" + "time" + "github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/dev" "github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/node" "github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/pod" @@ -11,9 +15,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog/log" - "net/http" - "strconv" - "time" ) var ( @@ -36,6 +37,9 @@ type ephemeralStorageMetrics struct { AvailableBytes float64 `json:"availableBytes"` CapacityBytes float64 `json:"capacityBytes"` UsedBytes float64 `json:"usedBytes"` + Inodes float64 `json:"inodes"` + InodesFree float64 `json:"inodesFree"` + InodesUsed float64 `json:"inodesUsed"` } `json:"ephemeral-storage"` Volumes []pod.Volume `json:"volume,omitempty"` @@ -63,12 +67,15 @@ func setMetrics(nodeName string) { usedBytes := p.EphemeralStorage.UsedBytes availableBytes := p.EphemeralStorage.AvailableBytes capacityBytes := p.EphemeralStorage.CapacityBytes - if podNamespace == "" || (usedBytes == 0 && availableBytes == 0 && capacityBytes == 0) { + inodes := p.EphemeralStorage.Inodes + inodesFree := p.EphemeralStorage.InodesFree + inodesUsed := p.EphemeralStorage.InodesUsed + if podNamespace == "" || (usedBytes == 0 && availableBytes == 0 && capacityBytes == 0 && inodes == 0 && inodesFree == 0 && inodesUsed == 0) { log.Warn().Msg(fmt.Sprintf("pod %s/%s on %s has no metrics on its ephemeral storage usage", podName, podNamespace, nodeName)) continue } Node.SetMetrics(nodeName, availableBytes, capacityBytes) - Pod.SetMetrics(podName, podNamespace, nodeName, usedBytes, availableBytes, capacityBytes, p.Volumes) + Pod.SetMetrics(podName, podNamespace, nodeName, usedBytes, availableBytes, capacityBytes, inodes, inodesFree, inodesUsed, p.Volumes) } adjustTime := sampleIntervalMill - time.Now().Sub(start).Milliseconds() diff --git a/pkg/node/metrics.go b/pkg/node/metrics.go index c9e3a77..cb5e7c1 100644 --- a/pkg/node/metrics.go +++ b/pkg/node/metrics.go @@ -2,10 +2,11 @@ package node import ( "fmt" + "math" + "github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/pod" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog/log" - "math" ) var ( @@ -72,7 +73,7 @@ func (n *Node) SetMetrics(nodeName string, availableBytes float64, capacityBytes if n.nodeAvailable { nodeAvailableGaugeVec.With(prometheus.Labels{"node_name": nodeName}).Set(availableBytes) - log.Debug().Msg(fmt.Sprintf("Node: %s availble bytes: %f", nodeName, availableBytes)) + log.Debug().Msg(fmt.Sprintf("Node: %s available bytes: %f", nodeName, availableBytes)) } if n.nodeCapacity { diff --git a/pkg/pod/factory.go b/pkg/pod/factory.go index e733e40..d328f41 100644 --- a/pkg/pod/factory.go +++ b/pkg/pod/factory.go @@ -1,9 +1,10 @@ package pod import ( - "github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/dev" "strconv" "sync" + + "github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/dev" ) var ( @@ -15,6 +16,7 @@ type Collector struct { containerVolumeUsage bool containerLimitsPercentage bool containerVolumeLimitsPercentage bool + inodes bool lookup *map[string]pod lookupMutex *sync.RWMutex podUsage bool @@ -27,12 +29,14 @@ func NewCollector(sampleInterval int64) Collector { containerVolumeUsage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_VOLUME_USAGE", "false")) containerLimitsPercentage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_LIMIT_PERCENTAGE", "false")) containerVolumeLimitsPercentage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE", "false")) + inodes, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_INODES", "false")) lookup := make(map[string]pod) var c = Collector{ containerVolumeUsage: containerVolumeUsage, containerLimitsPercentage: containerLimitsPercentage, containerVolumeLimitsPercentage: containerVolumeLimitsPercentage, + inodes: inodes, lookup: &lookup, lookupMutex: &lookupMutex, podUsage: podUsage, diff --git a/pkg/pod/metrics.go b/pkg/pod/metrics.go index 91324a8..0c8de90 100644 --- a/pkg/pod/metrics.go +++ b/pkg/pod/metrics.go @@ -14,6 +14,9 @@ var ( containerVolumeUsageVec *prometheus.GaugeVec containerPercentageLimitsVec *prometheus.GaugeVec containerPercentageVolumeLimitsVec *prometheus.GaugeVec + inodesGaugeVec *prometheus.GaugeVec + inodesFreeGaugeVec *prometheus.GaugeVec + inodesUsedGaugeVec *prometheus.GaugeVec ) type Volume struct { @@ -104,9 +107,57 @@ func (cr Collector) createMetrics() { ) prometheus.MustRegister(containerPercentageVolumeLimitsVec) + + inodesGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_inodes", + Help: "Maximum number of inodes in the pod", + }, + []string{ + // name of pod for Ephemeral Storage + "pod_name", + // namespace of pod for Ephemeral Storage + "pod_namespace", + // Name of Node where pod is placed. + "node_name", + }, + ) + + prometheus.MustRegister(inodesGaugeVec) + + inodesFreeGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_inodes_free", + Help: "Number of free inodes in the pod", + }, + []string{ + // name of pod for Ephemeral Storage + "pod_name", + // namespace of pod for Ephemeral Storage + "pod_namespace", + // Name of Node where pod is placed. + "node_name", + }, + ) + + prometheus.MustRegister(inodesFreeGaugeVec) + + inodesUsedGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ephemeral_storage_inodes_used", + Help: "Number of used inodes in the pod", + }, + []string{ + // name of pod for Ephemeral Storage + "pod_name", + // namespace of pod for Ephemeral Storage + "pod_namespace", + // Name of Node where pod is placed. + "node_name", + }, + ) + + prometheus.MustRegister(inodesUsedGaugeVec) } -func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName string, usedBytes float64, availableBytes float64, capacityBytes float64, volumes []Volume) { +func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName string, usedBytes float64, availableBytes float64, capacityBytes float64, inodes float64, inodesFree float64, inodesUsed float64, volumes []Volume) { var setValue float64 cr.lookupMutex.RLock() @@ -198,11 +249,24 @@ func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName str podGaugeVec.With(labels).Set(usedBytes) log.Debug().Msg(fmt.Sprintf("pod %s/%s on %s with usedBytes: %f", podNamespace, podName, nodeName, usedBytes)) } + + if cr.inodes { + labels := prometheus.Labels{"pod_namespace": podNamespace, + "pod_name": podName, "node_name": nodeName} + inodesGaugeVec.With(labels).Set(inodes) + inodesFreeGaugeVec.With(labels).Set(inodesFree) + inodesUsedGaugeVec.With(labels).Set(inodesUsed) + log.Debug().Msg(fmt.Sprintf("pod %s/%s on %s with inodes: %f, inodesFree: %f, inodesUsed: %f", podNamespace, podName, nodeName, inodes, inodesFree, inodesUsed)) + } } // Evicts exporter metrics by pod and container name func evictPodByName(p v1.Pod) { podGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name}) + inodesGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name}) + inodesFreeGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name}) + inodesUsedGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name}) + // TODO: Look into removing this for loop and delete by pod_name // e.g. containerVolumeUsageVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name}) for _, c := range p.Spec.Containers { diff --git a/tests/e2e/deployment_test.go b/tests/e2e/deployment_test.go index d4bd119..6a1a0cd 100644 --- a/tests/e2e/deployment_test.go +++ b/tests/e2e/deployment_test.go @@ -232,6 +232,30 @@ func getContainerVolumeUsage(podName string) float64 { return currentPodSize } +func getInodes(podName string) float64 { + output := requestPrometheusString() + re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_inodes.+pod_name="%s.+\}\s(.+)`, podName)) + match := re.FindAllStringSubmatch(output, 2) + inodes, _ := strconv.ParseFloat(match[0][1], 64) + return inodes +} + +func getInodesFree(podName string) float64 { + output := requestPrometheusString() + re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_inodes_free.+pod_name="%s.+\}\s(.+)`, podName)) + match := re.FindAllStringSubmatch(output, 2) + inodesFree, _ := strconv.ParseFloat(match[0][1], 64) + return inodesFree +} + +func getInodesUsed(podName string) float64 { + output := requestPrometheusString() + re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_inodes_used.+pod_name="%s.+\}\s(.+)`, podName)) + match := re.FindAllStringSubmatch(output, 2) + inodesUsed, _ := strconv.ParseFloat(match[0][1], 64) + return inodesUsed +} + func WatchEphemeralSize(podName string, desiredSizeChange float64, timeout time.Duration, getPodSize getPodSize) { // Watch Prometheus Metrics until the ephemeral storage shrinks or grows to a certain desiredSizeChange. var currentPodSize float64