Add inode metrics (#136)

README.md

@@ -53,6 +53,7 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral
 | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node |
 | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node |
 | metrics.ephemeral_storage_pod_usage | bool | `true` | Current ephemeral byte usage of pod |
+| metrics.ephemeral_storage_inodes | bool | `true` | Current ephemeral inode usage of pod |
 | metrics.port | int | `9100` | Adjust the metric port as needed (default 9100) |
 | nodeSelector | object | `{}` |  |
 | podAnnotations | object | `{}` |  |

chart/README.md

@@ -35,6 +35,7 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral
 | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node |
 | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node |
 | metrics.ephemeral_storage_pod_usage | bool | `true` | Current ephemeral byte usage of pod |
+| metrics.ephemeral_storage_inodes | bool | `true` | Current ephemeral inode usage of pod |
 | metrics.port | int | `9100` | Adjust the metric port as needed (default 9100) |
 | nodeSelector | object | `{}` |  |
 | podAnnotations | object | `{}` |  |

chart/templates/DeployType.yaml

@@ -121,6 +121,10 @@ spec:
             - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE
               value: "{{ .Values.metrics.ephemeral_storage_container_volume_limit_percentage }}"
               {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_inodes }}
+            - name: EPHEMERAL_STORAGE_INODES
+              value: "{{ .Values.metrics.ephemeral_storage_inodes }}"
+              {{- end }}
               {{- if .Values.kubelet.scrape }}
             - name: SCRAPE_FROM_KUBELET
               value: "{{ .Values.kubelet.scrape }}"

chart/test-values.yaml

@@ -27,6 +27,8 @@ metrics:
   ephemeral_storage_container_volume_limit_percentage: true
   # -- Current ephemeral byte usage of pod
   ephemeral_storage_pod_usage: true
+  # -- Current ephemeral inode usage of pod
+  ephemeral_storage_inodes: true
   # -- Available ephemeral storage for a node
   ephemeral_storage_node_available: true
   # -- Capacity of ephemeral storage for a node

chart/values.yaml

@@ -36,6 +36,8 @@ metrics:
   ephemeral_storage_container_volume_limit_percentage: true
   # -- Current ephemeral byte usage of pod
   ephemeral_storage_pod_usage: true
+  # -- Current ephemeral inode usage of pod
+  ephemeral_storage_inodes: true
   # -- Available ephemeral storage for a node
   ephemeral_storage_node_available: true
   # -- Capacity of ephemeral storage for a node

cmd/app/main.go

@@ -4,16 +4,17 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
 	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/dev"
 	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/node"
 	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/pod"
 	"github.com/panjf2000/ants/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog/log"
-	"net/http"
-	"strconv"
-	"time"
 )
 
 var (
@@ -36,6 +37,9 @@ type ephemeralStorageMetrics struct {
 			AvailableBytes float64 `json:"availableBytes"`
 			CapacityBytes  float64 `json:"capacityBytes"`
 			UsedBytes      float64 `json:"usedBytes"`
+			Inodes         float64 `json:"inodes"`
+			InodesFree     float64 `json:"inodesFree"`
+			InodesUsed     float64 `json:"inodesUsed"`
 		} `json:"ephemeral-storage"`
 
 		Volumes []pod.Volume `json:"volume,omitempty"`
@@ -63,12 +67,15 @@ func setMetrics(nodeName string) {
 		usedBytes := p.EphemeralStorage.UsedBytes
 		availableBytes := p.EphemeralStorage.AvailableBytes
 		capacityBytes := p.EphemeralStorage.CapacityBytes
-		if podNamespace == "" || (usedBytes == 0 && availableBytes == 0 && capacityBytes == 0) {
+		inodes := p.EphemeralStorage.Inodes
+		inodesFree := p.EphemeralStorage.InodesFree
+		inodesUsed := p.EphemeralStorage.InodesUsed
+		if podNamespace == "" || (usedBytes == 0 && availableBytes == 0 && capacityBytes == 0 && inodes == 0 && inodesFree == 0 && inodesUsed == 0) {
 			log.Warn().Msg(fmt.Sprintf("pod %s/%s on %s has no metrics on its ephemeral storage usage", podName, podNamespace, nodeName))
 			continue
 		}
 		Node.SetMetrics(nodeName, availableBytes, capacityBytes)
-		Pod.SetMetrics(podName, podNamespace, nodeName, usedBytes, availableBytes, capacityBytes, p.Volumes)
+		Pod.SetMetrics(podName, podNamespace, nodeName, usedBytes, availableBytes, capacityBytes, inodes, inodesFree, inodesUsed, p.Volumes)
 	}
 
 	adjustTime := sampleIntervalMill - time.Now().Sub(start).Milliseconds()

pkg/node/metrics.go

@@ -2,10 +2,11 @@ package node
 
 import (
 	"fmt"
+	"math"
+
 	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/pod"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog/log"
-	"math"
 )
 
 var (
@@ -72,7 +73,7 @@ func (n *Node) SetMetrics(nodeName string, availableBytes float64, capacityBytes
 
 	if n.nodeAvailable {
 		nodeAvailableGaugeVec.With(prometheus.Labels{"node_name": nodeName}).Set(availableBytes)
-		log.Debug().Msg(fmt.Sprintf("Node: %s availble bytes: %f", nodeName, availableBytes))
+		log.Debug().Msg(fmt.Sprintf("Node: %s available bytes: %f", nodeName, availableBytes))
 	}
 
 	if n.nodeCapacity {

pkg/pod/factory.go

@@ -1,9 +1,10 @@
 package pod
 
 import (
-	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/dev"
 	"strconv"
 	"sync"
+
+	"github.com/jmcgrath207/k8s-ephemeral-storage-metrics/pkg/dev"
 )
 
 var (
@@ -15,6 +16,7 @@ type Collector struct {
 	containerVolumeUsage            bool
 	containerLimitsPercentage       bool
 	containerVolumeLimitsPercentage bool
+	inodes                          bool
 	lookup                          *map[string]pod
 	lookupMutex                     *sync.RWMutex
 	podUsage                        bool
@@ -27,12 +29,14 @@ func NewCollector(sampleInterval int64) Collector {
 	containerVolumeUsage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_VOLUME_USAGE", "false"))
 	containerLimitsPercentage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_LIMIT_PERCENTAGE", "false"))
 	containerVolumeLimitsPercentage, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE", "false"))
+	inodes, _ := strconv.ParseBool(dev.GetEnv("EPHEMERAL_STORAGE_INODES", "false"))
 	lookup := make(map[string]pod)
 
 	var c = Collector{
 		containerVolumeUsage:            containerVolumeUsage,
 		containerLimitsPercentage:       containerLimitsPercentage,
 		containerVolumeLimitsPercentage: containerVolumeLimitsPercentage,
+		inodes:                          inodes,
 		lookup:                          &lookup,
 		lookupMutex:                     &lookupMutex,
 		podUsage:                        podUsage,

pkg/pod/metrics.go

@@ -14,6 +14,9 @@ var (
 	containerVolumeUsageVec            *prometheus.GaugeVec
 	containerPercentageLimitsVec       *prometheus.GaugeVec
 	containerPercentageVolumeLimitsVec *prometheus.GaugeVec
+	inodesGaugeVec                     *prometheus.GaugeVec
+	inodesFreeGaugeVec                 *prometheus.GaugeVec
+	inodesUsedGaugeVec                 *prometheus.GaugeVec
 )
 
 type Volume struct {
@@ -104,9 +107,57 @@ func (cr Collector) createMetrics() {
 	)
 
 	prometheus.MustRegister(containerPercentageVolumeLimitsVec)
+
+	inodesGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "ephemeral_storage_inodes",
+		Help: "Maximum number of inodes in the pod",
+	},
+		[]string{
+			// name of pod for Ephemeral Storage
+			"pod_name",
+			// namespace of pod for Ephemeral Storage
+			"pod_namespace",
+			// Name of Node where pod is placed.
+			"node_name",
+		},
+	)
+
+	prometheus.MustRegister(inodesGaugeVec)
+
+	inodesFreeGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "ephemeral_storage_inodes_free",
+		Help: "Number of free inodes in the pod",
+	},
+		[]string{
+			// name of pod for Ephemeral Storage
+			"pod_name",
+			// namespace of pod for Ephemeral Storage
+			"pod_namespace",
+			// Name of Node where pod is placed.
+			"node_name",
+		},
+	)
+
+	prometheus.MustRegister(inodesFreeGaugeVec)
+
+	inodesUsedGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "ephemeral_storage_inodes_used",
+		Help: "Number of used inodes in the pod",
+	},
+		[]string{
+			// name of pod for Ephemeral Storage
+			"pod_name",
+			// namespace of pod for Ephemeral Storage
+			"pod_namespace",
+			// Name of Node where pod is placed.
+			"node_name",
+		},
+	)
+
+	prometheus.MustRegister(inodesUsedGaugeVec)
 }
 
-func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName string, usedBytes float64, availableBytes float64, capacityBytes float64, volumes []Volume) {
+func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName string, usedBytes float64, availableBytes float64, capacityBytes float64, inodes float64, inodesFree float64, inodesUsed float64, volumes []Volume) {
 
 	var setValue float64
 	cr.lookupMutex.RLock()
@@ -198,11 +249,24 @@ func (cr Collector) SetMetrics(podName string, podNamespace string, nodeName str
 		podGaugeVec.With(labels).Set(usedBytes)
 		log.Debug().Msg(fmt.Sprintf("pod %s/%s on %s with usedBytes: %f", podNamespace, podName, nodeName, usedBytes))
 	}
+
+	if cr.inodes {
+		labels := prometheus.Labels{"pod_namespace": podNamespace,
+			"pod_name": podName, "node_name": nodeName}
+		inodesGaugeVec.With(labels).Set(inodes)
+		inodesFreeGaugeVec.With(labels).Set(inodesFree)
+		inodesUsedGaugeVec.With(labels).Set(inodesUsed)
+		log.Debug().Msg(fmt.Sprintf("pod %s/%s on %s with inodes: %f, inodesFree: %f, inodesUsed: %f", podNamespace, podName, nodeName, inodes, inodesFree, inodesUsed))
+	}
 }
 
 // Evicts exporter metrics by pod and container name
 func evictPodByName(p v1.Pod) {
 	podGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
+	inodesGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
+	inodesFreeGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
+	inodesUsedGaugeVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
+
 	// TODO: Look into removing this for loop and delete by pod_name
 	// e.g. containerVolumeUsageVec.DeletePartialMatch(prometheus.Labels{"pod_name": p.Name})
 	for _, c := range p.Spec.Containers {

tests/e2e/deployment_test.go

@@ -232,6 +232,30 @@ func getContainerVolumeUsage(podName string) float64 {
 	return currentPodSize
 }
 
+func getInodes(podName string) float64 {
+	output := requestPrometheusString()
+	re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_inodes.+pod_name="%s.+\}\s(.+)`, podName))
+	match := re.FindAllStringSubmatch(output, 2)
+	inodes, _ := strconv.ParseFloat(match[0][1], 64)
+	return inodes
+}
+
+func getInodesFree(podName string) float64 {
+	output := requestPrometheusString()
+	re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_inodes_free.+pod_name="%s.+\}\s(.+)`, podName))
+	match := re.FindAllStringSubmatch(output, 2)
+	inodesFree, _ := strconv.ParseFloat(match[0][1], 64)
+	return inodesFree
+}
+
+func getInodesUsed(podName string) float64 {
+	output := requestPrometheusString()
+	re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_inodes_used.+pod_name="%s.+\}\s(.+)`, podName))
+	match := re.FindAllStringSubmatch(output, 2)
+	inodesUsed, _ := strconv.ParseFloat(match[0][1], 64)
+	return inodesUsed
+}
+
 func WatchEphemeralSize(podName string, desiredSizeChange float64, timeout time.Duration, getPodSize getPodSize) {
 	// Watch Prometheus Metrics until the ephemeral storage shrinks or grows to a certain desiredSizeChange.
 	var currentPodSize float64