updated poll rate testing (#23)

README.md

@@ -24,10 +24,11 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral
 |-----|------|---------|-------------|
 | deploy_type | string | `"DaemonSet"` |  |
 | dev.enabled | bool | `false` |  |
+| extra.adjusted_polling_rate | bool | `false` | Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing. |
 | image.imagePullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics"` |  |
-| image.tag | string | `"1.0.1"` |  |
-| interval | int | `15` |  |
+| image.tag | string | `"1.0.2"` |  |
+| interval | int | `15` | Polling rate for exporter |
 | log_level | string | `"info"` |  |
 | prometheus.release | string | `"kube-prometheus-stack"` |  |
 

chart/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: k8s-ephemeral-storage-metrics
-version: 1.0.1
-appVersion: 1.0.1
+version: 1.0.2
+appVersion: 1.0.2
 kubeVersion: ">=1.21.0-0"
 description: Ephemeral storage metrics for prometheus operator.
 home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics

chart/README.md

@@ -11,10 +11,11 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral
 |-----|------|---------|-------------|
 | deploy_type | string | `"DaemonSet"` |  |
 | dev.enabled | bool | `false` |  |
+| extra.adjusted_polling_rate | bool | `false` | Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing. |
 | image.imagePullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics"` |  |
-| image.tag | string | `"1.0.1"` |  |
-| interval | int | `15` |  |
+| image.tag | string | `"1.0.2"` |  |
+| interval | int | `15` | Polling rate for exporter |
 | log_level | string | `"info"` |  |
 | prometheus.release | string | `"kube-prometheus-stack"` |  |
 

chart/index.yaml

@@ -1,6 +1,28 @@
 apiVersion: v1
 entries:
   k8s-ephemeral-storage-metrics:
+  - annotations:
+      artifacthub.io/license: MIT
+      artifacthub.io/links: |
+        - name: Documentation
+          url: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
+      artifacthub.io/prerelease: "false"
+    apiVersion: v2
+    appVersion: 1.0.2
+    created: "2023-10-20T13:24:12.480216334-05:00"
+    description: Ephemeral storage metrics for prometheus operator.
+    digest: 9cf902d55ce385eae40e3542510b8bfb8be7121c4c5ff6fa71c02176ff643fa5
+    home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
+    keywords:
+    - kubernetes
+    - metrics
+    kubeVersion: '>=1.21.0-0'
+    name: k8s-ephemeral-storage-metrics
+    sources:
+    - https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
+    urls:
+    - k8s-ephemeral-storage-metrics-1.0.2.tgz
+    version: 1.0.2
   - annotations:
       artifacthub.io/license: MIT
       artifacthub.io/links: |
@@ -9,7 +31,7 @@ entries:
       artifacthub.io/prerelease: "false"
     apiVersion: v2
     appVersion: 1.0.1
-    created: "2023-10-17T23:57:10.241409772-05:00"
+    created: "2023-10-20T13:24:12.479989623-05:00"
     description: Ephemeral storage metrics for prometheus operator.
     digest: b81465591319d6afd6d750061d7a52fa6888a433e38bbd9ec49670e7349a5578
     home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
@@ -31,7 +53,7 @@ entries:
       artifacthub.io/prerelease: "false"
     apiVersion: v2
     appVersion: 1.0.0
-    created: "2023-10-17T23:57:10.240782502-05:00"
+    created: "2023-10-20T13:24:12.479774744-05:00"
     description: Ephemeral storage metrics for prometheus operator.
     digest: 1b3adbccb37a844e66826426c3e735f65f65639f910eafc9ca2ff85b5f083a40
     home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
@@ -45,4 +67,4 @@ entries:
     urls:
     - k8s-ephemeral-storage-metrics-1.0.0.tgz
     version: 1.0.0
-generated: "2023-10-17T23:57:10.240230462-05:00"
+generated: "2023-10-20T13:24:12.479459598-05:00"

chart/templates/DeployType.yaml

@@ -57,6 +57,10 @@ spec:
               value: "{{ .Values.interval }}"
             - name: LOG_LEVEL
               value: "{{ .Values.log_level }}"
+              {{ if .Values.extra.adjusted_polling_rate }}
+            - name: ADJUSTED_POLLING_RATE
+              value: "{{ .Values.extra.adjusted_polling_rate }}"
+              {{ end }}
             - name: CURRENT_NODE_NAME
               valueFrom:
                 fieldRef:

chart/values.yaml

@@ -1,14 +1,19 @@
 image:
   repository: ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics
-  tag: 1.0.1
+  tag: 1.0.2
   imagePullPolicy: IfNotPresent
 log_level: info
 deploy_type: DaemonSet
 # Note in testing, Kube API does not refresh faster than 10 seconds
+# -- Polling rate for exporter
 interval: 15 # Seconds
 prometheus:
   release: kube-prometheus-stack
 
+extra:
+  # -- Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing.
+  adjusted_polling_rate: false
+
 # For local development and testing
 dev:
   enabled: false

main.go

@@ -22,10 +22,12 @@ import (
 )
 
 var (
-	inCluster      string
-	clientset      *kubernetes.Clientset
-	currentNode    string
-	sampleInterval int64
+	inCluster           string
+	clientset           *kubernetes.Clientset
+	currentNode         string
+	sampleInterval      int64
+	adjustedPollingRate bool
+	adjustedTimeGauge   prometheus.Gauge
 )
 
 func getEnv(key, fallback string) string {
@@ -99,7 +101,7 @@ func getMetrics() {
 
 	opsQueued := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "ephemeral_storage_pod_usage",
-		Help: "Used to expose Ephemeral Storage metrics for pod ",
+		Help: "Used to expose Ephemeral Storage metrics for pod in bytes ",
 	},
 		[]string{
 			// name of pod for Ephemeral Storage
@@ -116,13 +118,14 @@ func getMetrics() {
 	log.Debug().Msg(fmt.Sprintf("getMetrics has been invoked"))
 	currentNode = getEnv("CURRENT_NODE_NAME", "")
 
-	adjustedTimeGauge := prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ephemeral_storage_polling_rate_seconds",
-		Help: "A Node's adjustTime polling rate time after a Node API query.",
-	})
-
-	prometheus.MustRegister(adjustedTimeGauge)
+	if adjustedPollingRate {
+		adjustedTimeGauge = prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "ephemeral_storage_adjusted_polling_rate",
+			Help: "AdjustTime polling rate time after a Node API queries in Milliseconds",
+		})
 
+		prometheus.MustRegister(adjustedTimeGauge)
+	}
 	sampleInterval, _ = strconv.ParseInt(getEnv("SCRAPE_INTERVAL", "15"), 10, 64)
 	sampleInterval = sampleInterval * 1000
 	adjustTime := sampleInterval
@@ -138,8 +141,7 @@ func getMetrics() {
 		var data ephemeralStorageMetrics
 		_ = json.Unmarshal(content, &data)
 
-		opsQueued.Reset() // reset this metrics in the Exporter
-		adjustedTimeGauge.Set(float64(adjustTime / 1000.0))
+		opsQueued.Reset() // reset this metrics in the Exporter to flush dead pods
 
 		nodeName := data.Node.NodeName
 		for _, pod := range data.Pods {
@@ -152,11 +154,13 @@ func getMetrics() {
 			}
 			opsQueued.With(prometheus.Labels{"pod_namespace": podNamespace,
 				"pod_name": podName, "node_name": nodeName}).Set(usedBytes)
+			if adjustedPollingRate {
+				adjustedTimeGauge.Set(float64(adjustTime))
+			}
 
 			log.Debug().Msg(fmt.Sprintf("pod %s/%s on %s with usedBytes: %f", podNamespace, podName, nodeName, usedBytes))
 		}
 
-		// TODO: Fix Sleep time for polling.
 		elapsedTime := time.Now().Sub(start).Milliseconds()
 		adjustTime = sampleInterval - elapsedTime
 		log.Debug().Msgf("Adjusted Poll Rate: %d ms", adjustTime)
@@ -191,6 +195,7 @@ func main() {
 	getK8sClient()
 	go getMetrics()
 	port := getEnv("METRICS_PORT", "9100")
+	adjustedPollingRate, _ = strconv.ParseBool(getEnv("ADJUSTED_POLLING_RATE", "false"))
 	http.Handle("/metrics", promhttp.Handler())
 	log.Info().Msg(fmt.Sprintf("Starting server listening on :%s", port))
 	err := http.ListenAndServe(fmt.Sprintf(":%s", port), nil)

scripts/deploy.sh

@@ -28,7 +28,8 @@ function main() {
   "image.imagePullPolicy=Never",
   "deploy_type=Deployment",
   "log_level=debug"
-  "dev.enabled=true"
+  "dev.enabled=true",
+  "extra.adjusted_polling_rate=true"
   )
 
   if [[ $ENV =~ "e2e"   ]]; then

tests/e2e/deployment_test.go

@@ -110,16 +110,16 @@ func checkPrometheus(checkSlice []string) {
 func WatchPollingRate(pollRateUpper float64, pollingRateLower float64, timeout time.Duration) {
 	status := 0
 	startTime := time.Now()
-	re := regexp.MustCompile(`ephemeral_storage_pod_usage\{adjusted_polling_rate="(\d+)`)
+	re := regexp.MustCompile(`ephemeral_storage_adjusted_polling_rate\s+(.+)`)
 	for {
 		elapsed := time.Since(startTime)
 		if elapsed >= timeout {
 			ginkgo.GinkgoWriter.Printf("Watch for rate polling timed out")
 			break
 		}
 		output := requestPrometheusString()
-		match := re.FindAllStringSubmatch(output, 2)
-		floatValue, _ := strconv.ParseFloat(match[0][1], 64)
+		match := re.FindAllStringSubmatch(output, -1)
+		floatValue, _ := strconv.ParseFloat(match[2][1], 64)
 		if pollRateUpper >= floatValue && pollingRateLower <= floatValue {
 			status = 1
 			break
@@ -178,7 +178,8 @@ var _ = ginkgo.Describe("Test Metrics\n", func() {
 	ginkgo.Context("Observe labels\n", func() {
 		ginkgo.Specify("\nReturn A Record IP addresses and Proxy IP address", func() {
 			var checkSlice []string
-			checkSlice = append(checkSlice, "ephemeral_storage_pod_usage", "pod_name=\"k8s-ephemeral-storage")
+			checkSlice = append(checkSlice, "ephemeral_storage_pod_usage",
+				"pod_name=\"k8s-ephemeral-storage", "ephemeral_storage_adjusted_polling_rate")
 			checkPrometheus(checkSlice)
 		})
 	})