From 6dc2f72531a0275ddb27ee762165770e8c5a6a7c Mon Sep 17 00:00:00 2001 From: John McGrath <8764013+jmcgrath207@users.noreply.github.com> Date: Thu, 25 Apr 2024 23:10:05 -0500 Subject: [PATCH] added test for ephemeral_storage_container_volume_usage and update deps (#88) --- Makefile | 4 ++-- README.md | 5 +++-- chart/Chart.yaml | 4 ++-- chart/README.md | 5 +++-- chart/index.yaml | 24 +++++++++++++++++++++++- chart/templates/DeployType.yaml | 4 ++++ chart/test-values.yaml | 2 ++ chart/values.yaml | 4 +++- go.mod | 2 +- go.sum | 2 ++ tests/e2e/deployment_test.go | 22 +++++++++++++++++++++- 11 files changed, 66 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 6c51025..3c0db84 100644 --- a/Makefile +++ b/Makefile @@ -86,13 +86,13 @@ release-helm: release: release-docker release-helm helm-docs # Prod release - # ex. make VERSION=1.8.0 release + # ex. make VERSION=1.9.0 release # Prerelease Candidate # ex. make VERSION=1.6.3-rc01 release release-github: # Prod release - # ex. make VERSION=1.8.0 release-github + # ex. make VERSION=1.9.0 release-github gh release create ${VERSION} --generate-notes gh release upload ${VERSION} "chart/k8s-ephemeral-storage-metrics-${VERSION}.tgz" rm chart/k8s-ephemeral-storage-metrics-*.tgz diff --git a/README.md b/README.md index 44887d5..f41311e 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,15 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral | dev | object | `{"enabled":false,"grow":{"image":"ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest","imagePullPolicy":"IfNotPresent"},"shrink":{"image":"ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest","imagePullPolicy":"IfNotPresent"}}` | For local development or testing that will deploy grow and shrink pods and debug service | | image.imagePullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics"` | | -| image.tag | string | `"1.8.0"` | | +| image.tag | string | `"1.9.0"` | | | interval | int | `15` | Polling node rate for exporter | | log_level | string | `"info"` | | | max_node_concurrency | int | `10` | Max number of concurrent query requests to the kubernetes API. | -| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_container_volume_limit_percentage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | +| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_container_volume_limit_percentage":true,"ephemeral_storage_container_volume_usage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | | metrics.adjusted_polling_rate | bool | `false` | Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing. | | metrics.ephemeral_storage_container_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container in a pod | | metrics.ephemeral_storage_container_volume_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container's volume in a pod | +| metrics.ephemeral_storage_container_volume_usage | bool | `true` | Current ephemeral storage used by a container's volume in a pod | | metrics.ephemeral_storage_node_available | bool | `true` | Available ephemeral storage for a node | | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node | | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node | diff --git a/chart/Chart.yaml b/chart/Chart.yaml index cfee169..cd63267 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: k8s-ephemeral-storage-metrics -version: 1.8.0 -appVersion: 1.8.0 +version: 1.9.0 +appVersion: 1.9.0 kubeVersion: ">=1.21.0-0" description: Ephemeral storage metrics for prometheus operator. home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics diff --git a/chart/README.md b/chart/README.md index 9a91ccf..3e630aa 100644 --- a/chart/README.md +++ b/chart/README.md @@ -15,14 +15,15 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral | dev | object | `{"enabled":false,"grow":{"image":"ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest","imagePullPolicy":"IfNotPresent"},"shrink":{"image":"ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest","imagePullPolicy":"IfNotPresent"}}` | For local development or testing that will deploy grow and shrink pods and debug service | | image.imagePullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics"` | | -| image.tag | string | `"1.8.0"` | | +| image.tag | string | `"1.9.0"` | | | interval | int | `15` | Polling node rate for exporter | | log_level | string | `"info"` | | | max_node_concurrency | int | `10` | Max number of concurrent query requests to the kubernetes API. | -| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_container_volume_limit_percentage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | +| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_container_volume_limit_percentage":true,"ephemeral_storage_container_volume_usage":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true}` | Set metrics you want to enable | | metrics.adjusted_polling_rate | bool | `false` | Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing. | | metrics.ephemeral_storage_container_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container in a pod | | metrics.ephemeral_storage_container_volume_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container's volume in a pod | +| metrics.ephemeral_storage_container_volume_usage | bool | `true` | Current ephemeral storage used by a container's volume in a pod | | metrics.ephemeral_storage_node_available | bool | `true` | Available ephemeral storage for a node | | metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node | | metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node | diff --git a/chart/index.yaml b/chart/index.yaml index bc371ab..36425da 100644 --- a/chart/index.yaml +++ b/chart/index.yaml @@ -1,6 +1,28 @@ apiVersion: v1 entries: k8s-ephemeral-storage-metrics: + - annotations: + artifacthub.io/license: MIT + artifacthub.io/links: | + - name: Documentation + url: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics + artifacthub.io/prerelease: "false" + apiVersion: v2 + appVersion: 1.9.0 + created: "2024-04-25T23:04:34.999485611-05:00" + description: Ephemeral storage metrics for prometheus operator. + digest: bc2d126156fb3410d86ead11bc57632bf73082c04994d3e14db0b8f81cc13c98 + home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics + keywords: + - kubernetes + - metrics + kubeVersion: '>=1.21.0-0' + name: k8s-ephemeral-storage-metrics + sources: + - https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics + urls: + - https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/releases/download/1.9.0/k8s-ephemeral-storage-metrics-1.9.0.tgz + version: 1.9.0 - annotations: artifacthub.io/license: MIT artifacthub.io/links: | @@ -529,4 +551,4 @@ entries: urls: - https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics/releases/download/1.0.0/k8s-ephemeral-storage-metrics-1.0.0.tgz version: 1.0.0 -generated: "2024-04-20T17:57:20.422764699-05:00" +generated: "2024-04-25T23:04:34.999072963-05:00" diff --git a/chart/templates/DeployType.yaml b/chart/templates/DeployType.yaml index 06443d7..06ba294 100644 --- a/chart/templates/DeployType.yaml +++ b/chart/templates/DeployType.yaml @@ -98,6 +98,10 @@ spec: - name: EPHEMERAL_STORAGE_CONTAINER_LIMIT_PERCENTAGE value: "{{ .Values.metrics.ephemeral_storage_container_limit_percentage }}" {{- end }} + {{- if .Values.metrics.ephemeral_storage_container_volume_usage }} + - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_USAGE + value: "{{ .Values.metrics.ephemeral_storage_container_volume_usage }}" + {{- end }} {{- if .Values.metrics.ephemeral_storage_container_volume_limit_percentage }} - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE value: "{{ .Values.metrics.ephemeral_storage_container_volume_limit_percentage }}" diff --git a/chart/test-values.yaml b/chart/test-values.yaml index 7e26a6c..220f91f 100644 --- a/chart/test-values.yaml +++ b/chart/test-values.yaml @@ -21,6 +21,8 @@ serviceMonitor: metrics: # -- Percentage of ephemeral storage used by a container in a pod ephemeral_storage_container_limit_percentage: true + # -- Current ephemeral storage used by a container's volume in a pod + ephemeral_storage_container_volume_usage: true # -- Percentage of ephemeral storage used by a container's volume in a pod ephemeral_storage_container_volume_limit_percentage: true # -- Current ephemeral byte usage of pod diff --git a/chart/values.yaml b/chart/values.yaml index f451774..fa811c1 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics - tag: 1.8.0 + tag: 1.9.0 imagePullPolicy: IfNotPresent # -- Configure the Service Monitor @@ -21,6 +21,8 @@ serviceMonitor: metrics: # -- Percentage of ephemeral storage used by a container in a pod ephemeral_storage_container_limit_percentage: true + # -- Current ephemeral storage used by a container's volume in a pod + ephemeral_storage_container_volume_usage: true # -- Percentage of ephemeral storage used by a container's volume in a pod ephemeral_storage_container_volume_limit_percentage: true # -- Current ephemeral byte usage of pod diff --git a/go.mod b/go.mod index 6f8c1bf..919947a 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/rs/zerolog v1.32.0 k8s.io/api v0.30.0 k8s.io/apimachinery v0.30.0 - k8s.io/client-go v0.29.3 + k8s.io/client-go v0.30.0 ) diff --git a/go.sum b/go.sum index d6141ad..bc1a616 100644 --- a/go.sum +++ b/go.sum @@ -178,6 +178,8 @@ k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= +k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/tests/e2e/deployment_test.go b/tests/e2e/deployment_test.go index 4e7ecfa..956f968 100644 --- a/tests/e2e/deployment_test.go +++ b/tests/e2e/deployment_test.go @@ -220,6 +220,16 @@ func getContainerVolumeLimitPercentage(podName string) float64 { return currentPodSize } +func getContainerVolumeUsage(podName string) float64 { + output := requestPrometheusString() + re := regexp.MustCompile( + fmt.Sprintf(`ephemeral_storage_container_volume_usage.+container="%s",mount_path="\/cache".+\}\s(.+)`, + podName)) + match := re.FindAllStringSubmatch(output, 2) + currentPodSize, _ := strconv.ParseFloat(match[0][1], 64) + return currentPodSize +} + func WatchEphemeralSize(podName string, desiredSizeChange float64, timeout time.Duration, getPodSize getPodSize) { // Watch Prometheus Metrics until the ephemeral storage shrinks or grows to a certain desiredSizeChange. var currentPodSize float64 @@ -291,7 +301,9 @@ var _ = ginkgo.Describe("Test Metrics\n", func() { "ephemeral_storage_node_percentage", "pod_name=\"k8s-ephemeral-storage", "ephemeral_storage_adjusted_polling_rate", "node_name=\"minikube", - "ephemeral_storage_container_limit_percentage") + "ephemeral_storage_container_limit_percentage", + "ephemeral_storage_container_volume_limit_percentage", + "ephemeral_storage_container_volume_usage") checkPrometheus(checkSlice, false) }) }) @@ -336,6 +348,14 @@ var _ = ginkgo.Describe("Test Metrics\n", func() { WatchEphemeralSize("shrink-test", 0.2, time.Second*180, getContainerVolumeLimitPercentage) }) }) + ginkgo.Context("Observe change in ephemeral_storage_container_volume_usage metric\n", func() { + ginkgo.Specify("\nWatch Pod grow to 0.2 percent", func() { + WatchEphemeralSize("grow-test", 100000, time.Second*180, getContainerVolumeUsage) + }) + ginkgo.Specify("\nWatch Pod shrink to 0.2 percent", func() { + WatchEphemeralSize("shrink-test", 100000, time.Second*180, getContainerVolumeUsage) + }) + }) ginkgo.Context("\nMake sure percentage is not over 100", func() { ginkgo.Specify("\nTest ephemeral_storage_node_percentage", func() { WatchNodePercentage()