Add metric relabeling (#76)

Co-authored-by: Mateusz Drab <mateuszd@outlook.com>

Makefile

@@ -30,6 +30,9 @@ helm-docs:
 	$(LOCALBIN)/helm-docs  --template-files "${GITROOT}/chart/README.md.gotmpl"
 	cat "${GITROOT}/Header.md" "${GITROOT}/chart/README.md" > "${GITROOT}/README.md"
 
+test-helm-render:
+	helm template ./chart -f ./chart/test-values.yaml 1> /dev/null
+
 minikube_new:
 	./scripts/create-minikube.sh
 
@@ -53,10 +56,13 @@ deploy_local: init
 deploy_observability:
 	ENV='observability' ./scripts/deploy.sh
 
-deploy_e2e: init ginkgo crane minikube_new
+deploy_test: init
+	ENV='test' ./scripts/deploy.sh
+
+deploy_e2e: init test-helm-render ginkgo crane minikube_new
 	ENV='e2e' ./scripts/deploy.sh
 
-deploy_e2e_dirty: init
+deploy_e2e_dirty: init test-helm-render
 	ENV='e2e' ./scripts/deploy.sh
 
 deploy_many_pods:
@@ -79,9 +85,20 @@ release-helm:
 	cd ..
 
 release: release-docker release-helm helm-docs
+	# Prod release
 	# ex. make VERSION=1.6.2 release
+	# Prerelease Candidate
+	# ex. make VERSION=1.6.3-rc01 release
+
+prerelease-github:
+	# Prerelease Candidate
+	# ex. make VERSION=1.6.3-rc01 prerelease-github
+	gh release create ${VERSION} --generate-notes --prerelease
+	gh release upload ${VERSION} "chart/k8s-ephemeral-storage-metrics-${VERSION}.tgz"
+	rm chart/k8s-ephemeral-storage-metrics-*.tgz
 
 release-github:
+	# Prod release
 	# ex. make VERSION=1.6.2 release-github
 	gh release create ${VERSION} --generate-notes
 	gh release upload ${VERSION} "chart/k8s-ephemeral-storage-metrics-${VERSION}.tgz"

chart/.helmignore

@@ -21,4 +21,5 @@
 *.tmproj
 .vscode/
 README.md.gotmpl
-*.tgz
+*.tgz
+test-values.yaml

chart/README.md.gotmpl

@@ -10,9 +10,9 @@ helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral
 
 ## Contribute
 
-### Start Kind
+### Start minikube
 ```bash
-make new_kind
+make new_minikube
 ```
 
 ### Run locally

chart/templates/metrics.yaml

@@ -11,13 +11,33 @@ spec:
       port: metrics
       scheme: http
       interval: "{{ .Values.interval }}s"
+{{- if .Values.serviceMonitor.metricRelabelings }}
+      metricRelabelings:
+{{ toYaml .Values.serviceMonitor.metricRelabelings | indent 8 }}
+{{- end }}
+{{- if .Values.serviceMonitor.relabelings }}
+      relabelings:
+{{ toYaml .Values.serviceMonitor.relabelings | indent 8 }}
+{{- end }}
   namespaceSelector:
     matchNames:
       - {{ .Release.Namespace }}
   selector:
     matchLabels:
       {{- include "chart.selectorLabels" . | nindent 6 }}
-
+  {{- if .Values.serviceMonitor.targetLabels }}
+  targetLabels:
+    {{- range .Values.serviceMonitor.targetLabels }}
+    - {{ . }}
+    {{- end }}
+  {{- end }}
+  {{- if .Values.serviceMonitor.podTargetLabels }}
+  podTargetLabels:
+    {{- range .Values.serviceMonitor.podTargetLabels }}
+    - {{ . }}
+    {{- end }}
+  {{- end }}
+
 ---
 
 apiVersion: v1

chart/test-values.yaml

@@ -0,0 +1,69 @@
+
+image:
+  repository: ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics
+  tag: 1.6.2
+  imagePullPolicy: IfNotPresent
+
+serviceMonitor:
+  relabelings:
+    - sourceLabels: [pod_name]
+      targetLabel: pod
+    - sourceLabels: [pod_namespace]
+      targetLabel: namespace
+    # drop labels pod_.*
+    - regex: "pod_(.*)"
+      action: labeldrop
+
+
+# -- Set metrics you want to enable
+metrics:
+  # -- Percentage of ephemeral storage used by a container in a pod
+  ephemeral_storage_container_limit_percentage: true
+  # -- Percentage of ephemeral storage used by a container's volume in a pod
+  ephemeral_storage_container_volume_limit_percentage: true
+  # -- Current ephemeral byte usage of pod
+  ephemeral_storage_pod_usage: true
+  # -- Available ephemeral storage for a node
+  ephemeral_storage_node_available: true
+  # -- Capacity of ephemeral storage for a node
+  ephemeral_storage_node_capacity: true
+  # -- Percentage of ephemeral storage used on a node
+  ephemeral_storage_node_percentage: true
+  # -- Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing.
+  adjusted_polling_rate: false
+
+log_level: info
+# -- Set as Deployment for single controller to query all nodes or Daemonset
+deploy_type: Deployment
+# Note in testing, Kube API does not refresh faster than 10 seconds
+# -- Polling node rate for exporter
+interval: 15 # Seconds
+# -- Max number of concurrent query requests to the kubernetes API.
+max_node_concurrency: 10
+prometheus:
+  enable: true
+  release: kube-prometheus-stack
+
+# -- Enable Pprof
+pprof: false
+
+
+# -- For local development or testing that will deploy grow and shrink pods and debug service
+dev:
+  enabled: false
+  shrink:
+    image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest
+    imagePullPolicy: IfNotPresent
+  grow:
+    image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest
+    imagePullPolicy: IfNotPresent
+
+podAnnotations: {}
+
+nodeSelector: {}
+
+affinity: {}
+
+tolerations: []
+
+

chart/values.yaml

@@ -3,6 +3,17 @@ image:
   tag: 1.6.2
   imagePullPolicy: IfNotPresent
 
+serviceMonitor:
+   # Set relabelings as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig
+   relabelings: []
+  # Set metricRelabelings as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig
+   metricRelabelings: []
+   # Set targetLabels as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitorSpec
+   targetLabels: []
+   # Set podTargetLabels as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitorSpec
+   podTargetLabels: []
+
+
 # -- Set metrics you want to enable
 metrics:
   # -- Percentage of ephemeral storage used by a container in a pod

scripts/deploy.sh

@@ -112,10 +112,18 @@ function main() {
 
   common_set_values=$(echo ${common_set_values_arr[*]} | sed 's/, /,/g' | sed 's/ /,/g')
 
-  helm upgrade --install "${DEPLOYMENT_NAME}" ../chart \
-    --set "${common_set_values}" \
-    --create-namespace \
-    --namespace "${DEPLOYMENT_NAME}"
+  if [[ $ENV == "test" ]]; then
+    helm upgrade --install "${DEPLOYMENT_NAME}" ../chart \
+      -f ../chart/test-values.yaml \
+      --create-namespace \
+      --namespace "${DEPLOYMENT_NAME}"
+  else
+    # All other deployments
+    helm upgrade --install "${DEPLOYMENT_NAME}" ../chart \
+      --set "${common_set_values}" \
+      --create-namespace \
+      --namespace "${DEPLOYMENT_NAME}"
+  fi
 
   # Patch deploy so minikube image upload works.
   if [[ $ENV == "debug" ]]; then

scripts/release-docker.sh

@@ -18,14 +18,18 @@ gh auth token | docker login ghcr.io --username jmcgrath207 --password-stdin
 docker build -f Dockerfile -t ghcr.io/jmcgrath207/$package:$VERSION .
 docker build -f Dockerfile -t ghcr.io/jmcgrath207/$package:latest .
 docker push ghcr.io/jmcgrath207/$package:$VERSION
-docker push ghcr.io/jmcgrath207/$package:latest
 
 docker build -f DockerfileTestGrow -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest .
 docker build -f DockerfileTestGrow -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:$VERSION .
 docker push ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:$VERSION
-docker push ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest
 
 docker build -f DockerfileTestGrow -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest .
 docker build -f DockerfileTestGrow -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:$VERSION .
 docker push ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:$VERSION
-docker push ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest
+
+# Don't push the latest image tags if they are a release candidate
+if ! [[ $ENV =~ "rc" ]]; then
+  docker push ghcr.io/jmcgrath207/$package:latest
+  docker push ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest
+  docker push ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest
+fi