fix: add autoscaling for tempo-distributed metrics-generator

This adds autoscaling via hpa and keda for tempo-distributed metrics-generator. Implementation is analog to the already existing autoscaling options for the compactor. Signed-off-by: Marius Svechla <m.svechla@gmail.com>
grafana · Nov 15, 2024 · 752b7dc · 752b7dc
1 parent d9107be
commit 752b7dc
Show file tree

Hide file tree

Showing 3 changed files with 103 additions and 0 deletions.
diff --git a/charts/tempo-distributed/templates/metrics-generator/hpa.yaml b/charts/tempo-distributed/templates/metrics-generator/hpa.yaml
@@ -0,0 +1,47 @@
+{{- if and .Values.metricsGenerator.autoscaling.enabled .Values.metricsGenerator.autoscaling.hpa.enabled }}
+{{- $apiVersion := include "tempo.hpa.apiVersion" . -}}
+{{ $dict := dict "ctx" . "component" "metrics-generator" }}
+apiVersion: {{ $apiVersion }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "tempo.resourceName" $dict }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "tempo.labels" $dict | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: {{ .Values.metricsGenerator.kind }}
+    name: {{ include "tempo.resourceName" $dict }}
+  minReplicas: {{ .Values.metricsGenerator.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.metricsGenerator.autoscaling.maxReplicas }}
+  {{- with .Values.metricsGenerator.autoscaling.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  metrics:
+  {{- with .Values.metricsGenerator.autoscaling.hpa.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if (eq $apiVersion "autoscaling/v2") }}
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+        {{- else }}
+        targetAverageUtilization: {{ . }}
+        {{- end }}
+  {{- end }}
+  {{- with .Values.metricsGenerator.autoscaling.hpa.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if (eq $apiVersion "autoscaling/v2") }}
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+        {{- else }}
+        targetAverageUtilization: {{ . }}
+        {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/charts/tempo-distributed/templates/metrics-generator/keda-scaled-object.yaml b/charts/tempo-distributed/templates/metrics-generator/keda-scaled-object.yaml
@@ -0,0 +1,26 @@
+{{- if and .Values.metricsGenerator.autoscaling.enabled .Values.metricsGenerator.autoscaling.keda.enabled }}
+{{ $dict := dict "ctx" . "component" "metrics-generator" }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{ include "tempo.resourceName" $dict }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "tempo.labels" $dict | nindent 4 }}
+spec:
+  minReplicaCount: {{ .Values.metricsGenerator.autoscaling.minReplicas }}
+  maxReplicaCount: {{ .Values.metricsGenerator.autoscaling.maxReplicas }}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "tempo.resourceName" $dict }}
+  triggers:
+  {{- range .Values.metricsGenerator.autoscaling.keda.triggers }}
+  - type: {{ .type | quote }}
+    metadata:
+      serverAddress: {{ .metadata.serverAddress }}
+      threshold: {{ .metadata.threshold | quote }}
+      query: |
+        {{- .metadata.query | nindent 8 }}
+  {{- end }}
+{{- end }}
diff --git a/charts/tempo-distributed/values.yaml b/charts/tempo-distributed/values.yaml
@@ -313,6 +313,36 @@ metricsGenerator:
     repository: null
     # -- Docker image tag for the metrics-generator image. Overrides `tempo.image.tag`
     tag: null
+  # -- Autoscaling configurations
+  autoscaling:
+    # -- Enable autoscaling for the metrics-generator
+    enabled: false
+    # -- Minimum autoscaling replicas for the metrics-generator
+    minReplicas: 1
+    # -- Maximum autoscaling replicas for the metrics-generator
+    maxReplicas: 3
+    # -- Autoscaling via HPA object
+    hpa:
+      enabled: false
+      # -- Autoscaling behavior configuration for the metrics-generator
+      behavior: {}
+      # -- Target CPU utilisation percentage for the metrics-generator
+      targetCPUUtilizationPercentage: 100
+      # -- Target memory utilisation percentage for the metrics-generator
+      targetMemoryUtilizationPercentage:
+    # -- Autoscaling via keda/ScaledObject
+    keda:
+      # requires https://keda.sh/
+      enabled: false
+      # -- List of autoscaling triggers for the metrics-generator
+      triggers: []
+      # - type: prometheus
+      #   metadata:
+      #     serverAddress: "http://<prometheus-host>:9090"
+      #     threshold: "250"
+      #     query: |-
+      #       sum(prometheus_remote_storage_shards_desired{job="default/metrics-generator"} /
+      #       prometheus_remote_storage_shards_max{job="default/metrics-generator"})by(job)
   # -- The name of the PriorityClass for metrics-generator pods
   priorityClassName: null
   # -- Labels for metrics-generator pods