Switch to sli_minio image

vshn · Nov 2, 2023 · 8f500a6 · 8f500a6
1 parent ed76643
commit 8f500a6
Show file tree

Hide file tree

Showing 37 changed files with 1,137 additions and 51 deletions.
diff --git a/component/class/defaults.yml b/component/class/defaults.yml
@@ -39,7 +39,7 @@ parameters:
       appcat:
         registry: ghcr.io
         repository: vshn/appcat
-        tag: v4.37.0
+        tag: sli_minio
       apiserver:
         registry: ghcr.io
         repository: vshn/appcat-apiserver
@@ -168,6 +168,16 @@ parameters:
                 # If the alert is pending for more than 5m this indicates a real problem.
                 for: 6m
               ticket_alert: {}
+        minio:
+          uptime:
+            objective: 99.9
+            alerting:
+              page_alert:
+                # This should reduce non actionable alerts because of single instance restarts.
+                # The page alert looks (ammong other things) at the burn rate over the last 5min.
+                # If the alert is pending for more than 5m this indicates a real problem.
+                for: 6m
+              ticket_alert: {}
 
     providers:
       cloudscale:
@@ -420,6 +430,7 @@ parameters:
           helmChartVersion: ${appcat:charts:minio:version}
           grpcEndpoint: ${appcat:grpcEndpoint}
           defaultPlan: standard-1
+          sla: 99.25
           plans:
             standard-1:
               size:

diff --git a/component/component/slos.libsonnet b/component/component/slos.libsonnet
@@ -58,17 +58,25 @@ local prometheusRule(name) =
     spec: patchedRules,
   };
 
+local getEvents(serviceName) = {
+  // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
+  error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="' + serviceName + '", ha="false"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="' + serviceName + '"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0 or sum(0*rate(appcat_probes_seconds_count{service="' + serviceName + '"}[{{.window}}])) by (service, namespace, name, organization, sla)',
+  total_query: 'sum(rate(appcat_probes_seconds_count{service="' + serviceName + '", ha="false"}[{{.window}}])) by (service, namespace, name, organization, sla)',
+};
+
+local getEventsHA(serviceName) = {
+  // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
+  error_query: 'sum(rate(appcat_probes_seconds_count{reason!="success", service="' + serviceName + '", ha="true"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="' + serviceName + '"}[{{.window}}])) by (service, namespace, name, organization, sla) or sum(0*rate(appcat_probes_seconds_count{service="' + serviceName + '"}[{{.window}}])) by (service, namespace, name, organization, sla)',
+  total_query: 'sum(rate(appcat_probes_seconds_count{service="' + serviceName + '", ha="true"}[{{.window}}])) by (service, namespace, name, organization, sla)',
+};
+
 {
   slothInput: {
     'vshn-postgresql': [
       newSLO('uptime', 'vshn-postgresql', params.slos.vshn.postgres.uptime) {
         description: 'Uptime SLO for PostgreSQL by VSHN',
         sli: {
-          events: {
-            // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
-            error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNPostgreSQL", ha="false"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNPostgreSQL"}[{{.window}}])) by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0 or sum(0*rate(appcat_probes_seconds_count{service="VSHNPostgreSQL"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-            total_query: 'sum(rate(appcat_probes_seconds_count{service="VSHNPostgreSQL", ha="false"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-          },
+          events: getEvents('VSHNPostgreSQL'),
         },
         alerting+: {
           name: 'SLO_AppCat_VSHNPostgreSQLUptime',
@@ -86,11 +94,7 @@ local prometheusRule(name) =
       newSLO('uptime', 'vshn-postgresql-ha', params.slos.vshn.postgres.uptime) {
         description: 'Uptime SLO for High Available PostgreSQL by VSHN',
         sli: {
-          events: {
-            // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
-            error_query: 'sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNPostgreSQL", ha="true"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNPostgreSQL"}[{{.window}}])) by (service, namespace, name, organization, sla) or sum(0*rate(appcat_probes_seconds_count{service="VSHNPostgreSQL"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-            total_query: 'sum(rate(appcat_probes_seconds_count{service="VSHNPostgreSQL", ha="true"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-          },
+          events: getEventsHA('VSHNPostgreSQL'),
         },
         alerting+: {
           name: 'SLO_AppCat_HAVSHNPosgtreSQLUptime',
@@ -109,11 +113,7 @@ local prometheusRule(name) =
       newSLO('uptime', 'vshn-redis', params.slos.vshn.redis.uptime) {
         description: 'Uptime SLO for Redis by VSHN',
         sli: {
-          events: {
-            // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
-            error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNRedis", ha="false"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}]))  by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0 or sum(0*rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-            total_query: 'sum(rate(appcat_probes_seconds_count{service="VSHNRedis", ha="false"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-          },
+          events: getEvents('VSHNRedis'),
         },
         alerting+: {
           name: 'SLO_AppCat_VSHNRedisUptime',
@@ -131,11 +131,7 @@ local prometheusRule(name) =
       newSLO('uptime', 'vshn-redis-ha', params.slos.vshn.redis.uptime) {
         description: 'Uptime SLO for High Available Redis by VSHN',
         sli: {
-          events: {
-            // The  0*rate(...) makes sure that the query reports an error rate for all instances, even if that instance has never produced a single error
-            error_query: '(sum(rate(appcat_probes_seconds_count{reason!="success", service="VSHNRedis", ha="true"}[{{.window}}]) or 0*rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}]))  by (service, namespace, name, organization, sla) or vector(0)) - scalar(appcat:cluster:maintenance) > 0 or sum(0*rate(appcat_probes_seconds_count{service="VSHNRedis"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-            total_query: 'sum(rate(appcat_probes_seconds_count{service="VSHNRedis", ha="true"}[{{.window}}])) by (service, namespace, name, organization, sla)',
-          },
+          events: getEventsHA('VSHNRedis'),
         },
         alerting+: {
           name: 'SLO_AppCat_HAVSHNRedisUptime',
@@ -149,6 +145,42 @@ local prometheusRule(name) =
         },
       },
     ],
+    'vshn-minio': [
+      newSLO('uptime', 'vshn-minio', params.slos.vshn.minio.uptime) {
+        description: 'Uptime SLO for Minio by VSHN',
+        sli: {
+          events: getEvents('VSHNMinio'),
+        },
+        alerting+: {
+          name: 'SLO_AppCat_VSHNMinioUptime',
+          annotations+: {
+            summary: 'Probes to Minio by VSHN instance fail',
+          },
+          labels+: {
+            service: 'VSHNMinio',
+            OnCall: '{{ if eq $labels.sla "guaranteed" }}true{{ else }}false{{ end }}',
+          },
+        },
+      },
+    ],
+    'vshn-minio-ha': [
+      newSLO('uptime', 'vshn-postgresql-ha', params.slos.vshn.minio.uptime) {
+        description: 'Uptime SLO for High Available Minio by VSHN',
+        sli: {
+          events: getEventsHA('VSHNMinio'),
+        },
+        alerting+: {
+          name: 'SLO_AppCat_HAVSHNMinioUptime',
+          annotations+: {
+            summary: 'Probes to HA Minio by VSHN instance fail',
+          },
+          labels+: {
+            service: 'VSHNMinio',
+            OnCall: '{{ if eq $labels.sla "guaranteed" }}true{{ else }}false{{ end }}',
+          },
+        },
+      },
+    ],
   },
   Get(name): prometheusRule(name),
 }
diff --git a/component/component/vshn_minio.jsonnet b/component/component/vshn_minio.jsonnet
@@ -8,6 +8,8 @@ local crossplane = import 'lib/crossplane.libsonnet';
 local common = import 'common.libsonnet';
 local xrds = import 'xrds.libsonnet';
 
+local slos = import 'slos.libsonnet';
+
 local inv = kap.inventory();
 local params = inv.parameters.appcat;
 local minioParams = params.services.vshn.minio;
@@ -21,6 +23,8 @@ local connectionSecretKeys = [
   'AWS_ACCESS_KEY_ID',
 ];
 
+local promRuleMinioSLA = common.PromRuleSLA(params.services.vshn.minio.sla, 'VSHNMinio');
+
 local minioPlans = common.FilterDisabledParams(minioParams.plans);
 
 local xrd = xrds.XRDFromCRD(
@@ -97,5 +101,8 @@ if params.services.vshn.enabled && minioParams.enabled then {
   '20_xrd_vshn_minio': xrd,
   '20_rbac_vshn_minio': xrds.CompositeClusterRoles(xrd),
   '21_composition_vshn_minio': composition,
+  '22_prom_rule_sla_minio': promRuleMinioSLA,
   [if std.length(instances) != 0 then '22_minio_instances']: instances,
+  [if params.services.vshn.enabled && params.services.vshn.minio.enabled then 'sli_exporter/90_slo_vshn_minio']: slos.Get('vshn-minio'),
+  [if params.services.vshn.enabled && params.services.vshn.minio.enabled then 'sli_exporter/90_slo_vshn_minio_ha']: slos.Get('vshn-minio-ha'),
 } else {}
diff --git a/component/component/vshn_postgres.jsonnet b/component/component/vshn_postgres.jsonnet
@@ -9,6 +9,8 @@ local crossplane = import 'lib/crossplane.libsonnet';
 local common = import 'common.libsonnet';
 local xrds = import 'xrds.libsonnet';
 
+local slos = import 'slos.libsonnet';
+
 local inv = kap.inventory();
 local params = inv.parameters.appcat;
 local pgParams = params.services.vshn.postgres;
@@ -1101,4 +1103,6 @@ if params.services.vshn.enabled && pgParams.enabled then
     '21_composition_vshn_postgresrestore': restoreComp,
     '22_prom_rule_sla_postgres': promRulePostgresSLA,
     [if isOpenshift then '21_openshift_template_postgresql_vshn']: osTemplate,
+    [if params.slos.enabled && params.services.vshn.enabled && params.services.vshn.postgres.enabled then 'sli_exporter/90_slo_vshn_postgresql']: slos.Get('vshn-postgresql'),
+    [if params.slos.enabled && params.services.vshn.enabled && params.services.vshn.postgres.enabled then 'sli_exporter/90_slo_vshn_postgresql_ha']: slos.Get('vshn-postgresql-ha'),
   } else {}
diff --git a/component/component/vshn_redis.jsonnet b/component/component/vshn_redis.jsonnet
@@ -8,6 +8,8 @@ local crossplane = import 'lib/crossplane.libsonnet';
 local common = import 'common.libsonnet';
 local xrds = import 'xrds.libsonnet';
 
+local slos = import 'slos.libsonnet';
+
 local inv = kap.inventory();
 local params = inv.parameters.appcat;
 local redisParams = params.services.vshn.redis;
@@ -737,4 +739,6 @@ if params.services.vshn.enabled && redisParams.enabled then {
   '21_composition_vshn_redis': composition,
   '22_prom_rule_sla_redis': promRuleRedisSLA,
   [if isOpenshift then '21_openshift_template_redis_vshn']: osTemplate,
+  [if params.services.vshn.enabled && params.services.vshn.redis.enabled then 'sli_exporter/90_slo_vshn_redis']: slos.Get('vshn-redis'),
+  [if params.services.vshn.enabled && params.services.vshn.redis.enabled then 'sli_exporter/90_slo_vshn_redis_ha']: slos.Get('vshn-redis-ha'),
 } else {}
diff --git a/component/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml b/component/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml
@@ -272,6 +272,18 @@ rules:
       - patch
       - create
       - delete
+  - apiGroups:
+      - appcat.vshn.io
+    resources:
+      - objectbuckets
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+      - create
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/component/tests/golden/controllers/appcat/appcat/controllers/appcat/30_deployment.yaml b/component/tests/golden/controllers/appcat/appcat/controllers/appcat/30_deployment.yaml
@@ -23,7 +23,7 @@ spec:
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
-          image: ghcr.io/vshn/appcat:v4.37.0
+          image: ghcr.io/vshn/appcat:sli_minio
           livenessProbe:
             httpGet:
               path: /healthz

diff --git a/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml b/.../appcat/appcat/sli_exporter/apps_v1_deployment_appcat-sliexporter-controller-manager.yaml
@@ -28,7 +28,7 @@ spec:
           value: "false"
         - name: APPCAT_SLI_VSHNREDIS
           value: "false"
-        image: ghcr.io/vshn/appcat:v4.37.0
+        image: ghcr.io/vshn/appcat:sli_minio
         livenessProbe:
           httpGet:
             path: /healthz

diff --git a/...rter/rbac.authorization.k8s.io_v1_clusterrole_appcat-sliexporter-appcat-sli-exporter.yaml b/...rter/rbac.authorization.k8s.io_v1_clusterrole_appcat-sliexporter-appcat-sli-exporter.yaml
@@ -19,6 +19,20 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - vshn.appcat.vshn.io
+  resources:
+  - vshnminios
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - vshn.appcat.vshn.io
+  resources:
+  - vshnminios/status
+  verbs:
+  - get
 - apiGroups:
   - vshn.appcat.vshn.io
   resources:
@@ -47,6 +61,20 @@ rules:
   - vshnredis/status
   verbs:
   - get
+- apiGroups:
+  - vshn.appcat.vshn.io
+  resources:
+  - xvshnminios
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - vshn.appcat.vshn.io
+  resources:
+  - xvshnminios/status
+  verbs:
+  - get
 - apiGroups:
   - vshn.appcat.vshn.io
   resources:

diff --git a/component/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml b/component/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml
@@ -272,6 +272,18 @@ rules:
       - patch
       - create
       - delete
+  - apiGroups:
+      - appcat.vshn.io
+    resources:
+      - objectbuckets
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+      - create
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/component/tests/golden/minio/appcat/appcat/10_provider_kubernetes.yaml b/component/tests/golden/minio/appcat/appcat/10_provider_kubernetes.yaml
@@ -272,6 +272,18 @@ rules:
       - patch
       - create
       - delete
+  - apiGroups:
+      - appcat.vshn.io
+    resources:
+      - objectbuckets
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+      - create
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/component/tests/golden/minio/appcat/appcat/21_composition_vshn_minio.yaml b/component/tests/golden/minio/appcat/appcat/21_composition_vshn_minio.yaml
@@ -25,7 +25,7 @@ spec:
         data:
           controlNamespace: syn-appcat-control
           defaultPlan: standard-1
-          imageTag: v4.37.0
+          imageTag: sli_minio
           maintenanceSA: helm-based-service-maintenance
           minioChartRepository: https://charts.min.io
           minioChartVersion: 5.0.13

diff --git a/component/tests/golden/minio/appcat/appcat/22_prom_rule_sla_minio.yaml b/component/tests/golden/minio/appcat/appcat/22_prom_rule_sla_minio.yaml
@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: vshn-vshnminio-sla
+  name: vshn-vshnminio-sla
+  namespace: appcat-slos
+spec:
+  groups:
+    - name: appcat-vshnminio-sla-target
+      rules:
+        - expr: vector(99.25)
+          labels:
+            service: VSHNMinio
+          record: sla:objective:ratio
diff --git a/component/tests/golden/minio/appcat/appcat/controllers/appcat/30_deployment.yaml b/component/tests/golden/minio/appcat/appcat/controllers/appcat/30_deployment.yaml
@@ -23,7 +23,7 @@ spec:
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
-          image: ghcr.io/vshn/appcat:v4.37.0
+          image: ghcr.io/vshn/appcat:sli_minio
           livenessProbe:
             httpGet:
               path: /healthz

diff --git a/component/tests/golden/minio/appcat/appcat/sla_reporter/01_cronjob.yaml b/component/tests/golden/minio/appcat/appcat/sla_reporter/01_cronjob.yaml
@@ -30,7 +30,7 @@ spec:
               envFrom:
                 - secretRef:
                     name: appcat-sla-reports-creds
-              image: ghcr.io/vshn/appcat:v4.37.0
+              image: ghcr.io/vshn/appcat:sli_minio
               name: sla-reporter
               resources:
                 limits: