Merge pull request #78 from statisticsnorway/add-alerts

Add inital alerts and deploy to NAIS

.github/workflows/alert-deploy.yml

@@ -0,0 +1,37 @@
+name: Deploy alerts
+run-name: Deploy alerts for Fyllut to dev and prod
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.nais/alerts.yaml'
+      - '.github/workflows/alert-deploy.yml'
+permissions:
+  id-token: write
+
+jobs:
+  test-deploy:
+    name: Deploy alerts to test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Deploy to test
+        uses: nais/deploy/actions/deploy@v2
+        env:
+          CLUSTER: test
+          RESOURCE: .nais/alerts.yaml
+
+  prod-deploy:
+    name: Deploy alerts to prod
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Deploy to prod
+        uses: nais/deploy/actions/deploy@v2
+        env:
+          CLUSTER: prod
+          RESOURCE: .nais/alerts.yaml

.nais/alerts.yaml

@@ -0,0 +1,33 @@
+apiVersion: "monitoring.coreos.com/v1"
+kind: PrometheusRule
+metadata:
+  name: alert-maskinporten-guardian
+  namespace: dapla-stat
+  labels:
+    team: dapla-stat
+spec:
+  groups:
+    - name: dapla-stat
+      rules:
+        - alert: High number of errors
+          expr: (100 * sum by (app, namespace) (rate(log_messages_errors{app="maskinporten-guardian",level=~"Error"}[3m])) / sum by (app, namespace) (rate(log_messages_total{app="skjemautfylling"}[3m]))) > 10
+          for: 3m
+          annotations:
+            title: "High number of errors logged"
+            consequence: "There can be different causes for errors, check logs for cause and evaluation of consequences."
+            action: "`kubectl describe pod -l app=maskinporten-guardian -n dapla-stat` -> `kubectl logs <podname>`"
+          labels:
+            service: maskinporten-guardian
+            namespace: dapla-stat
+            severity: critical
+
+        - alert: Guardian is unavailable
+          expr: kube_deployment_status_replicas_available{deployment="maskinporten-guardian"} == 0
+          for: 1m
+          annotations:
+            title: "Maskipnporten Guardian is unavailable"
+            consequence: "Service is unavailable to users. "
+          labels:
+            service: maskinporten-guardian
+            namespace: dapla-stat
+            severity: critical