From 00f1006905afc72cec3b45307f12bb2f44b52a46 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Tue, 13 Aug 2024 10:14:04 +0000
Subject: [PATCH] Update upstream alerts

---
 .../master/lokistack_prometheus_alerts.yaml   | 18 ++++++++++++++++
 .../lokistack_prometheus_alerts.yaml          | 18 ++++++++++++++++
 .../lokistack_prometheus_alerts.yaml          | 18 ++++++++++++++++
 .../lokistack_prometheus_alerts.yaml          | 18 ++++++++++++++++
 .../60_lokistack_alerts.yaml                  | 21 +++++++++++++++++++
 .../60_lokistack_alerts.yaml                  | 21 +++++++++++++++++++
 .../60_lokistack_alerts.yaml                  | 21 +++++++++++++++++++
 .../60_lokistack_alerts.yaml                  | 21 +++++++++++++++++++
 8 files changed, 156 insertions(+)

diff --git a/component/extracted_alerts/master/lokistack_prometheus_alerts.yaml b/component/extracted_alerts/master/lokistack_prometheus_alerts.yaml
index 15cc424..799c280 100644
--- a/component/extracted_alerts/master/lokistack_prometheus_alerts.yaml
+++ b/component/extracted_alerts/master/lokistack_prometheus_alerts.yaml
@@ -175,6 +175,24 @@ groups:
     for: 15m
     labels:
       severity: warning
+  - alert: LokiDiscardedSamplesWarning
+    annotations:
+      message: |-
+        Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+        Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+      summary: Loki is discarding samples during ingestion because they fail validation.
+      runbook_url: "[[ .RunbookURL]]#Loki-Discarded-Samples-Warning"
+    expr: |
+      sum by(namespace, tenant, reason) (
+        irate(loki_discarded_samples_total{
+          reason!="rate_limited",
+          reason!="per_stream_rate_limit",
+          reason!="stream_limit"}[2m])
+      )
+      > 0
+    for: 15m
+    labels:
+      severity: warning
   - alert: LokistackSchemaUpgradesRequired
     annotations:
       message: |-
diff --git a/component/extracted_alerts/release-5.6/lokistack_prometheus_alerts.yaml b/component/extracted_alerts/release-5.6/lokistack_prometheus_alerts.yaml
index f378c49..e0c49d6 100644
--- a/component/extracted_alerts/release-5.6/lokistack_prometheus_alerts.yaml
+++ b/component/extracted_alerts/release-5.6/lokistack_prometheus_alerts.yaml
@@ -175,3 +175,21 @@ groups:
     for: 15m
     labels:
       severity: warning
+  - alert: LokiDiscardedSamplesWarning
+    annotations:
+      message: |-
+        Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+        Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+      summary: Loki is discarding samples during ingestion because they fail validation.
+      runbook_url: "[[ .RunbookURL]]#Loki-Discarded-Samples-Warning"
+    expr: |
+      sum by(namespace, tenant, reason) (
+        irate(loki_discarded_samples_total{
+          reason!="rate_limited",
+          reason!="per_stream_rate_limit",
+          reason!="stream_limit"}[2m])
+      )
+      > 0
+    for: 15m
+    labels:
+      severity: warning
diff --git a/component/extracted_alerts/release-5.8/lokistack_prometheus_alerts.yaml b/component/extracted_alerts/release-5.8/lokistack_prometheus_alerts.yaml
index f378c49..e0c49d6 100644
--- a/component/extracted_alerts/release-5.8/lokistack_prometheus_alerts.yaml
+++ b/component/extracted_alerts/release-5.8/lokistack_prometheus_alerts.yaml
@@ -175,3 +175,21 @@ groups:
     for: 15m
     labels:
       severity: warning
+  - alert: LokiDiscardedSamplesWarning
+    annotations:
+      message: |-
+        Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+        Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+      summary: Loki is discarding samples during ingestion because they fail validation.
+      runbook_url: "[[ .RunbookURL]]#Loki-Discarded-Samples-Warning"
+    expr: |
+      sum by(namespace, tenant, reason) (
+        irate(loki_discarded_samples_total{
+          reason!="rate_limited",
+          reason!="per_stream_rate_limit",
+          reason!="stream_limit"}[2m])
+      )
+      > 0
+    for: 15m
+    labels:
+      severity: warning
diff --git a/component/extracted_alerts/release-5.9/lokistack_prometheus_alerts.yaml b/component/extracted_alerts/release-5.9/lokistack_prometheus_alerts.yaml
index 15cc424..799c280 100644
--- a/component/extracted_alerts/release-5.9/lokistack_prometheus_alerts.yaml
+++ b/component/extracted_alerts/release-5.9/lokistack_prometheus_alerts.yaml
@@ -175,6 +175,24 @@ groups:
     for: 15m
     labels:
       severity: warning
+  - alert: LokiDiscardedSamplesWarning
+    annotations:
+      message: |-
+        Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+        Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+      summary: Loki is discarding samples during ingestion because they fail validation.
+      runbook_url: "[[ .RunbookURL]]#Loki-Discarded-Samples-Warning"
+    expr: |
+      sum by(namespace, tenant, reason) (
+        irate(loki_discarded_samples_total{
+          reason!="rate_limited",
+          reason!="per_stream_rate_limit",
+          reason!="stream_limit"}[2m])
+      )
+      > 0
+    for: 15m
+    labels:
+      severity: warning
   - alert: LokistackSchemaUpgradesRequired
     annotations:
       message: |-
diff --git a/tests/golden/defaults/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml b/tests/golden/defaults/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
index 65a573e..4f6c7da 100644
--- a/tests/golden/defaults/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
+++ b/tests/golden/defaults/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
@@ -204,6 +204,27 @@ spec:
             severity: warning
             syn: 'true'
             syn_component: openshift4-logging
+        - alert: SYN_LokiDiscardedSamplesWarning
+          annotations:
+            message: |-
+              Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+              Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+            runbook_url: '[[ .RunbookURL]]#Loki-Discarded-Samples-Warning'
+            summary: Loki is discarding samples during ingestion because they fail
+              validation.
+          expr: |
+            sum by(namespace, tenant, reason) (
+              irate(loki_discarded_samples_total{
+                reason!="rate_limited",
+                reason!="per_stream_rate_limit",
+                reason!="stream_limit"}[2m])
+            )
+            > 0
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: openshift4-logging
         - alert: SYN_LokistackSchemaUpgradesRequired
           annotations:
             message: |-
diff --git a/tests/golden/legacy/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml b/tests/golden/legacy/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
index 65a573e..4f6c7da 100644
--- a/tests/golden/legacy/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
+++ b/tests/golden/legacy/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
@@ -204,6 +204,27 @@ spec:
             severity: warning
             syn: 'true'
             syn_component: openshift4-logging
+        - alert: SYN_LokiDiscardedSamplesWarning
+          annotations:
+            message: |-
+              Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+              Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+            runbook_url: '[[ .RunbookURL]]#Loki-Discarded-Samples-Warning'
+            summary: Loki is discarding samples during ingestion because they fail
+              validation.
+          expr: |
+            sum by(namespace, tenant, reason) (
+              irate(loki_discarded_samples_total{
+                reason!="rate_limited",
+                reason!="per_stream_rate_limit",
+                reason!="stream_limit"}[2m])
+            )
+            > 0
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: openshift4-logging
         - alert: SYN_LokistackSchemaUpgradesRequired
           annotations:
             message: |-
diff --git a/tests/golden/master/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml b/tests/golden/master/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
index 65a573e..4f6c7da 100644
--- a/tests/golden/master/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
+++ b/tests/golden/master/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
@@ -204,6 +204,27 @@ spec:
             severity: warning
             syn: 'true'
             syn_component: openshift4-logging
+        - alert: SYN_LokiDiscardedSamplesWarning
+          annotations:
+            message: |-
+              Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+              Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+            runbook_url: '[[ .RunbookURL]]#Loki-Discarded-Samples-Warning'
+            summary: Loki is discarding samples during ingestion because they fail
+              validation.
+          expr: |
+            sum by(namespace, tenant, reason) (
+              irate(loki_discarded_samples_total{
+                reason!="rate_limited",
+                reason!="per_stream_rate_limit",
+                reason!="stream_limit"}[2m])
+            )
+            > 0
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: openshift4-logging
         - alert: SYN_LokistackSchemaUpgradesRequired
           annotations:
             message: |-
diff --git a/tests/golden/multilineerr/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml b/tests/golden/multilineerr/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
index 65a573e..4f6c7da 100644
--- a/tests/golden/multilineerr/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
+++ b/tests/golden/multilineerr/openshift4-logging/openshift4-logging/60_lokistack_alerts.yaml
@@ -204,6 +204,27 @@ spec:
             severity: warning
             syn: 'true'
             syn_component: openshift4-logging
+        - alert: SYN_LokiDiscardedSamplesWarning
+          annotations:
+            message: |-
+              Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion.
+              Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second.
+            runbook_url: '[[ .RunbookURL]]#Loki-Discarded-Samples-Warning'
+            summary: Loki is discarding samples during ingestion because they fail
+              validation.
+          expr: |
+            sum by(namespace, tenant, reason) (
+              irate(loki_discarded_samples_total{
+                reason!="rate_limited",
+                reason!="per_stream_rate_limit",
+                reason!="stream_limit"}[2m])
+            )
+            > 0
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: openshift4-logging
         - alert: SYN_LokistackSchemaUpgradesRequired
           annotations:
             message: |-