Merge pull request #3976 from Yelp/joycek/default-endpoint-alerting

Add and document new fields for default endpoint alerting
Yelp · Oct 2, 2024 · 4854ceb · 4854ceb
2 parents 117cdb5 + 1e43b43
commit 4854ceb
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 5 deletions.
diff --git a/docs/source/yelpsoa_configs.rst b/docs/source/yelpsoa_configs.rst
@@ -1088,6 +1088,7 @@ These keys provide optional overrides for the default alerting behaviour.
           page_nonprod: true
           error_threshold_ratio: 0.02
           minimum_error_rps: 10
+          default_endpoint_alerting: true
           endpoints:
             - name: GET /something
             - name: GET /something/else
@@ -1107,12 +1108,15 @@ These keys provide optional overrides for the default alerting behaviour.
      respects the paging behaviour set in the monitoring.yaml file. Override that here if required.
    - ``page_nonprod``: Override the default paging behaviour for non-production
      environments. Defaults to **false**.
-   - ``error_threshold_ratio``: Error threshold ratio (0-1). Defaults to **0.01**.
-   - ``minimum_error_rps``: Minimum error rate per second, minimum is zero. Defaults to **5**.
-   - ``endpoints``: List of endpoints to create alerts for.
+   - ``error_threshold_ratio``: Error threshold ratio (0-1) for errors under this namespace. Defaults to **0.01**.
+   - ``minimum_error_rps``: Minimum error rate per second for errors under this namespace before an alert can be triggered, minimum is zero. Defaults to **5**.
+   - ``default_endpoint_alerting``: Turn on alerts for all endpoints in this namespace. Defaults to **false**.
+   - ``endpoint_error_threshold_ratio``: Error threshold ratio (0-1) for errors to any singular endpoint. Defaults to the namespace ``error_threshold_ratio`` if specified, or **0.01**.
+   - ``endpoint_minimum_error_rps``: Minimum error rate per second for errors to any singular endpoint before an alert can be triggered for errors to any singular endpoint. Defaults to the namespace ``minimum_error_rps`` if specified, or **5**.
+   - ``endpoints``: List of endpoints to create specific alerts for.
       - ``name``: The name of the endpoint.
-      - ``error_threshold_ratio``: Error threshold ratio (0-1). If not specified the threshold will be inherited from the parent.
-      - ``minimum_error_rps``: Minimum error rate per second for the endpoint. Minimum is zero. Defaults to **5**. If not specified the threshold will be inherited from the parent.
+      - ``error_threshold_ratio``: Error threshold ratio (0-1). If not specified the threshold will be inherited from the ``endpoint_error_threshold_ratio``; if that is not specified then the namespace's ``error_threshold_ratio``; otherwise **0.01**.
+      - ``minimum_error_rps``: Minimum error rate per second for the endpoint. Minimum is zero. If not specified the threshold will be inherited from the ``endpoint_minimum_error_rps``; if that is not specified then the namespace's ``minimum_error_rps``; otherwise **5**.
 
 Moving a Service to a different location type
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/paasta_tools/cli/schemas/smartstack_schema.json b/paasta_tools/cli/schemas/smartstack_schema.json
@@ -258,6 +258,18 @@
                         "type": "integer",
                         "minimum": 0
                     },
+                    "default_endpoint_alerting": {
+                        "type": "boolean"
+                    },
+                    "endpoint_error_threshold_ratio": {
+                        "type": "number",
+                        "minimum": 0,
+                        "maximum": 1
+                    },
+                    "endpoint_minimum_error_rps": {
+                        "type": "integer",
+                        "minimum": 0
+                    },
                     "endpoints": {
                         "type": "array",
                         "items": {