From 2151717cd9c98d648c7c8de587248dcd7d53986b Mon Sep 17 00:00:00 2001 From: Joyce Kung Date: Wed, 2 Oct 2024 13:25:29 -0700 Subject: [PATCH 1/2] Add and document new fields for default endpoint alerting --- docs/source/yelpsoa_configs.rst | 10 +++++++--- paasta_tools/cli/schemas/smartstack_schema.json | 12 ++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/source/yelpsoa_configs.rst b/docs/source/yelpsoa_configs.rst index a3ac7ccb53..b011d8d760 100644 --- a/docs/source/yelpsoa_configs.rst +++ b/docs/source/yelpsoa_configs.rst @@ -1088,6 +1088,7 @@ These keys provide optional overrides for the default alerting behaviour. page_nonprod: true error_threshold_ratio: 0.02 minimum_error_rps: 10 + default_endpoint_alerting: true endpoints: - name: GET /something - name: GET /something/else @@ -1107,9 +1108,12 @@ These keys provide optional overrides for the default alerting behaviour. respects the paging behaviour set in the monitoring.yaml file. Override that here if required. - ``page_nonprod``: Override the default paging behaviour for non-production environments. Defaults to **false**. - - ``error_threshold_ratio``: Error threshold ratio (0-1). Defaults to **0.01**. - - ``minimum_error_rps``: Minimum error rate per second, minimum is zero. Defaults to **5**. - - ``endpoints``: List of endpoints to create alerts for. + - ``error_threshold_ratio``: Error threshold ratio (0-1) for errors under this namespace. Defaults to **0.01**. + - ``minimum_error_rps``: Minimum error rate per second for errors under this namespace before an alert can be triggered, minimum is zero. Defaults to **5**. + - ``default_endpoint_alerting``: Turn on alerts for all endpoints in this namespace. Defaults to **false**. + - ``endpoint_error_threshold_ratio``: Error threshold ratio (0-1) for errors to any singular endpoint. Defaults to the namespace ``error_threshold_ratio`` if specified, or **0.01**. + - ``endpoint_minimum_error_rps``: Minimum error rate per second for errors to any singular endpoint before an alert can be triggered for errors to any singular endpoint. Defaults to the namespace ``minimum_error_rps`` if specified, or **5**. + - ``endpoints``: List of endpoints to create specific alerts for. - ``name``: The name of the endpoint. - ``error_threshold_ratio``: Error threshold ratio (0-1). If not specified the threshold will be inherited from the parent. - ``minimum_error_rps``: Minimum error rate per second for the endpoint. Minimum is zero. Defaults to **5**. If not specified the threshold will be inherited from the parent. diff --git a/paasta_tools/cli/schemas/smartstack_schema.json b/paasta_tools/cli/schemas/smartstack_schema.json index 49c89913e7..03bde4b082 100644 --- a/paasta_tools/cli/schemas/smartstack_schema.json +++ b/paasta_tools/cli/schemas/smartstack_schema.json @@ -258,6 +258,18 @@ "type": "integer", "minimum": 0 }, + "default_endpoint_alerting": { + "type": "boolean" + }, + "endpoint_error_threshold_ratio": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "endpoint_minimum_error_rps": { + "type": "integer", + "minimum": 0 + }, "endpoints": { "type": "array", "items": { From 1e43b432ff6607b8ff842807f8131afcbf9fd271 Mon Sep 17 00:00:00 2001 From: Joyce Kung Date: Wed, 2 Oct 2024 14:08:05 -0700 Subject: [PATCH 2/2] Update endpoint notes to make it more clear inheritance --- docs/source/yelpsoa_configs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/yelpsoa_configs.rst b/docs/source/yelpsoa_configs.rst index b011d8d760..a247eca71b 100644 --- a/docs/source/yelpsoa_configs.rst +++ b/docs/source/yelpsoa_configs.rst @@ -1115,8 +1115,8 @@ These keys provide optional overrides for the default alerting behaviour. - ``endpoint_minimum_error_rps``: Minimum error rate per second for errors to any singular endpoint before an alert can be triggered for errors to any singular endpoint. Defaults to the namespace ``minimum_error_rps`` if specified, or **5**. - ``endpoints``: List of endpoints to create specific alerts for. - ``name``: The name of the endpoint. - - ``error_threshold_ratio``: Error threshold ratio (0-1). If not specified the threshold will be inherited from the parent. - - ``minimum_error_rps``: Minimum error rate per second for the endpoint. Minimum is zero. Defaults to **5**. If not specified the threshold will be inherited from the parent. + - ``error_threshold_ratio``: Error threshold ratio (0-1). If not specified the threshold will be inherited from the ``endpoint_error_threshold_ratio``; if that is not specified then the namespace's ``error_threshold_ratio``; otherwise **0.01**. + - ``minimum_error_rps``: Minimum error rate per second for the endpoint. Minimum is zero. If not specified the threshold will be inherited from the ``endpoint_minimum_error_rps``; if that is not specified then the namespace's ``minimum_error_rps``; otherwise **5**. Moving a Service to a different location type ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~