From 7bf5931fcd75f2dab857e773f03a4fa01914a8a6 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:02:54 +0100 Subject: [PATCH 01/65] Create opensearch_discover.py --- elastalert/opensearch_discover.py | 205 ++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 elastalert/opensearch_discover.py diff --git a/elastalert/opensearch_discover.py b/elastalert/opensearch_discover.py new file mode 100644 index 00000000..3a59e52f --- /dev/null +++ b/elastalert/opensearch_discover.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import datetime +import logging +import json +import os.path +import prison +import urllib.parse + +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key +from .util import ts_add + +kibana_default_timedelta = datetime.timedelta(minutes=10) + +opensearch_versions = frozenset([ + '2.11' + ]) + +def generate_opensearch_discover_url(rule, match): + ''' Creates a link for a kibana discover app. ''' + + discover_app_url = rule.get('kibana_discover_app_url') + if not discover_app_url: + elastalert_logger.warning( + 'Missing kibana_discover_app_url for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + opensearch_version = rule.get('kibana_discover_version') + if not opensearch_version: + elastalert_logger.warning( + 'Missing kibana_discover_version for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + index = rule.get('kibana_discover_index_pattern_id') + if not index: + elastalert_logger.warning( + 'Missing opensearch_discover_index_pattern_id for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + columns = rule.get('kibana_discover_columns', ['_source']) + filters = rule.get('filter', []) + + if 'query_key' in rule: + query_keys = rule.get('compound_query_key', [rule['query_key']]) + else: + query_keys = [] + + timestamp = lookup_es_key(match, rule['timestamp_field']) + timeframe = rule.get('timeframe', kibana_default_timedelta) + from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe) + from_time = ts_add(timestamp, -from_timedelta) + to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe) + to_time = ts_add(timestamp, to_timedelta) + + if opensearch_version in opensearch_versions: + globalState = opensearch_disover_global_state(from_time, to_time) + appState = opensearch_discover_app_state(index, columns, filters, query_keys, match) + appFilter = opensearch_discover_app_filter(index, columns, filters, query_keys, match) + + else: + elastalert_logger.warning( + 'Unknown opensearch discover application version %s for rule %s' % ( + opensearch_version, + rule.get('name', '') + ) + ) + return None + + urlqueryOriginal = "%s?_g=%s&_a=%s&_q=%s" % ( + os.path.expandvars(discover_app_url), + urllib.parse.quote(globalState), + urllib.parse.quote(appState), + urllib.parse.quote(appFilter) + ) + + word_to_replace = 'tobereplacedbylucenequery' + replacement_word = 'query' + max_replacements = 1 # Replace only the first occurrence + urlquery = urlqueryOriginal.replace(word_to_replace, replacement_word, max_replacements) + return urlquery + + +def opensearch_disover_global_state(from_time, to_time): + return prison.dumps( { + 'filters': [], + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'to': to_time + } + } ) + + +def opensearch_discover_app_state(index, columns, filters, query_keys, match): + return prison.dumps( { + 'discover': { + 'columns': columns, + 'isDirty': False, + 'sort': [] + }, + 'metadata': { + 'indexPattern': index, + 'view': 'discover' + } + } ) + +def opensearch_discover_app_filter(index, columns, filters, query_keys, match): + app_filters = [] + + if filters: + + # Remove nested query since the outer most query key will break Kibana 8. + new_filters = [] + for filter in filters: + if 'query' in filter: + filter = filter['query'] + new_filters.append(filter) + filters = new_filters + + bool_filter = {'bool': {'must': filters } } + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': 'filter', + 'disabled': False, + 'index': index, + 'key': 'query', + 'negate': False, + 'type': 'custom', + 'value': json.dumps(bool_filter, separators=(',', ':')) + }, + 'query': bool_filter + } ) + + for query_key in query_keys: + query_value = lookup_es_key(match, query_key) + + if query_value is None: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'exists': { + 'field': query_key + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': True, + 'type': 'exists', + 'view': 'discover', + 'value': 'exists' + } + } ) + + else: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': False, + 'params': { + 'query': query_value, + 'type': 'phrase' + }, + 'type': 'phrase', + 'value': str(query_value) + }, + 'query': { + 'match': { + query_key: { + 'query': query_value, + 'type': 'phrase' + } + } + } + } ) + + return prison.dumps( { + 'filters': app_filters, + 'tobereplacedbylucenequery': {'language': 'lucene','query': ''} + } ) From 72c6a1fd9e99293521ed31eaf27938d26b91a227 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:05:52 +0100 Subject: [PATCH 02/65] Update kibana_external_url_formatter.py --- elastalert/kibana_external_url_formatter.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/elastalert/kibana_external_url_formatter.py b/elastalert/kibana_external_url_formatter.py index 5d739e72..f95f2b3e 100644 --- a/elastalert/kibana_external_url_formatter.py +++ b/elastalert/kibana_external_url_formatter.py @@ -157,3 +157,15 @@ def create_kibana_external_url_formatter( return ShortKibanaExternalUrlFormatter(base_url, auth, security_tenant, new_shortener, verify) return AbsoluteKibanaExternalUrlFormatter(base_url, security_tenant) + +def create_opensearch_external_url_formatter( + rule, + shorten: bool, + security_tenant: str, +) -> KibanaExternalUrlFormatter: + '''Creates a Kibana external url formatter''' + + base_url = rule.get('kibana_url') + + return AbsoluteKibanaExternalUrlFormatter(base_url, security_tenant) + From c6143b85e28b16d8507b312b683fb7c61baf1bb9 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:13:21 +0100 Subject: [PATCH 03/65] Update elastalert.py --- elastalert/elastalert.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 28de6e8e..9a23412b 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -34,6 +34,8 @@ from elastalert.enhancements import DropMatchException from elastalert.kibana_discover import generate_kibana_discover_url from elastalert.kibana_external_url_formatter import create_kibana_external_url_formatter +from elastalert.opensearch_discover import generate_opensearch_discover_url +from elastalert.kibana_external_url_formatter import create_opensearch_external_url_formatter from elastalert.prometheus_wrapper import PrometheusWrapper from elastalert.ruletypes import FlatlineRule from elastalert.util import (add_keyword_postfix, cronite_datetime_to_timestamp, dt_to_ts, dt_to_unix, EAException, @@ -1347,6 +1349,14 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): kb_link_formatter = self.get_kibana_discover_external_url_formatter(rule) matches[0]['kibana_discover_url'] = kb_link_formatter.format(kb_link) + if rule.get('generate_opensearch_discover_url'): + kb_link = generate_opensearch_discover_url(rule, matches[0]) + if kb_link: + kb_link_formatter = self.get_opensearch_discover_external_url_formatter(rule) + matches[0]['kibana_discover_url'] = kb_link_formatter.format(kb_link) + + + # Enhancements were already run at match time if # run_enhancements_first is set or # retried==True, which means this is a retry of a failed alert @@ -1439,6 +1449,18 @@ def get_kibana_discover_external_url_formatter(self, rule): rule[key] = formatter return formatter + + def get_opensearch_discover_external_url_formatter(self, rule): + """ Gets or create the external url formatter for kibana discover links """ + key = '__kibana_discover_external_url_formatter__' + formatter = rule.get(key) + if formatter is None: + shorten = rule.get('shorten_kibana_discover_url') + security_tenant = rule.get('kibana_discover_security_tenant') + formatter = create_opensearch_external_url_formatter(rule, shorten, security_tenant) + rule[key] = formatter + return formatter + def writeback(self, doc_type, body, rule=None, match_body=None): # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: From e2e24d5fb5fb6f4394da491560dbb658014e68a9 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:15:39 +0100 Subject: [PATCH 04/65] Update schema.yaml --- elastalert/schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 81b954b4..4c2b8627 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -284,7 +284,7 @@ properties: query_key: *arrayOfString replace_dots_in_field_names: {type: boolean} scan_entire_timeframe: {type: boolean} - + ### summary table summary_table_fields: {type: array, items: {type: string}} summary_table_type: {type: string, enum: ['ascii', 'html', 'markdown']} @@ -302,7 +302,7 @@ properties: kibana_discover_from_timedelta: *timedelta kibana_discover_to_timedelta: *timedelta kibana_discover_security_tenant: {type:string} - + generate_opensearch_discover_url: {type: boolean} # Alert Content alert_text: {type: string} # Python format string alert_text_args: {type: array, items: {type: string}} From a1e78ff87778973d8ca7d0adb5506e4b7d11de9a Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:27:58 +0100 Subject: [PATCH 05/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 3aab79c9..b0b89d5c 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -70,6 +70,8 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``generate_kibana_discover_url`` (boolean, default False) | | +--------------------------------------------------------------+ | +| ``generate_opensearch_discover_url`` (boolean, default False)| | ++--------------------------------------------------------------+ | | ``shorten_kibana_discover_url`` (boolean, default False) | | +--------------------------------------------------------------+ | | ``kibana_discover_app_url`` (string, no default) | | From 98c3e4038a2d67fe4ae6e7fc94bb49885bef070c Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:42:59 +0100 Subject: [PATCH 06/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index b0b89d5c..c54b99d2 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -627,7 +627,7 @@ kibana_url ``kibana_url``: The base url of the Kibana application. If not specified, a URL will be constructed using ``es_host`` and ``es_port``. -This value will be used if ``generate_kibana_discover_url`` is true and ``kibana_discover_app_url`` is a relative path +This value will be used if ``generate_kibana_discover_url`` or ``generate_opensearch_discover_url`` is true and ``kibana_discover_app_url`` is a relative path (Optional, string, default ``http://:/_plugin/kibana/``) @@ -685,6 +685,39 @@ Example kibana_url + kibana_discover_app_url usage:: alert_text_args: [ kibana_discover_url ] alert_text_type: alert_text_only +generate_opensearch_discover_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``generate_opensearch_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Opensearch Discover application. +This setting requires the following settings are also configured: + +- ``kibana_discover_app_url`` +- ``kibana_discover_version`` +- ``kibana_discover_index_pattern_id`` + +``generate_opensearch_discover_url: true`` + +Example kibana_discover_app_url only usage for opensearch:: + + generate_opensearch_discover_url: true + kibana_discover_app_url: "http://localhost:5601/app/data-explorer/discover?security_tenant=Admin#" + kibana_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + kibana_discover_version: "2.11" + alert_text: '{}' + alert_text_args: [ kibana_discover_url ] + alert_text_type: alert_text_only + +Example kibana_url + kibana_discover_app_url usage for opensearch:: + + generate_kibana_discover_url: true + kibana_url: "http://localhost:5601/" + kibana_discover_app_url: "app/data-explorer/discover?security_tenant=Admin#" + kibana_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + kibana_discover_version: "2.11" + alert_text: '{}' + alert_text_args: [ kibana_discover_url ] + alert_text_type: alert_text_only + shorten_kibana_discover_url ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 043575802fb93b26bd2b8865b722fe8259869c0b Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:01:21 +0100 Subject: [PATCH 07/65] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f714b89f..42ff5edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ## New features - [Iris] Alerter added - [#1301](https://github.com/jertel/elastalert2/pull/1301) - @malinkinsa - +- [Opensearch] Add the possibility to generate an opensearch discovery url - [#1310](https://github.com/jertel/elastalert2/pull/1310) ## Other changes - Refactored FlatlineRule to make it more extensible - [#1291](https://github.com/jertel/elastalert2/pull/1291) - @rundef - Add support for Kibana 8.11 for Kibana Discover - [#1305](https://github.com/jertel/elastalert2/pull/1305) - @nsano-rururu From 7f5b3ae338c21e2d10f7adfef324d55ebb658ed9 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:36:07 +0100 Subject: [PATCH 08/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index c54b99d2..ed35adb1 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -686,7 +686,7 @@ Example kibana_url + kibana_discover_app_url usage:: alert_text_type: alert_text_only generate_opensearch_discover_url -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``generate_opensearch_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Opensearch Discover application. This setting requires the following settings are also configured: From 8eb32521d3fa99cbacf276ad73278f2e4d417ba7 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:21:20 +0100 Subject: [PATCH 09/65] Update schema.yaml --- elastalert/schema.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 4c2b8627..85885f02 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -302,7 +302,17 @@ properties: kibana_discover_from_timedelta: *timedelta kibana_discover_to_timedelta: *timedelta kibana_discover_security_tenant: {type:string} + + ### Opensearch discover App Link generate_opensearch_discover_url: {type: boolean} + opensearch_url: {type: string, format: uri} + opensearch_discover_app_url: {type: string} + opensearch_discover_version: {type: string, enum: ['2.11']} + opensearch_discover_index_pattern_id: {type: string, minLength: 1} + opensearch_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} + opensearch_discover_from_timedelta: *timedelta + opensearch_discover_to_timedelta: *timedelta + # Alert Content alert_text: {type: string} # Python format string alert_text_args: {type: array, items: {type: string}} From 72eef1b914b46459272a4c895b892f0b325bedb8 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:21:29 +0100 Subject: [PATCH 10/65] Update opensearch_discover.py --- elastalert/opensearch_discover.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/elastalert/opensearch_discover.py b/elastalert/opensearch_discover.py index 3a59e52f..e35b0f25 100644 --- a/elastalert/opensearch_discover.py +++ b/elastalert/opensearch_discover.py @@ -12,7 +12,7 @@ from .util import lookup_es_key from .util import ts_add -kibana_default_timedelta = datetime.timedelta(minutes=10) +opensearch_default_timedelta = datetime.timedelta(minutes=10) opensearch_versions = frozenset([ '2.11' @@ -30,16 +30,16 @@ def generate_opensearch_discover_url(rule, match): ) return None - opensearch_version = rule.get('kibana_discover_version') + opensearch_version = rule.get('opensearch_discover_version') if not opensearch_version: elastalert_logger.warning( - 'Missing kibana_discover_version for rule %s' % ( + 'Missing opensearch_discover_version for rule %s' % ( rule.get('name', '') ) ) return None - index = rule.get('kibana_discover_index_pattern_id') + index = rule.get('opensearch_discover_index_pattern_id') if not index: elastalert_logger.warning( 'Missing opensearch_discover_index_pattern_id for rule %s' % ( @@ -48,7 +48,7 @@ def generate_opensearch_discover_url(rule, match): ) return None - columns = rule.get('kibana_discover_columns', ['_source']) + columns = rule.get('opensearch_discover_columns', ['_source']) filters = rule.get('filter', []) if 'query_key' in rule: @@ -57,10 +57,10 @@ def generate_opensearch_discover_url(rule, match): query_keys = [] timestamp = lookup_es_key(match, rule['timestamp_field']) - timeframe = rule.get('timeframe', kibana_default_timedelta) - from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe) + timeframe = rule.get('timeframe', opensearch_default_timedelta) + from_timedelta = rule.get('opensearch_discover_from_timedelta', timeframe) from_time = ts_add(timestamp, -from_timedelta) - to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe) + to_timedelta = rule.get('opensearch_discover_to_timedelta', timeframe) to_time = ts_add(timestamp, to_timedelta) if opensearch_version in opensearch_versions: From a21862e67e7c22b376f2b7601cf52f30ab80801e Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:21:40 +0100 Subject: [PATCH 11/65] Update elastalert.py --- elastalert/elastalert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 9a23412b..7a736b26 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1353,7 +1353,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): kb_link = generate_opensearch_discover_url(rule, matches[0]) if kb_link: kb_link_formatter = self.get_opensearch_discover_external_url_formatter(rule) - matches[0]['kibana_discover_url'] = kb_link_formatter.format(kb_link) + matches[0]['opensearch_discover_url'] = kb_link_formatter.format(kb_link) From 01a882c9827b46c39226a529577549a01cb742e2 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:23:04 +0100 Subject: [PATCH 12/65] Update loaders.py --- elastalert/loaders.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index aa175dca..a94e9001 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -331,6 +331,10 @@ def load_options(self, rule, conf, filename, args=None): rule['kibana_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta']) if 'kibana_discover_to_timedelta' in rule: rule['kibana_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) + if 'opensearch_discover_from_timedelta' in rule: + rule['opensearch_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta']) + if 'opensearch_discover_to_timedelta' in rule: + rule['opensearch_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) except (KeyError, TypeError) as e: raise EAException('Invalid time format used: %s' % e) From b3209b8ae9a3eb8f7443b8c49460d405e4007d95 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 10:56:47 +0100 Subject: [PATCH 13/65] Update slack.py --- elastalert/alerters/slack.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/elastalert/alerters/slack.py b/elastalert/alerters/slack.py index 39357264..424fe6ac 100644 --- a/elastalert/alerters/slack.py +++ b/elastalert/alerters/slack.py @@ -36,6 +36,9 @@ def __init__(self, rule): self.slack_attach_kibana_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) self.slack_kibana_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') self.slack_kibana_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Kibana') + self.slack_attach_opensearch_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) + self.slack_opensearch_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') + self.slack_opensearch_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Opensearch') self.slack_footer = self.rule.get('slack_footer', '') self.slack_footer_icon = self.rule.get('slack_footer_icon', '') self.slack_image_url = self.rule.get('slack_image_url', '') @@ -141,6 +144,14 @@ def alert(self, matches): 'title': self.slack_kibana_discover_title, 'title_link': kibana_discover_url }) + if self.slack_attach_opensearch_discover_url: + opensearch_discover_url = lookup_es_key(matches[0], 'opensearch_discover_url') + if opensearch_discover_url: + payload['attachments'].append({ + 'color': self.slack_opensearch_discover_color, + 'title': self.slack_opensearch_discover_title, + 'title_link': opensearch_discover_url + }) if self.slack_attach_jira_ticket_url and self.pipeline is not None and 'jira_ticket' in self.pipeline: jira_url = '%s/browse/%s' % (self.pipeline['jira_server'], self.pipeline['jira_ticket']) From 94941dce2a813a592af51125f8c21839904219fc Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:42:05 +0100 Subject: [PATCH 14/65] Update teams.py --- elastalert/alerters/teams.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/alerters/teams.py b/elastalert/alerters/teams.py index 731ef31e..b072b1f5 100644 --- a/elastalert/alerters/teams.py +++ b/elastalert/alerters/teams.py @@ -25,6 +25,8 @@ def __init__(self, rule): self.ms_teams_alert_facts = self.rule.get('ms_teams_alert_facts', '') self.ms_teams_attach_kibana_discover_url = self.rule.get('ms_teams_attach_kibana_discover_url', False) self.ms_teams_kibana_discover_title = self.rule.get('ms_teams_kibana_discover_title', 'Discover in Kibana') + self.ms_teams_attach_opensearch_discover_url = self.rule.get('ms_teams_attach_opensearch_discover_url', False) + self.ms_teams_opensearch_discover_title = self.rule.get('ms_teams_opensearch_discover_title', 'Discover in opensearch') def format_body(self, body): if self.ms_teams_alert_fixed_width: From 39ea6f02a244b75e4bbd32e8d4950b7688b0dc29 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:44:43 +0100 Subject: [PATCH 15/65] Update mattermost.py --- elastalert/alerters/mattermost.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/elastalert/alerters/mattermost.py b/elastalert/alerters/mattermost.py index 3dd30c16..e04c774d 100644 --- a/elastalert/alerters/mattermost.py +++ b/elastalert/alerters/mattermost.py @@ -47,6 +47,9 @@ def __init__(self, rule): self.mattermost_attach_kibana_discover_url = self.rule.get('mattermost_attach_kibana_discover_url', False) self.mattermost_kibana_discover_color = self.rule.get('mattermost_kibana_discover_color', '#ec4b98') self.mattermost_kibana_discover_title = self.rule.get('mattermost_kibana_discover_title', 'Discover in Kibana') + self.mattermost_attach_opensearch_discover_url = self.rule.get('mattermost_attach_opensearch_discover_url', False) + self.mattermost_opensearch_discover_color = self.rule.get('mattermost_opensearch_discover_color', '#ec4b98') + self.mattermost_opensearch_discover_title = self.rule.get('mattermost_opensearch_discover_title', 'Discover in Kibana') def get_aggregation_summary_text__maximum_width(self): width = super(MattermostAlerter, self).get_aggregation_summary_text__maximum_width() @@ -143,7 +146,17 @@ def alert(self, matches): 'title': self.mattermost_kibana_discover_title, 'title_link': kibana_discover_url }) + + if self.mattermost_attach_opensearch_discover_url: + opensearch_discover_url = lookup_es_key(matches[0], 'opensearch_discover_url') + if kibana_discover_url: + payload['attachments'].append({ + 'color': self.mattermost_opensearch_discover_color, + 'title': self.mattermost_opensearch_discover_title, + 'title_link': opensearch_discover_url + }) + for url in self.mattermost_webhook_url: for channel_override in self.mattermost_channel_override: try: From b99e173b5047dc9c6aaeb251c6eb02d3312879da Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:46:25 +0100 Subject: [PATCH 16/65] Update teams.py --- elastalert/alerters/teams.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/elastalert/alerters/teams.py b/elastalert/alerters/teams.py index b072b1f5..7e6882ab 100644 --- a/elastalert/alerters/teams.py +++ b/elastalert/alerters/teams.py @@ -90,6 +90,21 @@ def alert(self, matches): ], } ] + if self.ms_teams_attach_opensearch_discover_url: + opensearch_discover_url = lookup_es_key(matches[0], 'opensearch_discover_url') + if opensearch_discover_url: + payload['potentialAction'] = [ + { + '@type': 'OpenUri', + 'name': self.ms_teams_opensearch_discover_title, + 'targets': [ + { + 'os': 'default', + 'uri': opensearch_discover_url, + } + ], + } + ] for url in self.ms_teams_webhook_url: try: From 9cd64964918ce77eb6806d4239c7ea3613247a46 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:48:36 +0100 Subject: [PATCH 17/65] Update rocketchat.py --- elastalert/alerters/rocketchat.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/elastalert/alerters/rocketchat.py b/elastalert/alerters/rocketchat.py index 6b54ba94..0ba781ae 100644 --- a/elastalert/alerters/rocketchat.py +++ b/elastalert/alerters/rocketchat.py @@ -31,6 +31,9 @@ def __init__(self, rule): self.rocket_chat_attach_kibana_discover_url = self.rule.get('rocket_chat_attach_kibana_discover_url', False) self.rocket_chat_kibana_discover_color = self.rule.get('rocket_chat_kibana_discover_color', '#ec4b98') self.rocket_chat_kibana_discover_title = self.rule.get('rocket_chat_kibana_discover_title', 'Discover in Kibana') + self.rocket_chat_attach_opensearch_discover_url = self.rule.get('rocket_chat_attach_opensearch_discover_url', False) + self.rocket_chat_opensearch_discover_color = self.rule.get('rocket_chat_opensearch_discover_color', '#ec4b98') + self.rocket_chat_opensearch_discover_title = self.rule.get('rocket_chat_opensearch_discover_title', 'Discover in Kibana') self.rocket_chat_ignore_ssl_errors = self.rule.get('rocket_chat_ignore_ssl_errors', False) self.rocket_chat_timeout = self.rule.get('rocket_chat_timeout', 10) self.rocket_chat_ca_certs = self.rule.get('rocket_chat_ca_certs') @@ -92,6 +95,15 @@ def alert(self, matches): 'title_link': kibana_discover_url }) + if self.rocket_chat_attach_opensearch_discover_url: + opensearch_discover_url = lookup_es_key(matches[0], 'opensearch_discover_url') + if opensearch_discover_url: + payload['attachments'].append({ + 'color': self.rocket_chat_opensearch_discover_color, + 'title': self.rocket_chat_opensearch_discover_title, + 'title_link': opensearch_discover_url + }) + for url in self.rocket_chat_webhook_url: for channel_override in self.rocket_chat_channel_override: try: From 5c958b91e9a31bffef1380c71902cfabb8a7d80b Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:50:38 +0100 Subject: [PATCH 18/65] Update schema.yaml --- elastalert/schema.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 85885f02..a41c775d 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -619,6 +619,10 @@ properties: mattermost_attach_kibana_discover_url: {type: boolean} mattermost_kibana_discover_color: {type: string} mattermost_kibana_discover_title: {type: string} + mattermost_attach_opensearch_discover_url: {type: boolean} + mattermost_opensearch_discover_color: {type: string} + mattermost_opensearch_discover_title: {type: string} + ### Microsoft Teams ms_teams_webhook_url: *arrayOfString @@ -631,6 +635,8 @@ properties: ms_teams_kibana_discover_title: {type: string} ms_teams_ca_certs: {type: [boolean, string]} ms_teams_ignore_ssl_errors: {type: boolean} + ms_teams_attach_opensearch_discover_url: {type: boolean} + ms_teams_opensearch_discover_title: {type: string} ### Opsgenie opsgenie_key: {type: string} @@ -728,6 +734,9 @@ properties: slack_attach_kibana_discover_url: {type: boolean} slack_kibana_discover_color: {type: string} slack_kibana_discover_title: {type: string} + slack_attach_opensearch_discover_url: {type: boolean} + slack_opensearch_discover_color: {type: string} + slack_opensearch_discover_title: {type: string} slack_attach_jira_ticket_url: {type: boolean} slack_jira_ticket_color: {type: string} slack_jira_ticket_title: {type: string} From 47cc3d178555cef52b200536490d76f10d296835 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:36:40 +0100 Subject: [PATCH 19/65] Create opensearch_discover_test.py --- tests/opensearch_discover_test.py | 549 ++++++++++++++++++++++++++++++ 1 file changed, 549 insertions(+) create mode 100644 tests/opensearch_discover_test.py diff --git a/tests/opensearch_discover_test.py b/tests/opensearch_discover_test.py new file mode 100644 index 00000000..9ff60405 --- /dev/null +++ b/tests/opensearch_discover_test.py @@ -0,0 +1,549 @@ +# -*- coding: utf-8 -*- +from datetime import timedelta +import pytest + +from elastalert.opensearch_discover import generate_opensearch_discover_url + + +@pytest.mark.parametrize("opensearch_version", [ + '2.11' +]) +def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_url(opensearch_version): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': opensearch_version, + 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'view%3Adiscover%29%29' # app end + + '&_q=%28filters%3A%21%28%29%2C'#query and filter start + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' #query and filter start + ) + + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_url_without_http(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'app/discover#/', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': '620ad0e6-43df-4557-bda2-384960fa9086', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2021-10-08T00:30:00Z' + } + ) + + expectedUrl = ( + 'app/discover#/' + + '?_g=%28' + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272021-10-08T00%3A20%3A00Z%27%2C' + + 'to%3A%272021-10-08T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27620ad0e6-43df-4557-bda2-384960fa9086%27%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_missing_opensearch_discover_version(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_opensearch_discover_url_with_missing_opensearch_discover_app_url(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_version': '8.11', + 'opensearch_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_opensearch_discover_url_with_missing_opensearch_discover_index_pattern_id(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '8.11', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_opensearch_discover_url_with_invalid_opensearch_version(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '4.5', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + +def test_generate_opensearch_discover_url_with_from_timedelta_and_timeframe(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'opensearch_discover_from_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A20%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_to_timedelta_and_timeframe(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'opensearch_discover_to_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_custom_columns(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'opensearch_discover_columns': ['level', 'message'], + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27%' + + '29%' + + '29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28level%2Cmessage%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_single_filter(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + + + + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27%' + + '29%' + + '29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' + + '%29' + + '%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29' + + '%2Cmeta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29' + + '%2Cquery%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_multiple_filters(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'app': 'test'}}, + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28' + + 'alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' + + 'bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22app%22%3A%22test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_int_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo.dest': 200 + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29' + + '%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3A200%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3A%27200%27%29%2C' + + 'query%3A%28match%3A%28geo.dest%3A%28query%3A200%2C' + + 'type%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_str_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'dest': 'ok' + } + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29%' + + '29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' + + '%29' + + '%29' + + '&_q=' + + '%28filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2C' + + 'type%3Aphrase%2Cvalue%3Aok%29%2Cquery%3A%28match%3A%28geo.dest%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_missing_query_key_value(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' + + '%29' + + '%29&' + + '_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cexists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2Cnegate%3A%21t%2Ctype%3Aexists%2Cvalue%3Aexists%2Cview%3Adiscover%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_compound_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'compound_query_key': ['geo.src', 'geo.dest'], + 'query_key': 'geo.src,geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'src': 'CA', + 'dest': 'US' + } + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Ageo.src%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3ACA%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3ACA%29%2C' + + 'query%3A%28match%3A%28geo.src%3A%28query%3ACA%2Ctype%3Aphrase%29%29%29%29%' + + '2C%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2Ckey%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3AUS%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3AUS%29%2C' + + 'query%3A%28match%3A%28geo.dest%3A%28' + + 'query%3AUS%2Ctype%3Aphrase%29%29%29%29%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_filter_and_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' + + 'bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%2C%28%27%24' + + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Astatus' + + '%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' + + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + +def test_generate_opensearch_discover_url_with_querystring_filter_and_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'query': {'query_string': {'query': 'hello world'}}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3A' + + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22must%22%3A%5B%7B%22query_string%22%3A%7B%22' + + 'query%22%3A%22hello%20world%22%7D%7D%5D%7D%7D%27%29%2C' + + 'query%3A%28bool%3A%28must%3A%21%28%28query_string%3A%28query%3A%27hello%20world%27%29%29%29%29%29%29%2C%28%27%24' + + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Astatus%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' + + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl From 3896fe3b5808e924811b2206147a2cf95036ae8a Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:45:22 +0100 Subject: [PATCH 20/65] Update teams_test.py add opensearch discover related test --- tests/alerters/teams_test.py | 135 +++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/tests/alerters/teams_test.py b/tests/alerters/teams_test.py index e80c4d5e..5cdb1c7d 100644 --- a/tests/alerters/teams_test.py +++ b/tests/alerters/teams_test.py @@ -389,7 +389,142 @@ def test_ms_teams_kibana_discover_title(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data +def test_ms_teams_attach_opensearch_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'ms_teams_attach_opensearch_discover_url': True, + 'ms_teams_webhook_url': 'http://test.webhook.url', + 'ms_teams_alert_summary': 'Alert from ElastAlert', + 'alert': [], + 'alert_subject': 'Cool subject', + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = MsTeamsAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'opensearch_discover_url': 'http://opensearch#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + '@type': 'MessageCard', + '@context': 'http://schema.org/extensions', + 'summary': rule['ms_teams_alert_summary'], + 'title': rule['alert_subject'], + 'sections': [{'text': BasicMatchString(rule, match).__str__()}], + 'potentialAction': [ + { + '@type': 'OpenUri', + 'name': 'Discover in opensearch', + 'targets': [ + { + 'os': 'default', + 'uri': 'http://opensearch#discover', + } + ], + } + ], + } + mock_post_request.assert_called_once_with( + rule['ms_teams_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_ms_teams_attach_opensearch_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'ms_teams_attach_opensearch_discover_url': True, + 'ms_teams_webhook_url': 'http://test.webhook.url', + 'ms_teams_alert_summary': 'Alert from ElastAlert', + 'alert': [], + 'alert_subject': 'Cool subject', + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = MsTeamsAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + expected_data = { + '@type': 'MessageCard', + '@context': 'http://schema.org/extensions', + 'summary': rule['ms_teams_alert_summary'], + 'title': rule['alert_subject'], + 'sections': [{'text': BasicMatchString(rule, match).__str__()}], + } + mock_post_request.assert_called_once_with( + rule['ms_teams_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_ms_teams_opensearch_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'ms_teams_attach_opensearch_discover_url': True, + 'ms_teams_opensearch_discover_title': 'Click to discover in Kibana', + 'ms_teams_webhook_url': 'http://test.webhook.url', + 'ms_teams_alert_summary': 'Alert from ElastAlert', + 'alert': [], + 'alert_subject': 'Cool subject', + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = MsTeamsAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'opensearch_discover_url': 'http://opensearch#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + '@type': 'MessageCard', + '@context': 'http://schema.org/extensions', + 'summary': rule['ms_teams_alert_summary'], + 'title': rule['alert_subject'], + 'sections': [{'text': BasicMatchString(rule, match).__str__()}], + 'potentialAction': [ + { + '@type': 'OpenUri', + 'name': 'Click to discover in opensearch', + 'targets': [ + { + 'os': 'default', + 'uri': 'http://opensearch#discover', + } + ], + } + ], + } + mock_post_request.assert_called_once_with( + rule['ms_teams_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data def test_ms_teams_alert_facts(): rule = { 'name': 'Test Rule', From a5202017d784629ad26761ae6d4177ab3615dc27 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:49:49 +0100 Subject: [PATCH 21/65] Update mattermost.py --- elastalert/alerters/mattermost.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerters/mattermost.py b/elastalert/alerters/mattermost.py index e04c774d..545e43ad 100644 --- a/elastalert/alerters/mattermost.py +++ b/elastalert/alerters/mattermost.py @@ -49,7 +49,7 @@ def __init__(self, rule): self.mattermost_kibana_discover_title = self.rule.get('mattermost_kibana_discover_title', 'Discover in Kibana') self.mattermost_attach_opensearch_discover_url = self.rule.get('mattermost_attach_opensearch_discover_url', False) self.mattermost_opensearch_discover_color = self.rule.get('mattermost_opensearch_discover_color', '#ec4b98') - self.mattermost_opensearch_discover_title = self.rule.get('mattermost_opensearch_discover_title', 'Discover in Kibana') + self.mattermost_opensearch_discover_title = self.rule.get('mattermost_opensearch_discover_title', 'Discover in opensearch') def get_aggregation_summary_text__maximum_width(self): width = super(MattermostAlerter, self).get_aggregation_summary_text__maximum_width() From 24488fb648e9c6bc5f3d83f60b88bd85b03aa356 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:50:49 +0100 Subject: [PATCH 22/65] Update mattermost_test.py add opensearch discover url related test --- tests/alerters/mattermost_test.py | 198 ++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/tests/alerters/mattermost_test.py b/tests/alerters/mattermost_test.py index d8358a18..54d0866c 100644 --- a/tests/alerters/mattermost_test.py +++ b/tests/alerters/mattermost_test.py @@ -1142,6 +1142,204 @@ def test_mattermost_kibana_discover_color(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data +def test_mattermost_attach_opensearch_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_text_type': 'alert_text_only', + 'mattermost_attach_opensearch_discover_url': True, + 'mattermost_webhook_url': 'http://please.dontgohere.mattermost', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = MattermostAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'opensearch_discover_url': 'http://localhost:5601/app/discover#/' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'attachments': [ + { + 'fallback': 'Test Rule: ', + 'color': 'danger', + 'title': 'Test Rule', + 'pretext': '', + 'fields': [], + 'text': 'Test Rule\n\n' + }, + { + 'color': '#ec4b98', + 'title': 'Discover in opensearch', + 'title_link': 'http://localhost:5601/app/discover#/' + } + ], + 'username': 'elastalert', + 'channel': '', + 'icon_emoji': ':ghost:' + } + mock_post_request.assert_called_once_with( + rule['mattermost_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + verify=True, + proxies=None + ) + + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_mattermost_attach_opensearch_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_text_type': 'alert_text_only', + 'mattermost_attach_opensearch_discover_url': True, + 'mattermost_webhook_url': 'http://please.dontgohere.mattermost', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = MattermostAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'attachments': [ + { + 'fallback': 'Test Rule: ', + 'color': 'danger', + 'title': 'Test Rule', + 'pretext': '', + 'fields': [], + 'text': 'Test Rule\n\n' + } + ], + 'username': 'elastalert', + 'channel': '', + 'icon_emoji': ':ghost:' + } + mock_post_request.assert_called_once_with( + rule['mattermost_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + verify=True, + proxies=None + ) + + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_mattermost_opensearch_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_text_type': 'alert_text_only', + 'mattermost_attach_opensearch_discover_url': True, + 'mattermost_opensearch_discover_title': 'Click to discover in opensearch', + 'mattermost_webhook_url': 'http://please.dontgohere.mattermost', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = MattermostAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'opensearch_discover_url': 'http://localhost:5601/app/discover#/' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'attachments': [ + { + 'fallback': 'Test Rule: ', + 'color': 'danger', + 'title': 'Test Rule', + 'pretext': '', + 'fields': [], + 'text': 'Test Rule\n\n' + }, + { + 'color': '#ec4b98', + 'title': 'Click to discover in opensearch', + 'title_link': 'http://localhost:5601/app/discover#/' + } + ], + 'username': 'elastalert', + 'channel': '', + 'icon_emoji': ':ghost:' + } + mock_post_request.assert_called_once_with( + rule['mattermost_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + verify=True, + proxies=None + ) + + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_mattermost_opensearch_discover_color(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_text_type': 'alert_text_only', + 'mattermost_attach_opensearch_discover_url': True, + 'mattermost_opensearch_discover_color': 'blue', + 'mattermost_webhook_url': 'http://please.dontgohere.mattermost', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = MattermostAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'opensearch_discover_url': 'http://localhost:5601/app/discover#/' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'attachments': [ + { + 'fallback': 'Test Rule: ', + 'color': 'danger', + 'title': 'Test Rule', + 'pretext': '', + 'fields': [], + 'text': 'Test Rule\n\n' + }, + { + 'color': 'blue', + 'title': 'Discover in opensearch', + 'title_link': 'http://localhost:5601/app/discover#/' + } + ], + 'username': 'elastalert', + 'channel': '', + 'icon_emoji': ':ghost:' + } + mock_post_request.assert_called_once_with( + rule['mattermost_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + verify=True, + proxies=None + ) + + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data def test_mattermost_username_override(): rule = { From 9ffb9cede4b57eabef3f8cf7bfd358ff96603142 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:53:23 +0100 Subject: [PATCH 23/65] Update rocketchat.py --- elastalert/alerters/rocketchat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerters/rocketchat.py b/elastalert/alerters/rocketchat.py index 0ba781ae..6875fde8 100644 --- a/elastalert/alerters/rocketchat.py +++ b/elastalert/alerters/rocketchat.py @@ -33,7 +33,7 @@ def __init__(self, rule): self.rocket_chat_kibana_discover_title = self.rule.get('rocket_chat_kibana_discover_title', 'Discover in Kibana') self.rocket_chat_attach_opensearch_discover_url = self.rule.get('rocket_chat_attach_opensearch_discover_url', False) self.rocket_chat_opensearch_discover_color = self.rule.get('rocket_chat_opensearch_discover_color', '#ec4b98') - self.rocket_chat_opensearch_discover_title = self.rule.get('rocket_chat_opensearch_discover_title', 'Discover in Kibana') + self.rocket_chat_opensearch_discover_title = self.rule.get('rocket_chat_opensearch_discover_title', 'Discover in opensearch') self.rocket_chat_ignore_ssl_errors = self.rule.get('rocket_chat_ignore_ssl_errors', False) self.rocket_chat_timeout = self.rule.get('rocket_chat_timeout', 10) self.rocket_chat_ca_certs = self.rule.get('rocket_chat_ca_certs') From fb18f377148851469b574f0ceebcecf205d3216d Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:54:27 +0100 Subject: [PATCH 24/65] Update rocketchat_test.py --- tests/alerters/rocketchat_test.py | 206 ++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/tests/alerters/rocketchat_test.py b/tests/alerters/rocketchat_test.py index 67630562..d8dc470d 100644 --- a/tests/alerters/rocketchat_test.py +++ b/tests/alerters/rocketchat_test.py @@ -861,6 +861,212 @@ def test_rocket_chat_kibana_discover_color(): assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) +def test_rocket_chat_attach_opensearch_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_subject': 'Cool subject', + 'alert': [], + 'rocket_chat_webhook_url': 'http://please.dontgohere.rocketchat', + 'rocket_chat_attach_opensearch_discover_url': True + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = RocketChatAlerter(rule) + match = { + '@timestamp': '2021-01-01T00:00:00', + 'somefield': 'foobarbaz', + 'opensearch_discover_url': 'http://localhost:5601/app/discover#/' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert2', + 'channel': '', + 'emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Cool subject', + 'text': BasicMatchString(rule, match).__str__(), + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Discover in opensearch', + 'title_link': 'http://localhost:5601/app/discover#/' + } + ], + 'text': '' + } + + mock_post_request.assert_called_once_with( + rule['rocket_chat_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_rocket_chat_attach_opensearch_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_subject': 'Cool subject', + 'alert': [], + 'rocket_chat_webhook_url': 'http://please.dontgohere.rocketchat', + 'rocket_chat_attach_opensearch_discover_url': True + } + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = RocketChatAlerter(rule) + match = { + 'somefield': 'foobarbaz', + '@timestamp': '2021-01-01T00:00:00' + } + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert2', + 'channel': '', + 'emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Cool subject', + 'text': BasicMatchString(rule, match).__str__(), + 'fields': [] + } + ], + 'text': '' + } + + mock_post_request.assert_called_once_with( + rule['rocket_chat_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_rocket_chat_opensearch_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_subject': 'Cool subject', + 'alert': [], + 'rocket_chat_webhook_url': 'http://please.dontgohere.rocketchat', + 'rocket_chat_attach_opensearch_discover_url': True, + 'rocket_chat_opensearch_discover_title': 'Click to discover in opensearch' + } + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = RocketChatAlerter(rule) + match = { + 'somefield': 'foobarbaz', + '@timestamp': '2021-01-01T00:00:00', + 'opensearch_discover_url': 'http://localhost:5601/app/discover#/' + } + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert2', + 'channel': '', + 'emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Cool subject', + 'text': BasicMatchString(rule, match).__str__(), + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Click to discover in opensearch', + 'title_link': 'http://localhost:5601/app/discover#/' + } + ], + 'text': '' + } + + mock_post_request.assert_called_once_with( + rule['rocket_chat_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_rocket_chat_opensearch_discover_color(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'alert_text_type': 'alert_text_only', + 'alert': [], + 'rocket_chat_webhook_url': 'http://please.dontgohere.rocket_chat', + 'rocket_chat_attach_opensearch_discover_url': True, + 'rocket_chat_opensearch_discover_color': 'blue' + } + + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = RocketChatAlerter(rule) + match = { + 'somefield': 'foobarbaz', + '@timestamp': '2021-01-01T00:00:00', + 'opensearch_discover_url': 'http://localhost:5601/app/discover#/' + } + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert2', + 'channel': '', + 'emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'fields': [] + }, + { + 'color': 'blue', + 'title': 'Discover in opensearch', + 'title_link': 'http://localhost:5601/app/discover#/' + } + ], + 'text': '' + } + + mock_post_request.assert_called_once_with( + rule['rocket_chat_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + timeout=10, + verify=True + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + @pytest.mark.parametrize('ca_certs, ignore_ssl_errors, excpet_verify', [ ('', '', True), ('', True, False), From a05b8e59fd969c16c3f156aeb01d3e8a2c0ebe9d Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:56:22 +0100 Subject: [PATCH 25/65] Update slack_test.py add opensearch related test --- tests/alerters/slack_test.py | 198 +++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/tests/alerters/slack_test.py b/tests/alerters/slack_test.py index c0ec291a..bd0eaa4a 100644 --- a/tests/alerters/slack_test.py +++ b/tests/alerters/slack_test.py @@ -455,6 +455,204 @@ def test_slack_kibana_discover_color(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data +def test_slack_attach_opensearch_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_opensearch_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'opensearch_discover_url': 'http://opensearch#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Discover in Opensearch', + 'title_link': 'http://opensearch#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_attach_opensearch_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_opensearch_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_opensearch_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_opensearch_discover_url': True, + 'slack_opensearch_discover_title': 'Click to Discover in Opensearch', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'opensearch_discover_url': 'http://opensearch#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Click to Discover in Opensearch', + 'title_link': 'http://opensearch#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_opensearch_discover_color(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_opensearch_discover_url': True, + 'slack_opensearch_discover_color': 'blue', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'opensearch_discover_url': 'http://opensearch#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': 'blue', + 'title': 'Discover in Opensearch', + 'title_link': 'http://opensearch#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data def test_slack_proxy(): rule = { From f91e6ab2f55f1a24ba2fce9c332f271e55443645 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:58:18 +0100 Subject: [PATCH 26/65] Update loaders_test.py add opensearch related test --- tests/loaders_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/loaders_test.py b/tests/loaders_test.py index ac389fb6..5cadd6a1 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -455,6 +455,24 @@ def test_kibana_discover_to_timedelta(): assert isinstance(test_rule_copy['kibana_discover_to_timedelta'], datetime.timedelta) assert test_rule_copy['kibana_discover_to_timedelta'] == datetime.timedelta(minutes=2) +def test_opensearch_discover_from_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['opensearch_discover_from_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['opensearch_discover_from_timedelta'], datetime.timedelta) + assert test_rule_copy['opensearch_discover_from_timedelta'] == datetime.timedelta(minutes=2) + + +def test_opensearch_discover_to_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['opensearch_discover_to_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['opensearch_discover_to_timedelta'], datetime.timedelta) + assert test_rule_copy['opensearch_discover_to_timedelta'] == datetime.timedelta(minutes=2) def test_custom_timestamp_type_timestamp_format(): test_config_copy = copy.deepcopy(test_config) From 423bd86aefb2a7dfbe46840caa543186331d127a Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:02:04 +0100 Subject: [PATCH 27/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ed35adb1..88f60645 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -88,6 +88,22 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana_discover_to_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | +| ``shorten_kibana_discover_url`` (boolean, default False) | | ++--------------------------------------------------------------+ | +| ``opensearch_discover_app_url`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``opensearch_discover_version`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``opensearch_discover_index_pattern_id`` (string, no default)| | ++--------------------------------------------------------------+ | +| ``opensearch_discover_security_tenant`` (string, no default)| | ++--------------------------------------------------------------+ | +| ``opensearch_discover_columns`` (list of strs, default _source)| | ++--------------------------------------------------------------+ | +| ``opensearch_discover_from_timedelta`` (time, default: 10 min)| | ++--------------------------------------------------------------+ | +| ``opensearch_discover_to_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | | ``use_local_time`` (boolean, default True) | | +--------------------------------------------------------------+ | | ``realert`` (time, default: 1 min) | | From ac135b032661ff18442f6741e235c0f3772cedac Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:02:33 +0100 Subject: [PATCH 28/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 88f60645..4c07fb45 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -98,9 +98,9 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``opensearch_discover_security_tenant`` (string, no default)| | +--------------------------------------------------------------+ | -| ``opensearch_discover_columns`` (list of strs, default _source)| | +| ``opensearch_discover_columns`` (list of strs, default _source)| | +--------------------------------------------------------------+ | -| ``opensearch_discover_from_timedelta`` (time, default: 10 min)| | +| ``opensearch_discover_from_timedelta`` (time, default: 10 min)| | +--------------------------------------------------------------+ | | ``opensearch_discover_to_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | From 66b1147e14a29aa112bba179f7a396d699fcc062 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:03:28 +0100 Subject: [PATCH 29/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 4c07fb45..34acb47d 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -98,9 +98,9 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``opensearch_discover_security_tenant`` (string, no default)| | +--------------------------------------------------------------+ | -| ``opensearch_discover_columns`` (list of strs, default _source)| | +|``opensearch_discover_columns``(list of strs, default _source)| | +--------------------------------------------------------------+ | -| ``opensearch_discover_from_timedelta`` (time, default: 10 min)| | +| ``opensearch_discover_from_timedelta``(time, default: 10 min)| | +--------------------------------------------------------------+ | | ``opensearch_discover_to_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | From e283036cdcea130f49fac2f01fd9481a2290223c Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:05:00 +0100 Subject: [PATCH 30/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 34acb47d..3070375c 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -98,9 +98,9 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``opensearch_discover_security_tenant`` (string, no default)| | +--------------------------------------------------------------+ | -|``opensearch_discover_columns``(list of strs, default _source)| | +|``opensearch_discover_columns`` (list of strs,default _source)| | +--------------------------------------------------------------+ | -| ``opensearch_discover_from_timedelta``(time, default: 10 min)| | +| ``opensearch_discover_from_timedelta`` (time,default: 10 min)| | +--------------------------------------------------------------+ | | ``opensearch_discover_to_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | From 99f503511ec36dbd553a9a881172b4b1c3dbd1ce Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:15:45 +0100 Subject: [PATCH 31/65] Update ruletypes.rst add opensearch doc --- docs/source/ruletypes.rst | 116 ++++++++++++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 17 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 3070375c..12dd803f 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -643,7 +643,17 @@ kibana_url ``kibana_url``: The base url of the Kibana application. If not specified, a URL will be constructed using ``es_host`` and ``es_port``. -This value will be used if ``generate_kibana_discover_url`` or ``generate_opensearch_discover_url`` is true and ``kibana_discover_app_url`` is a relative path +This value will be used if ``generate_kibana_discover_url`` is true and ``kibana_discover_app_url`` is a relative path + +(Optional, string, default ``http://:/_plugin/kibana/``) + +opensearch_url +^^^^^^^^^^ + +``opensearch_url``: The base url of the opensearch application. If not specified, a URL will be constructed using ``es_host`` +and ``es_port``. + +This value will be used if ``generate_opensearch_discover_url`` is true and ``opensearch_discover_app_url`` is a relative path (Optional, string, default ``http://:/_plugin/kibana/``) @@ -704,34 +714,34 @@ Example kibana_url + kibana_discover_app_url usage:: generate_opensearch_discover_url ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``generate_opensearch_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Opensearch Discover application. +``generate_opensearch_discover_url``: Enables the generation of the ``opensearch_discover_url`` variable for the Opensearch Discover application. This setting requires the following settings are also configured: -- ``kibana_discover_app_url`` -- ``kibana_discover_version`` -- ``kibana_discover_index_pattern_id`` +- ``opensearch_discover_app_url`` +- ``opensearch_discover_version`` +- ``opensearch_discover_index_pattern_id`` ``generate_opensearch_discover_url: true`` -Example kibana_discover_app_url only usage for opensearch:: +Example opensearch_discover_app_url only usage for opensearch:: generate_opensearch_discover_url: true - kibana_discover_app_url: "http://localhost:5601/app/data-explorer/discover?security_tenant=Admin#" - kibana_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" - kibana_discover_version: "2.11" + opensearch_discover_app_url: "http://localhost:5601/app/data-explorer/discover?security_tenant=Admin#" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "2.11" alert_text: '{}' - alert_text_args: [ kibana_discover_url ] + alert_text_args: [ opensearch_discover_url ] alert_text_type: alert_text_only -Example kibana_url + kibana_discover_app_url usage for opensearch:: +Example opensearch_url + opensearch_discover_app_url usage for opensearch:: - generate_kibana_discover_url: true - kibana_url: "http://localhost:5601/" - kibana_discover_app_url: "app/data-explorer/discover?security_tenant=Admin#" - kibana_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" - kibana_discover_version: "2.11" + generate_opensearch_discover_url: true + opensearch_url: "http://localhost:5601/" + opensearch_discover_app_url: "app/data-explorer/discover?security_tenant=Admin#" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "2.11" alert_text: '{}' - alert_text_args: [ kibana_discover_url ] + alert_text_args: [ opensearch_discover_url ] alert_text_type: alert_text_only shorten_kibana_discover_url @@ -760,6 +770,17 @@ This value should be relative to the base kibana url defined by ``kibana_url`` a (Optional, string, no default) +opensearch_discover_app_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_app_url``: The url of the opensearch Discover application used to generate the ``opensearch_discover_url`` variable. +This value can use `$VAR` and `${VAR}` references to expand environment variables. +This value should be relative to the base opensearch url defined by ``opensearch_url`` and will vary depending on your installation. + +``opensearch_discover_app_url: app/discover#/`` + +(Optional, string, no default) + kibana_discover_security_tenant ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -780,6 +801,18 @@ The currently supported versions of Kibana Discover are: ``kibana_discover_version: '7.15'`` +opensearch_discover_version +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_version``: Specifies the version of the opensearch Discover application. + +The currently supported versions of opensearch Discover are: + +- `2.11` + +``opensearch_discover_version: '2.11'`` + + kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -804,6 +837,29 @@ You can modify an index pattern's id by exporting the saved object, modifying th ``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` +opensearch_discover_index_pattern_id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_index_pattern_id``: The id of the index pattern to link to in the opensearch Discover application. +These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object. + + +Example export of an index pattern's saved object: + +.. code-block:: text + + [ + { + "_id": "4e97d188-8a45-4418-8a37-07ed69b4d34c", + "_type": "index-pattern", + "_source": { ... } + } + ] + +You can modify an index pattern's id by exporting the saved object, modifying the ``_id`` field, and re-importing. + +``opensearch_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` + kibana_discover_columns ^^^^^^^^^^^^^^^^^^^^^^^ @@ -812,6 +868,15 @@ Defaults to the ``_source`` column. ``kibana_discover_columns: [ timestamp, message ]`` +opensearch_discover_columns +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_columns``: The columns to display in the generated opensearch Discover application link. +Defaults to the ``_source`` column. + +``opensearch_discover_columns: [ timestamp, message ]`` + + kibana_discover_from_timedelta ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -820,6 +885,15 @@ The `from` time is calculated by subtracting this timedelta from the event time. ``kibana_discover_from_timedelta: minutes: 2`` + +opensearch_discover_from_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_from_timedelta``: The offset to the `from` time of the opensearch Discover link's time range. +The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. + +``opensearch_discover_from_timedelta: minutes: 2`` + kibana_discover_to_timedelta ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -828,6 +902,14 @@ The `to` time is calculated by adding this timedelta to the event time. Default ``kibana_discover_to_timedelta: minutes: 2`` +opensearch_discover_to_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_to_timedelta``: The offset to the `to` time of the opensearch Discover link's time range. +The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. + +``opensearch_discover_to_timedelta: minutes: 2`` + use_local_time ^^^^^^^^^^^^^^ From 89bdaa4e4909ce761eb32d4fbb8d92798a9562f1 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:47:42 +0100 Subject: [PATCH 32/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 12dd803f..aabd3fb0 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -648,7 +648,7 @@ This value will be used if ``generate_kibana_discover_url`` is true and ``kibana (Optional, string, default ``http://:/_plugin/kibana/``) opensearch_url -^^^^^^^^^^ +^^^^^^^^^^^^^^ ``opensearch_url``: The base url of the opensearch application. If not specified, a URL will be constructed using ``es_host`` and ``es_port``. From 9701674959b6c853b08227c463baf05e11e3df3d Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:49:20 +0100 Subject: [PATCH 33/65] Update loaders.py --- elastalert/loaders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index a94e9001..fc416088 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -332,9 +332,9 @@ def load_options(self, rule, conf, filename, args=None): if 'kibana_discover_to_timedelta' in rule: rule['kibana_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) if 'opensearch_discover_from_timedelta' in rule: - rule['opensearch_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta']) + rule['opensearch_discover_from_timedelta'] = datetime.timedelta(**rule['opensearch_discover_from_timedelta']) if 'opensearch_discover_to_timedelta' in rule: - rule['opensearch_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) + rule['opensearch_discover_to_timedelta'] = datetime.timedelta(**rule['opensearch_discover_to_timedelta']) except (KeyError, TypeError) as e: raise EAException('Invalid time format used: %s' % e) From a43ef0a57b706e7d1a61ccec7ad746055452f8e7 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:53:48 +0100 Subject: [PATCH 34/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index aabd3fb0..476e8013 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -887,7 +887,7 @@ The `from` time is calculated by subtracting this timedelta from the event time. opensearch_discover_from_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``opensearch_discover_from_timedelta``: The offset to the `from` time of the opensearch Discover link's time range. The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. @@ -903,7 +903,7 @@ The `to` time is calculated by adding this timedelta to the event time. Default ``kibana_discover_to_timedelta: minutes: 2`` opensearch_discover_to_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``opensearch_discover_to_timedelta``: The offset to the `to` time of the opensearch Discover link's time range. The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. From 6317f953a53970aa5373f3d643b05528467f7e9d Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:00:11 +0100 Subject: [PATCH 35/65] Update opensearch_discover.py --- elastalert/opensearch_discover.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/opensearch_discover.py b/elastalert/opensearch_discover.py index e35b0f25..469f1ef5 100644 --- a/elastalert/opensearch_discover.py +++ b/elastalert/opensearch_discover.py @@ -21,10 +21,10 @@ def generate_opensearch_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' - discover_app_url = rule.get('kibana_discover_app_url') + discover_app_url = rule.get('opensearch_discover_app_url') if not discover_app_url: elastalert_logger.warning( - 'Missing kibana_discover_app_url for rule %s' % ( + 'Missing opensearch_discover_app_url for rule %s' % ( rule.get('name', '') ) ) From eb3979d6d31ae5d653be864638b796cf7fe17484 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:08:23 +0100 Subject: [PATCH 36/65] Update mattermost.py --- elastalert/alerters/mattermost.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerters/mattermost.py b/elastalert/alerters/mattermost.py index 545e43ad..ec22f2ab 100644 --- a/elastalert/alerters/mattermost.py +++ b/elastalert/alerters/mattermost.py @@ -149,7 +149,7 @@ def alert(self, matches): if self.mattermost_attach_opensearch_discover_url: opensearch_discover_url = lookup_es_key(matches[0], 'opensearch_discover_url') - if kibana_discover_url: + if opensearch_discover_url: payload['attachments'].append({ 'color': self.mattermost_opensearch_discover_color, 'title': self.mattermost_opensearch_discover_title, From 6291cc8a19538851878da0a696aa77ebdcf7a00e Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:10:41 +0100 Subject: [PATCH 37/65] Update slack.py --- elastalert/alerters/slack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/alerters/slack.py b/elastalert/alerters/slack.py index 424fe6ac..9a8cb323 100644 --- a/elastalert/alerters/slack.py +++ b/elastalert/alerters/slack.py @@ -36,9 +36,9 @@ def __init__(self, rule): self.slack_attach_kibana_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) self.slack_kibana_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') self.slack_kibana_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Kibana') - self.slack_attach_opensearch_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) - self.slack_opensearch_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') - self.slack_opensearch_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Opensearch') + self.slack_attach_opensearch_discover_url = self.rule.get('slack_attach_opensearch_discover_url', False) + self.slack_opensearch_discover_color = self.rule.get('slack_opensearch_discover_color', '#ec4b98') + self.slack_opensearch_discover_title = self.rule.get('slack_opensearch_discover_title', 'Discover in Opensearch') self.slack_footer = self.rule.get('slack_footer', '') self.slack_footer_icon = self.rule.get('slack_footer_icon', '') self.slack_image_url = self.rule.get('slack_image_url', '') From 9b20e4ec45edc263d12b4c4856b036b8c92ba180 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:16:13 +0100 Subject: [PATCH 38/65] Update teams_test.py --- tests/alerters/teams_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/alerters/teams_test.py b/tests/alerters/teams_test.py index 5cdb1c7d..abdd43ca 100644 --- a/tests/alerters/teams_test.py +++ b/tests/alerters/teams_test.py @@ -481,7 +481,7 @@ def test_ms_teams_opensearch_discover_title(): 'name': 'Test Rule', 'type': 'any', 'ms_teams_attach_opensearch_discover_url': True, - 'ms_teams_opensearch_discover_title': 'Click to discover in Kibana', + 'ms_teams_opensearch_discover_title': 'Click to discover in opensearch', 'ms_teams_webhook_url': 'http://test.webhook.url', 'ms_teams_alert_summary': 'Alert from ElastAlert', 'alert': [], From 703b60183307a343b0d0885d69e1cdc335940dd3 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:04:11 +0100 Subject: [PATCH 39/65] Update opensearch_discover_test.py --- tests/opensearch_discover_test.py | 63 ++++++++++++++++++------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/tests/opensearch_discover_test.py b/tests/opensearch_discover_test.py index 9ff60405..2c8137be 100644 --- a/tests/opensearch_discover_test.py +++ b/tests/opensearch_discover_test.py @@ -20,28 +20,27 @@ def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_ 'timestamp': '2019-09-01T00:30:00Z' } ) - expectedUrl = ( 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start + + '?_g=%28' # global start + 'filters%3A%21%28%29%2C' + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' # time start + + 'time%3A%28' # time start + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' # time end + + '%29' # time end + '%29' # global end - + '&_a=%28' # app start + + '&_a=%28' # app start + 'discover%3A%28columns%3A%21%28_source%29%2C' + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + 'view%3Adiscover%29%29' # app end - + '&_q=%28filters%3A%21%28%29%2C'#query and filter start - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' #query and filter start + + '&_q=%28filters%3A%21%28%29%2C' # query and filter start + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # query and filter start ) - assert url == expectedUrl + def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_url_without_http(): url = generate_opensearch_discover_url( rule={ @@ -54,7 +53,6 @@ def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_ 'timestamp': '2021-10-08T00:30:00Z' } ) - expectedUrl = ( 'app/discover#/' + '?_g=%28' @@ -71,9 +69,9 @@ def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_ + 'metadata%3A%28indexPattern%3A%27620ad0e6-43df-4557-bda2-384960fa9086%27%2Cview%3Adiscover%29%29' + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' ) - assert url == expectedUrl + def test_generate_opensearch_discover_url_with_missing_opensearch_discover_version(): url = generate_opensearch_discover_url( rule={ @@ -133,6 +131,7 @@ def test_generate_opensearch_discover_url_with_invalid_opensearch_version(): ) assert url is None + def test_generate_opensearch_discover_url_with_from_timedelta_and_timeframe(): url = generate_opensearch_discover_url( rule={ @@ -147,7 +146,6 @@ def test_generate_opensearch_discover_url_with_from_timedelta_and_timeframe(): 'timestamp': '2019-09-01T04:00:00Z' } ) - expectedUrl = ( 'http://opensearch:5601/#/discover' + '?_g=%28' @@ -165,6 +163,7 @@ def test_generate_opensearch_discover_url_with_from_timedelta_and_timeframe(): ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_to_timedelta_and_timeframe(): url = generate_opensearch_discover_url( rule={ @@ -196,6 +195,7 @@ def test_generate_opensearch_discover_url_with_to_timedelta_and_timeframe(): ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_custom_columns(): url = generate_opensearch_discover_url( rule={ @@ -209,7 +209,6 @@ def test_generate_opensearch_discover_url_with_custom_columns(): 'timestamp': '2019-09-01T00:30:00Z' } ) - expectedUrl = ( 'http://opensearch:5601/#/discover' + '?_g=%28' @@ -243,9 +242,6 @@ def test_generate_opensearch_discover_url_with_single_filter(): 'timestamp': '2019-09-01T00:30:00Z' } ) - - - expectedUrl = ( 'http://opensearch:5601/#/discover' + '?_g=%28' @@ -264,11 +260,14 @@ def test_generate_opensearch_discover_url_with_single_filter(): + '&_q=%28' + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29' + '%2Cmeta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29' - + '%2Cquery%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22 + + 'must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29' + + '%2Cquery%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_multiple_filters(): url = generate_opensearch_discover_url( rule={ @@ -301,12 +300,15 @@ def test_generate_opensearch_discover_url_with_multiple_filters(): + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28' + 'alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' - + 'bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22app%22%3A%22test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' - + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' + + 'bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22app%22%3A%22' + + 'test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28' + + 'app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_int_query_key(): url = generate_opensearch_discover_url( rule={ @@ -379,7 +381,8 @@ def test_generate_opensearch_discover_url_with_str_query_key(): + '%28filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + 'key%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2C' - + 'type%3Aphrase%2Cvalue%3Aok%29%2Cquery%3A%28match%3A%28geo.dest%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'type%3Aphrase%2Cvalue%3Aok%29%2Cquery%3A%28match%3A%28' + + 'geo.dest%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end ) assert url == expectedUrl @@ -414,11 +417,13 @@ def test_generate_opensearch_discover_url_with_missing_query_key_value(): + '_q=%28' + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cexists%3A%28field%3Astatus%29%2C' + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Astatus%2Cnegate%3A%21t%2Ctype%3Aexists%2Cvalue%3Aexists%2Cview%3Adiscover%29%29%29%2C' + + 'key%3Astatus%2Cnegate%3A%21t%2Ctype%3Aexists%2C' + + 'value%3Aexists%2Cview%3Adiscover%29%29%29%2C' + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end ) assert url == expectedUrl - + + def test_generate_opensearch_discover_url_with_compound_query_key(): url = generate_opensearch_discover_url( rule={ @@ -459,10 +464,12 @@ def test_generate_opensearch_discover_url_with_compound_query_key(): + 'index%3A%27logs-%2A%27%2Ckey%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28' + 'query%3AUS%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3AUS%29%2C' + 'query%3A%28match%3A%28geo.dest%3A%28' - + 'query%3AUS%2Ctype%3Aphrase%29%29%29%29%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + + 'query%3AUS%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_filter_and_query_key(): url = generate_opensearch_discover_url( rule={ @@ -505,6 +512,7 @@ def test_generate_opensearch_discover_url_with_filter_and_query_key(): ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_querystring_filter_and_query_key(): url = generate_opensearch_discover_url( rule={ @@ -538,10 +546,13 @@ def test_generate_opensearch_discover_url_with_querystring_filter_and_query_key( + '&_q=%28' + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3A' - + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22must%22%3A%5B%7B%22query_string%22%3A%7B%22' + + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22' + + 'must%22%3A%5B%7B%22query_string%22%3A%7B%22' + 'query%22%3A%22hello%20world%22%7D%7D%5D%7D%7D%27%29%2C' - + 'query%3A%28bool%3A%28must%3A%21%28%28query_string%3A%28query%3A%27hello%20world%27%29%29%29%29%29%29%2C%28%27%24' - + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Astatus%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3A%28bool%3A%28must%3A%21%28%28query_string%3A%28query%3A%27hello%20' + + 'world%27%29%29%29%29%29%29%2C%28%27%24' + + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2C' + + 'disabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Astatus%2Cnegate%3A%21f%2Cparams%3A%28' + 'query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end From f30d45acbc42f5264dba78dc65bf3a0f00b69bf2 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:05:45 +0100 Subject: [PATCH 40/65] Update mattermost_test.py --- tests/alerters/mattermost_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/alerters/mattermost_test.py b/tests/alerters/mattermost_test.py index 54d0866c..44afc8d1 100644 --- a/tests/alerters/mattermost_test.py +++ b/tests/alerters/mattermost_test.py @@ -1142,6 +1142,7 @@ def test_mattermost_kibana_discover_color(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data + def test_mattermost_attach_opensearch_discover_url_when_generated(): rule = { 'name': 'Test Rule', @@ -1341,6 +1342,7 @@ def test_mattermost_opensearch_discover_color(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data + def test_mattermost_username_override(): rule = { 'name': 'Test Mattermost Rule', From 93cdc69171d9124d98733ac5078a54e75a8bc33d Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:06:24 +0100 Subject: [PATCH 41/65] Update teams_test.py --- tests/alerters/teams_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/alerters/teams_test.py b/tests/alerters/teams_test.py index abdd43ca..70dc41ae 100644 --- a/tests/alerters/teams_test.py +++ b/tests/alerters/teams_test.py @@ -389,6 +389,7 @@ def test_ms_teams_kibana_discover_title(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data + def test_ms_teams_attach_opensearch_discover_url_when_generated(): rule = { 'name': 'Test Rule', @@ -525,6 +526,8 @@ def test_ms_teams_opensearch_discover_title(): ) actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data + + def test_ms_teams_alert_facts(): rule = { 'name': 'Test Rule', From 6dfe30858da99212d39d5301076747e3c1d9044c Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:07:02 +0100 Subject: [PATCH 42/65] Update slack_test.py --- tests/alerters/slack_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/alerters/slack_test.py b/tests/alerters/slack_test.py index bd0eaa4a..e6e54bf9 100644 --- a/tests/alerters/slack_test.py +++ b/tests/alerters/slack_test.py @@ -455,6 +455,7 @@ def test_slack_kibana_discover_color(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data + def test_slack_attach_opensearch_discover_url_when_generated(): rule = { 'name': 'Test Rule', From 342650bd6b480c4bf76ef9018cf9b0112242a2e6 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:07:56 +0100 Subject: [PATCH 43/65] Update loaders_test.py --- tests/loaders_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/loaders_test.py b/tests/loaders_test.py index 5cadd6a1..3424cfb4 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -455,6 +455,7 @@ def test_kibana_discover_to_timedelta(): assert isinstance(test_rule_copy['kibana_discover_to_timedelta'], datetime.timedelta) assert test_rule_copy['kibana_discover_to_timedelta'] == datetime.timedelta(minutes=2) + def test_opensearch_discover_from_timedelta(): test_config_copy = copy.deepcopy(test_config) rules_loader = FileRulesLoader(test_config_copy) @@ -474,6 +475,7 @@ def test_opensearch_discover_to_timedelta(): assert isinstance(test_rule_copy['opensearch_discover_to_timedelta'], datetime.timedelta) assert test_rule_copy['opensearch_discover_to_timedelta'] == datetime.timedelta(minutes=2) + def test_custom_timestamp_type_timestamp_format(): test_config_copy = copy.deepcopy(test_config) rules_loader = FileRulesLoader(test_config_copy) From 2f9b66c5f9ceadf66335cf3114d9b1cd2660b460 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:13:10 +0100 Subject: [PATCH 44/65] Update opensearch_discover_test.py --- tests/opensearch_discover_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/opensearch_discover_test.py b/tests/opensearch_discover_test.py index 2c8137be..2eaca308 100644 --- a/tests/opensearch_discover_test.py +++ b/tests/opensearch_discover_test.py @@ -260,7 +260,7 @@ def test_generate_opensearch_discover_url_with_single_filter(): + '&_q=%28' + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29' + '%2Cmeta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22 + + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22' + 'must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29' + '%2Cquery%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end From 74a016a96e63586fa855d99d5440ed2131a02fc2 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:18:27 +0100 Subject: [PATCH 45/65] Update opensearch_discover_test.py --- tests/opensearch_discover_test.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/opensearch_discover_test.py b/tests/opensearch_discover_test.py index 2eaca308..d0a04f0b 100644 --- a/tests/opensearch_discover_test.py +++ b/tests/opensearch_discover_test.py @@ -347,6 +347,7 @@ def test_generate_opensearch_discover_url_with_int_query_key(): ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_str_query_key(): url = generate_opensearch_discover_url( rule={ @@ -387,6 +388,7 @@ def test_generate_opensearch_discover_url_with_str_query_key(): ) assert url == expectedUrl + def test_generate_opensearch_discover_url_with_missing_query_key_value(): url = generate_opensearch_discover_url( rule={ @@ -502,10 +504,13 @@ def test_generate_opensearch_discover_url_with_filter_and_query_key(): + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + '&_q=%28' + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' - + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' - + 'bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3A' + + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' + + 'bool%22%3A%7B%22must%22%3A%5B%7B%22' + + 'term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%2C%28%27%24' - + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Astatus' + + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C + + 'index%3A%27logs-%2A%27%2Ckey%3Astatus' + '%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end From f1bfbe48cb70337a553ae9ed23f57181b8d734d8 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:18:58 +0100 Subject: [PATCH 46/65] Update slack_test.py --- tests/alerters/slack_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/alerters/slack_test.py b/tests/alerters/slack_test.py index e6e54bf9..7405f7c5 100644 --- a/tests/alerters/slack_test.py +++ b/tests/alerters/slack_test.py @@ -655,6 +655,7 @@ def test_slack_opensearch_discover_color(): actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data == actual_data + def test_slack_proxy(): rule = { 'name': 'Test Rule', From 687f3553c406cadd36f62e7785004d4fcbe298d8 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:23:36 +0100 Subject: [PATCH 47/65] Update opensearch_discover_test.py --- tests/opensearch_discover_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/opensearch_discover_test.py b/tests/opensearch_discover_test.py index d0a04f0b..959c102b 100644 --- a/tests/opensearch_discover_test.py +++ b/tests/opensearch_discover_test.py @@ -509,7 +509,7 @@ def test_generate_opensearch_discover_url_with_filter_and_query_key(): + 'bool%22%3A%7B%22must%22%3A%5B%7B%22' + 'term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%2C%28%27%24' - + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C + + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C' + 'index%3A%27logs-%2A%27%2Ckey%3Astatus' + '%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' From f06ef4bf21d58a3a09c16c911ecb05b3a4920c93 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:39:00 +0100 Subject: [PATCH 48/65] Update elastalert.py --- elastalert/elastalert.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 7a736b26..e7fbb492 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -35,7 +35,7 @@ from elastalert.kibana_discover import generate_kibana_discover_url from elastalert.kibana_external_url_formatter import create_kibana_external_url_formatter from elastalert.opensearch_discover import generate_opensearch_discover_url -from elastalert.kibana_external_url_formatter import create_opensearch_external_url_formatter +from elastalert.opensearch_external_url_formatter import create_opensearch_external_url_formatter from elastalert.prometheus_wrapper import PrometheusWrapper from elastalert.ruletypes import FlatlineRule from elastalert.util import (add_keyword_postfix, cronite_datetime_to_timestamp, dt_to_ts, dt_to_unix, EAException, @@ -1451,13 +1451,11 @@ def get_kibana_discover_external_url_formatter(self, rule): def get_opensearch_discover_external_url_formatter(self, rule): - """ Gets or create the external url formatter for kibana discover links """ - key = '__kibana_discover_external_url_formatter__' + """ Gets or create the external url formatter for Opensearch discover links """ + key = '__opensearch_discover_external_url_formatter__' formatter = rule.get(key) if formatter is None: - shorten = rule.get('shorten_kibana_discover_url') - security_tenant = rule.get('kibana_discover_security_tenant') - formatter = create_opensearch_external_url_formatter(rule, shorten, security_tenant) + formatter = create_opensearch_external_url_formatter(rule) rule[key] = formatter return formatter From 42762f968b98032c4dd7477af91581f98d127467 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:39:16 +0100 Subject: [PATCH 49/65] Create opensearch_external_url_formatter.py --- .../opensearch_external_url_formatter.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 elastalert/opensearch_external_url_formatter.py diff --git a/elastalert/opensearch_external_url_formatter.py b/elastalert/opensearch_external_url_formatter.py new file mode 100644 index 00000000..dfc0fbbb --- /dev/null +++ b/elastalert/opensearch_external_url_formatter.py @@ -0,0 +1,35 @@ +import boto3 +import os +from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlsplit, urlunsplit + +import requests +from requests import RequestException +from requests.auth import AuthBase, HTTPBasicAuth + +from elastalert.auth import RefeshableAWSRequestsAuth +from elastalert.util import EAException + +class OpensearchExternalUrlFormatter: + '''Interface for formatting external Opensearch urls''' + + def format(self, relative_url: str) -> str: + raise NotImplementedError() + +class AbsoluteOpensearchExternalUrlFormatter(OpensearchExternalUrlFormatter): + '''Formats absolute external Opensearch urls''' + + def __init__(self, base_url: str) -> None: + self.base_url = base_url + + def format(self, relative_url: str) -> str: + url = urljoin(self.base_url, relative_url) + return url + +def create_opensearch_external_url_formatter( + rule +) -> OpensearchExternalUrlFormatter: + '''Creates a Opensearch external url formatter''' + + base_url = rule.get('opensearch_url') + + return AbsoluteKibanaExternalUrlFormatter(base_url) From 8e0713208abc99802d8eaa0f93d515e0fc959fd0 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:41:18 +0100 Subject: [PATCH 50/65] Create opensearch_external_url_formatter_test.py Add a test file for opensearch_external_url_formatter --- .../opensearch_external_url_formatter_test.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/opensearch_external_url_formatter_test.py diff --git a/tests/opensearch_external_url_formatter_test.py b/tests/opensearch_external_url_formatter_test.py new file mode 100644 index 00000000..17600479 --- /dev/null +++ b/tests/opensearch_external_url_formatter_test.py @@ -0,0 +1,48 @@ +from typing import Any +import os +import pytest + +import requests +from requests.auth import AuthBase, HTTPBasicAuth + +from elastalert.opensearch_external_url_formatter import AbsoluteOpensearchExternalUrlFormatter +from elastalert.opensearch_external_url_formatter import OpensearchExternalUrlFormatter +from elastalert.opensearch_external_url_formatter import create_opensearch_external_url_formatter + +from elastalert.auth import RefeshableAWSRequestsAuth +from elastalert.util import EAException + +from unittest import mock + + +class AbsoluteFormatTestCase: + def __init__( + self, + base_url: str, + relative_url: str, + expected_url: str, + ) -> None: + self.base_url = base_url + self.relative_url = relative_url + self.expected_url = expected_url + + +@pytest.mark.parametrize("test_case", [ + # Relative to OpenSearch Dashboards + AbsoluteFormatTestCase( + base_url='http://opensearch.test.org/_dashboards/', + relative_url='app/dev_tools#/console', + expected_url='http://opensearch.test.org/_dashboards/app/dev_tools#/console' + ), +]) + + +def test_absolute_opensearch_external_url_formatter( + test_case: AbsoluteFormatTestCase +): + formatter = AbsoluteOpensearchExternalUrlFormatter( + base_url=test_case.base_url + ) + actualUrl = formatter.format(test_case.relative_url) + print(actualUrl) + assert actualUrl == test_case.expected_url From 5f9f553cd64823f70c773ed56b8127470af93dfd Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:45:30 +0100 Subject: [PATCH 51/65] Update opensearch_discover_test.py --- tests/opensearch_discover_test.py | 767 ++++++++---------------------- 1 file changed, 203 insertions(+), 564 deletions(-) diff --git a/tests/opensearch_discover_test.py b/tests/opensearch_discover_test.py index 959c102b..17dfd27f 100644 --- a/tests/opensearch_discover_test.py +++ b/tests/opensearch_discover_test.py @@ -1,565 +1,204 @@ # -*- coding: utf-8 -*- -from datetime import timedelta -import pytest - -from elastalert.opensearch_discover import generate_opensearch_discover_url - - -@pytest.mark.parametrize("opensearch_version", [ - '2.11' -]) -def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_url(opensearch_version): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': opensearch_version, - 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', - 'timestamp_field': 'timestamp' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2C' - + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' # time start - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' # time end - + '%29' # global end - + '&_a=%28' # app start - + 'discover%3A%28columns%3A%21%28_source%29%2C' - + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' - + 'view%3Adiscover%29%29' # app end - + '&_q=%28filters%3A%21%28%29%2C' # query and filter start - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # query and filter start - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_url_without_http(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'app/discover#/', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': '620ad0e6-43df-4557-bda2-384960fa9086', - 'timestamp_field': 'timestamp' - }, - match={ - 'timestamp': '2021-10-08T00:30:00Z' - } - ) - expectedUrl = ( - 'app/discover#/' - + '?_g=%28' - + 'filters%3A%21%28%29%2C' - + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272021-10-08T00%3A20%3A00Z%27%2C' - + 'to%3A%272021-10-08T00%3A40%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2C' - + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27620ad0e6-43df-4557-bda2-384960fa9086%27%2Cview%3Adiscover%29%29' - + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_missing_opensearch_discover_version(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_index_pattern_id': 'logs', - 'timestamp_field': 'timestamp', - 'name': 'test' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - assert url is None - - -def test_generate_opensearch_discover_url_with_missing_opensearch_discover_app_url(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_version': '8.11', - 'opensearch_discover_index_pattern_id': 'logs', - 'timestamp_field': 'timestamp', - 'name': 'test' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - assert url is None - - -def test_generate_opensearch_discover_url_with_missing_opensearch_discover_index_pattern_id(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '8.11', - 'timestamp_field': 'timestamp', - 'name': 'test' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - assert url is None - - -def test_generate_opensearch_discover_url_with_invalid_opensearch_version(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '4.5', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - assert url is None - - -def test_generate_opensearch_discover_url_with_from_timedelta_and_timeframe(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', - 'opensearch_discover_from_timedelta': timedelta(hours=1), - 'timeframe': timedelta(minutes=20), - 'timestamp_field': 'timestamp' - }, - match={ - 'timestamp': '2019-09-01T04:00:00Z' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' - + 'filters%3A%21%28%29%2C' - + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' - + 'to%3A%272019-09-01T04%3A20%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28discover%3A%28columns%3A%21%28_source%29%2C' - + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2Cview%3Adiscover%29%29' - + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_to_timedelta_and_timeframe(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', - 'opensearch_discover_to_timedelta': timedelta(hours=1), - 'timeframe': timedelta(minutes=20), - 'timestamp_field': 'timestamp' - }, - match={ - 'timestamp': '2019-09-01T04:00:00Z' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' - + 'filters%3A%21%28%29%2C' - + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C' - + 'to%3A%272019-09-01T05%3A00%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28discover%3A%28columns%3A%21%28_source%29%2C' - + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2Cview%3Adiscover%29%29' - + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_custom_columns(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'opensearch_discover_columns': ['level', 'message'], - 'timestamp_field': 'timestamp' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' - + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27%' - + '29%' - + '29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28level%2Cmessage%29%2C' - + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' - + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_single_filter(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp', - 'filter': [ - {'term': {'level': 30}} - ] - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' - + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27%' - + '29%' - + '29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2C' - + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' - + '%29' - + '%29' - + '&_q=%28' - + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29' - + '%2Cmeta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22' - + 'must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29' - + '%2Cquery%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_multiple_filters(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', - 'timestamp_field': 'timestamp', - 'filter': [ - {'term': {'app': 'test'}}, - {'term': {'level': 30}} - ] - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2Cview%3Adiscover%29%29' - + '&_q=%28' - + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28' - + 'alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' - + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' - + 'bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22app%22%3A%22' - + 'test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' - + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28' - + 'app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_int_query_key(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp', - 'query_key': 'geo.dest' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z', - 'geo.dest': 200 - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29' - + '%29' - + '&_q=%28' - + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' - + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28' - + 'query%3A200%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3A%27200%27%29%2C' - + 'query%3A%28match%3A%28geo.dest%3A%28query%3A200%2C' - + 'type%3Aphrase%29%29%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_str_query_key(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp', - 'query_key': 'geo.dest' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z', - 'geo': { - 'dest': 'ok' - } - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29%' - + '29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' - + '%29' - + '%29' - + '&_q=' - + '%28filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' - + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2C' - + 'type%3Aphrase%2Cvalue%3Aok%29%2Cquery%3A%28match%3A%28' - + 'geo.dest%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_missing_query_key_value(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp', - 'query_key': 'status' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' - + '%29' - + '%29&' - + '_q=%28' - + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cexists%3A%28field%3Astatus%29%2C' - + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Astatus%2Cnegate%3A%21t%2Ctype%3Aexists%2C' - + 'value%3Aexists%2Cview%3Adiscover%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_compound_query_key(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp', - 'compound_query_key': ['geo.src', 'geo.dest'], - 'query_key': 'geo.src,geo.dest' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z', - 'geo': { - 'src': 'CA', - 'dest': 'US' - } - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' - + '&_q=%28' - + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' - + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' - + 'key%3Ageo.src%2Cnegate%3A%21f%2Cparams%3A%28' - + 'query%3ACA%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3ACA%29%2C' - + 'query%3A%28match%3A%28geo.src%3A%28query%3ACA%2Ctype%3Aphrase%29%29%29%29%' - + '2C%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C' - + 'index%3A%27logs-%2A%27%2Ckey%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28' - + 'query%3AUS%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3AUS%29%2C' - + 'query%3A%28match%3A%28geo.dest%3A%28' - + 'query%3AUS%2Ctype%3Aphrase%29%29%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_filter_and_query_key(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp', - 'filter': [ - {'term': {'level': 30}} - ], - 'query_key': 'status' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z', - 'status': 'ok' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2C' - + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' - + '&_q=%28' - + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' - + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3A' - + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' - + 'bool%22%3A%7B%22must%22%3A%5B%7B%22' - + 'term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' - + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%2C%28%27%24' - + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C' - + 'index%3A%27logs-%2A%27%2Ckey%3Astatus' - + '%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' - + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl - - -def test_generate_opensearch_discover_url_with_querystring_filter_and_query_key(): - url = generate_opensearch_discover_url( - rule={ - 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', - 'opensearch_discover_version': '2.11', - 'opensearch_discover_index_pattern_id': 'logs-*', - 'timestamp_field': 'timestamp', - 'filter': [ - {'query': {'query_string': {'query': 'hello world'}}} - ], - 'query_key': 'status' - }, - match={ - 'timestamp': '2019-09-01T00:30:00Z', - 'status': 'ok' - } - ) - expectedUrl = ( - 'http://opensearch:5601/#/discover' - + '?_g=%28' # global start - + 'filters%3A%21%28%29%2C' - + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' - + 'time%3A%28' - + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' - + 'to%3A%272019-09-01T00%3A40%3A00Z%27' - + '%29' - + '%29' - + '&_a=%28' - + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' - + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' - + '&_q=%28' - + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' - + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3A' - + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22' - + 'must%22%3A%5B%7B%22query_string%22%3A%7B%22' - + 'query%22%3A%22hello%20world%22%7D%7D%5D%7D%7D%27%29%2C' - + 'query%3A%28bool%3A%28must%3A%21%28%28query_string%3A%28query%3A%27hello%20' - + 'world%27%29%29%29%29%29%29%2C%28%27%24' - + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2C' - + 'disabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Astatus%2Cnegate%3A%21f%2Cparams%3A%28' - + 'query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' - + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' - + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end - ) - assert url == expectedUrl +# flake8: noqa +import datetime +import logging +import json +import os.path +import prison +import urllib.parse + +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key +from .util import ts_add + +opensearch_default_timedelta = datetime.timedelta(minutes=10) + +opensearch_versions = frozenset([ + '2.11' + ]) + +def generate_opensearch_discover_url(rule, match): + ''' Creates a link for a opensearch discover app. ''' + + discover_app_url = rule.get('opensearch_discover_app_url') + if not discover_app_url: + elastalert_logger.warning( + 'Missing opensearch_discover_app_url for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + opensearch_version = rule.get('opensearch_discover_version') + if not opensearch_version: + elastalert_logger.warning( + 'Missing opensearch_discover_version for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + index = rule.get('opensearch_discover_index_pattern_id') + if not index: + elastalert_logger.warning( + 'Missing opensearch_discover_index_pattern_id for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + columns = rule.get('opensearch_discover_columns', ['_source']) + filters = rule.get('filter', []) + + if 'query_key' in rule: + query_keys = rule.get('compound_query_key', [rule['query_key']]) + else: + query_keys = [] + + timestamp = lookup_es_key(match, rule['timestamp_field']) + timeframe = rule.get('timeframe', opensearch_default_timedelta) + from_timedelta = rule.get('opensearch_discover_from_timedelta', timeframe) + from_time = ts_add(timestamp, -from_timedelta) + to_timedelta = rule.get('opensearch_discover_to_timedelta', timeframe) + to_time = ts_add(timestamp, to_timedelta) + + if opensearch_version in opensearch_versions: + globalState = opensearch_disover_global_state(from_time, to_time) + appState = opensearch_discover_app_state(index, columns, filters, query_keys, match) + appFilter = opensearch_discover_app_filter(index, columns, filters, query_keys, match) + + else: + elastalert_logger.warning( + 'Unknown opensearch discover application version %s for rule %s' % ( + opensearch_version, + rule.get('name', '') + ) + ) + return None + + urlqueryOriginal = "%s?_g=%s&_a=%s&_q=%s" % ( + os.path.expandvars(discover_app_url), + urllib.parse.quote(globalState), + urllib.parse.quote(appState), + urllib.parse.quote(appFilter) + ) + + word_to_replace = 'tobereplacedbylucenequery' + replacement_word = 'query' + max_replacements = 1 # Replace only the first occurrence + urlquery = urlqueryOriginal.replace(word_to_replace, replacement_word, max_replacements) + return urlquery + + +def opensearch_disover_global_state(from_time, to_time): + return prison.dumps( { + 'filters': [], + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'to': to_time + } + } ) + + +def opensearch_discover_app_state(index, columns, filters, query_keys, match): + return prison.dumps( { + 'discover': { + 'columns': columns, + 'isDirty': False, + 'sort': [] + }, + 'metadata': { + 'indexPattern': index, + 'view': 'discover' + } + } ) + +def opensearch_discover_app_filter(index, columns, filters, query_keys, match): + app_filters = [] + + if filters: + + new_filters = [] + for filter in filters: + if 'query' in filter: + filter = filter['query'] + new_filters.append(filter) + filters = new_filters + + bool_filter = {'bool': {'must': filters } } + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': 'filter', + 'disabled': False, + 'index': index, + 'key': 'query', + 'negate': False, + 'type': 'custom', + 'value': json.dumps(bool_filter, separators=(',', ':')) + }, + 'query': bool_filter + } ) + + for query_key in query_keys: + query_value = lookup_es_key(match, query_key) + + if query_value is None: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'exists': { + 'field': query_key + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': True, + 'type': 'exists', + 'view': 'discover', + 'value': 'exists' + } + } ) + + else: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': False, + 'params': { + 'query': query_value, + 'type': 'phrase' + }, + 'type': 'phrase', + 'value': str(query_value) + }, + 'query': { + 'match': { + query_key: { + 'query': query_value, + 'type': 'phrase' + } + } + } + } ) + + return prison.dumps( { + 'filters': app_filters, + 'tobereplacedbylucenequery': {'language': 'lucene','query': ''} + } ) From 5c82c89add224626982a0dce05f352610d226c24 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:55:09 +0100 Subject: [PATCH 52/65] Update opensearch_discover_test.py --- tests/opensearch_discover_test.py | 768 ++++++++++++++++++++++-------- 1 file changed, 565 insertions(+), 203 deletions(-) diff --git a/tests/opensearch_discover_test.py b/tests/opensearch_discover_test.py index 17dfd27f..d417cfc7 100644 --- a/tests/opensearch_discover_test.py +++ b/tests/opensearch_discover_test.py @@ -1,204 +1,566 @@ # -*- coding: utf-8 -*- -# flake8: noqa -import datetime -import logging -import json -import os.path -import prison -import urllib.parse - -from .util import EAException -from .util import elastalert_logger -from .util import lookup_es_key -from .util import ts_add - -opensearch_default_timedelta = datetime.timedelta(minutes=10) - -opensearch_versions = frozenset([ - '2.11' - ]) - -def generate_opensearch_discover_url(rule, match): - ''' Creates a link for a opensearch discover app. ''' - - discover_app_url = rule.get('opensearch_discover_app_url') - if not discover_app_url: - elastalert_logger.warning( - 'Missing opensearch_discover_app_url for rule %s' % ( - rule.get('name', '') - ) - ) - return None - - opensearch_version = rule.get('opensearch_discover_version') - if not opensearch_version: - elastalert_logger.warning( - 'Missing opensearch_discover_version for rule %s' % ( - rule.get('name', '') - ) - ) - return None - - index = rule.get('opensearch_discover_index_pattern_id') - if not index: - elastalert_logger.warning( - 'Missing opensearch_discover_index_pattern_id for rule %s' % ( - rule.get('name', '') - ) - ) - return None - - columns = rule.get('opensearch_discover_columns', ['_source']) - filters = rule.get('filter', []) - - if 'query_key' in rule: - query_keys = rule.get('compound_query_key', [rule['query_key']]) - else: - query_keys = [] - - timestamp = lookup_es_key(match, rule['timestamp_field']) - timeframe = rule.get('timeframe', opensearch_default_timedelta) - from_timedelta = rule.get('opensearch_discover_from_timedelta', timeframe) - from_time = ts_add(timestamp, -from_timedelta) - to_timedelta = rule.get('opensearch_discover_to_timedelta', timeframe) - to_time = ts_add(timestamp, to_timedelta) - - if opensearch_version in opensearch_versions: - globalState = opensearch_disover_global_state(from_time, to_time) - appState = opensearch_discover_app_state(index, columns, filters, query_keys, match) - appFilter = opensearch_discover_app_filter(index, columns, filters, query_keys, match) - - else: - elastalert_logger.warning( - 'Unknown opensearch discover application version %s for rule %s' % ( - opensearch_version, - rule.get('name', '') - ) - ) - return None - - urlqueryOriginal = "%s?_g=%s&_a=%s&_q=%s" % ( - os.path.expandvars(discover_app_url), - urllib.parse.quote(globalState), - urllib.parse.quote(appState), - urllib.parse.quote(appFilter) - ) - - word_to_replace = 'tobereplacedbylucenequery' - replacement_word = 'query' - max_replacements = 1 # Replace only the first occurrence - urlquery = urlqueryOriginal.replace(word_to_replace, replacement_word, max_replacements) - return urlquery - - -def opensearch_disover_global_state(from_time, to_time): - return prison.dumps( { - 'filters': [], - 'refreshInterval': { - 'pause': True, - 'value': 0 - }, - 'time': { - 'from': from_time, - 'to': to_time - } - } ) - - -def opensearch_discover_app_state(index, columns, filters, query_keys, match): - return prison.dumps( { - 'discover': { - 'columns': columns, - 'isDirty': False, - 'sort': [] - }, - 'metadata': { - 'indexPattern': index, - 'view': 'discover' - } - } ) - -def opensearch_discover_app_filter(index, columns, filters, query_keys, match): - app_filters = [] - - if filters: - - new_filters = [] - for filter in filters: - if 'query' in filter: - filter = filter['query'] - new_filters.append(filter) - filters = new_filters - - bool_filter = {'bool': {'must': filters } } - app_filters.append( { - '$state': { - 'store': 'appState' - }, - 'meta': { - 'alias': 'filter', - 'disabled': False, - 'index': index, - 'key': 'query', - 'negate': False, - 'type': 'custom', - 'value': json.dumps(bool_filter, separators=(',', ':')) - }, - 'query': bool_filter - } ) - - for query_key in query_keys: - query_value = lookup_es_key(match, query_key) - - if query_value is None: - app_filters.append( { - '$state': { - 'store': 'appState' - }, - 'exists': { - 'field': query_key - }, - 'meta': { - 'alias': None, - 'disabled': False, - 'index': index, - 'key': query_key, - 'negate': True, - 'type': 'exists', - 'view': 'discover', - 'value': 'exists' - } - } ) - - else: - app_filters.append( { - '$state': { - 'store': 'appState' - }, - 'meta': { - 'alias': None, - 'disabled': False, - 'index': index, - 'key': query_key, - 'negate': False, - 'params': { - 'query': query_value, - 'type': 'phrase' - }, - 'type': 'phrase', - 'value': str(query_value) - }, - 'query': { - 'match': { - query_key: { - 'query': query_value, - 'type': 'phrase' - } - } - } - } ) - - return prison.dumps( { - 'filters': app_filters, - 'tobereplacedbylucenequery': {'language': 'lucene','query': ''} - } ) +from datetime import timedelta +import pytest + +from elastalert.opensearch_discover import generate_opensearch_discover_url + + +@pytest.mark.parametrize("opensearch_version", [ + '2.11' +]) +def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_url(opensearch_version): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': opensearch_version, + 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + print(url) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'view%3Adiscover%29%29' # app end + + '&_q=%28filters%3A%21%28%29%2C' # query and filter start + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # query and filter start + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_relative_opensearch_discover_app_url_without_http(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'app/discover#/', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': '620ad0e6-43df-4557-bda2-384960fa9086', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2021-10-08T00:30:00Z' + } + ) + expectedUrl = ( + 'app/discover#/' + + '?_g=%28' + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272021-10-08T00%3A20%3A00Z%27%2C' + + 'to%3A%272021-10-08T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27620ad0e6-43df-4557-bda2-384960fa9086%27%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_missing_opensearch_discover_version(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_opensearch_discover_url_with_missing_opensearch_discover_app_url(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_version': '8.11', + 'opensearch_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_opensearch_discover_url_with_missing_opensearch_discover_index_pattern_id(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '8.11', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_opensearch_discover_url_with_invalid_opensearch_version(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '4.5', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_opensearch_discover_url_with_from_timedelta_and_timeframe(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'opensearch_discover_from_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A20%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_to_timedelta_and_timeframe(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'opensearch_discover_to_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_custom_columns(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'opensearch_discover_columns': ['level', 'message'], + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27%' + + '29%' + + '29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28level%2Cmessage%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28filters%3A%21%28%29%2Cquery%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_single_filter(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27%' + + '29%' + + '29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2C' + + 'isDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' + + '%29' + + '%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29' + + '%2Cmeta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22' + + 'must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29' + + '%2Cquery%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_multiple_filters(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'app': 'test'}}, + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28' + + 'alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'key%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' + + 'bool%22%3A%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22app%22%3A%22' + + 'test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28' + + 'app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_int_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo.dest': 200 + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29' + + '%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3A200%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3A%27200%27%29%2C' + + 'query%3A%28match%3A%28geo.dest%3A%28query%3A200%2C' + + 'type%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_str_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'dest': 'ok' + } + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29%' + + '29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' + + '%29' + + '%29' + + '&_q=' + + '%28filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2C' + + 'type%3Aphrase%2Cvalue%3Aok%29%2Cquery%3A%28match%3A%28' + + 'geo.dest%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_missing_query_key_value(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover' + + '%29' + + '%29&' + + '_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2Cexists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2Cnegate%3A%21t%2Ctype%3Aexists%2C' + + 'value%3Aexists%2Cview%3Adiscover%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_compound_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'compound_query_key': ['geo.src', 'geo.dest'], + 'query_key': 'geo.src,geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'src': 'CA', + 'dest': 'US' + } + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2CrefreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2C' + + 'key%3Ageo.src%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3ACA%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3ACA%29%2C' + + 'query%3A%28match%3A%28geo.src%3A%28query%3ACA%2Ctype%3Aphrase%29%29%29%29%' + + '2C%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2Ckey%3Ageo.dest%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3AUS%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3AUS%29%2C' + + 'query%3A%28match%3A%28geo.dest%3A%28' + + 'query%3AUS%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_filter_and_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3A' + + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22' + + 'bool%22%3A%7B%22must%22%3A%5B%7B%22' + + 'term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%7D%27%29%2C' + + 'query%3A%28bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%29%29%2C%28%27%24' + + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2Ckey%3Astatus' + + '%2Cnegate%3A%21f%2Cparams%3A%28query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' + + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl + + +def test_generate_opensearch_discover_url_with_querystring_filter_and_query_key(): + url = generate_opensearch_discover_url( + rule={ + 'opensearch_discover_app_url': 'http://opensearch:5601/#/discover', + 'opensearch_discover_version': '2.11', + 'opensearch_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'query': {'query_string': {'query': 'hello world'}}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://opensearch:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' + + '%29' + + '&_a=%28' + + 'discover%3A%28columns%3A%21%28_source%29%2CisDirty%3A%21f%2Csort%3A%21%28%29%29%2C' + + 'metadata%3A%28indexPattern%3A%27logs-%2A%27%2Cview%3Adiscover%29%29' + + '&_q=%28' + + 'filters%3A%21%28%28%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28alias%3Afilter%2Cdisabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3A' + + 'query%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22' + + 'must%22%3A%5B%7B%22query_string%22%3A%7B%22' + + 'query%22%3A%22hello%20world%22%7D%7D%5D%7D%7D%27%29%2C' + + 'query%3A%28bool%3A%28must%3A%21%28%28query_string%3A%28query%3A%27hello%20' + + 'world%27%29%29%29%29%29%29%2C%28%27%24' + + 'state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2C' + + 'disabled%3A%21f%2Cindex%3A%27logs-%2A%27%2Ckey%3Astatus%2Cnegate%3A%21f%2Cparams%3A%28' + + 'query%3Aok%2Ctype%3Aphrase%29%2Ctype%3Aphrase%2Cvalue%3Aok%29%2C' + + 'query%3A%28match%3A%28status%3A%28query%3Aok%2Ctype%3Aphrase%29%29%29%29%29%2C' + + 'query%3A%28language%3Alucene%2Cquery%3A%27%27%29%29' # app end + ) + assert url == expectedUrl From 1fa776c0fa0483fb8692e5d1e47e98115bd299a7 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:55:30 +0100 Subject: [PATCH 53/65] Update opensearch_discover.py --- elastalert/opensearch_discover.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elastalert/opensearch_discover.py b/elastalert/opensearch_discover.py index 469f1ef5..17dfd27f 100644 --- a/elastalert/opensearch_discover.py +++ b/elastalert/opensearch_discover.py @@ -19,7 +19,7 @@ ]) def generate_opensearch_discover_url(rule, match): - ''' Creates a link for a kibana discover app. ''' + ''' Creates a link for a opensearch discover app. ''' discover_app_url = rule.get('opensearch_discover_app_url') if not discover_app_url: @@ -123,7 +123,6 @@ def opensearch_discover_app_filter(index, columns, filters, query_keys, match): if filters: - # Remove nested query since the outer most query key will break Kibana 8. new_filters = [] for filter in filters: if 'query' in filter: From 4f44d05d9faf17c68c9d3bc2b7efe884bf3ea890 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:01:28 +0100 Subject: [PATCH 54/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 181 +++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 92 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index df4aa8bc..fec4e42c 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -647,16 +647,6 @@ This value will be used if ``generate_kibana_discover_url`` is true and ``kibana (Optional, string, default ``http://:/_plugin/kibana/``) -opensearch_url -^^^^^^^^^^^^^^ - -``opensearch_url``: The base url of the opensearch application. If not specified, a URL will be constructed using ``es_host`` -and ``es_port``. - -This value will be used if ``generate_opensearch_discover_url`` is true and ``opensearch_discover_app_url`` is a relative path - -(Optional, string, default ``http://:/_plugin/kibana/``) - kibana_username ^^^^^^^^^^^^^^^ @@ -711,39 +701,6 @@ Example kibana_url + kibana_discover_app_url usage:: alert_text_args: [ kibana_discover_url ] alert_text_type: alert_text_only -generate_opensearch_discover_url -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``generate_opensearch_discover_url``: Enables the generation of the ``opensearch_discover_url`` variable for the Opensearch Discover application. -This setting requires the following settings are also configured: - -- ``opensearch_discover_app_url`` -- ``opensearch_discover_version`` -- ``opensearch_discover_index_pattern_id`` - -``generate_opensearch_discover_url: true`` - -Example opensearch_discover_app_url only usage for opensearch:: - - generate_opensearch_discover_url: true - opensearch_discover_app_url: "http://localhost:5601/app/data-explorer/discover?security_tenant=Admin#" - opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" - opensearch_discover_version: "2.11" - alert_text: '{}' - alert_text_args: [ opensearch_discover_url ] - alert_text_type: alert_text_only - -Example opensearch_url + opensearch_discover_app_url usage for opensearch:: - - generate_opensearch_discover_url: true - opensearch_url: "http://localhost:5601/" - opensearch_discover_app_url: "app/data-explorer/discover?security_tenant=Admin#" - opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" - opensearch_discover_version: "2.11" - alert_text: '{}' - alert_text_args: [ opensearch_discover_url ] - alert_text_type: alert_text_only - shorten_kibana_discover_url ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -770,17 +727,6 @@ This value should be relative to the base kibana url defined by ``kibana_url`` a (Optional, string, no default) -opensearch_discover_app_url -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``opensearch_discover_app_url``: The url of the opensearch Discover application used to generate the ``opensearch_discover_url`` variable. -This value can use `$VAR` and `${VAR}` references to expand environment variables. -This value should be relative to the base opensearch url defined by ``opensearch_url`` and will vary depending on your installation. - -``opensearch_discover_app_url: app/discover#/`` - -(Optional, string, no default) - kibana_discover_security_tenant ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -801,18 +747,6 @@ The currently supported versions of Kibana Discover are: ``kibana_discover_version: '7.15'`` -opensearch_discover_version -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``opensearch_discover_version``: Specifies the version of the opensearch Discover application. - -The currently supported versions of opensearch Discover are: - -- `2.11` - -``opensearch_discover_version: '2.11'`` - - kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -837,6 +771,95 @@ You can modify an index pattern's id by exporting the saved object, modifying th ``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` +kibana_discover_columns +^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. +Defaults to the ``_source`` column. + +``kibana_discover_columns: [ timestamp, message ]`` + +kibana_discover_from_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. +The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. + +``kibana_discover_from_timedelta: minutes: 2`` + +kibana_discover_to_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. +The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. + +``kibana_discover_to_timedelta: minutes: 2`` + +opensearch_url +^^^^^^^^^^^^^^ + +``opensearch_url``: The base url of the opensearch application. If not specified, a URL will be constructed using ``es_host`` +and ``es_port``. + +This value will be used if ``generate_opensearch_discover_url`` is true and ``opensearch_discover_app_url`` is a relative path + +(Optional, string, default ``http://:/_plugin/_dashboards/``) + +generate_opensearch_discover_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``generate_opensearch_discover_url``: Enables the generation of the ``opensearch_discover_url`` variable for the Opensearch Discover application. +This setting requires the following settings are also configured: + +- ``opensearch_discover_app_url`` +- ``opensearch_discover_version`` +- ``opensearch_discover_index_pattern_id`` + +``generate_opensearch_discover_url: true`` + +Example opensearch_discover_app_url only usage for opensearch:: + + generate_opensearch_discover_url: true + opensearch_discover_app_url: "http://localhost:5601/app/data-explorer/discover?security_tenant=Admin#" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "2.11" + alert_text: '{}' + alert_text_args: [ opensearch_discover_url ] + alert_text_type: alert_text_only + +Example opensearch_url + opensearch_discover_app_url usage for opensearch:: + + generate_opensearch_discover_url: true + opensearch_url: "http://localhost:5601/" + opensearch_discover_app_url: "app/data-explorer/discover?security_tenant=Admin#" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "2.11" + alert_text: '{}' + alert_text_args: [ opensearch_discover_url ] + alert_text_type: alert_text_only + +opensearch_discover_app_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_app_url``: The url of the opensearch Discover application used to generate the ``opensearch_discover_url`` variable. +This value can use `$VAR` and `${VAR}` references to expand environment variables. +This value should be relative to the base opensearch url defined by ``opensearch_url`` and will vary depending on your installation. + +``opensearch_discover_app_url: app/discover#/`` + +(Optional, string, no default) + +opensearch_discover_version +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``opensearch_discover_version``: Specifies the version of the opensearch Discover application. + +The currently supported versions of opensearch Discover are: + +- `2.11` + +``opensearch_discover_version: '2.11'`` + opensearch_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -860,14 +883,6 @@ You can modify an index pattern's id by exporting the saved object, modifying th ``opensearch_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` -kibana_discover_columns -^^^^^^^^^^^^^^^^^^^^^^^ - -``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. -Defaults to the ``_source`` column. - -``kibana_discover_columns: [ timestamp, message ]`` - opensearch_discover_columns ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -876,16 +891,6 @@ Defaults to the ``_source`` column. ``opensearch_discover_columns: [ timestamp, message ]`` - -kibana_discover_from_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. -The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. - -``kibana_discover_from_timedelta: minutes: 2`` - - opensearch_discover_from_timedelta ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -894,14 +899,6 @@ The `from` time is calculated by subtracting this timedelta from the event time. ``opensearch_discover_from_timedelta: minutes: 2`` -kibana_discover_to_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. -The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. - -``kibana_discover_to_timedelta: minutes: 2`` - opensearch_discover_to_timedelta ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From c2ad28603e77d76fec8cf84baf9fe7d46ebff63d Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:13:49 +0100 Subject: [PATCH 55/65] Update opensearch_external_url_formatter_test.py --- .../opensearch_external_url_formatter_test.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/opensearch_external_url_formatter_test.py b/tests/opensearch_external_url_formatter_test.py index 17600479..c484c00d 100644 --- a/tests/opensearch_external_url_formatter_test.py +++ b/tests/opensearch_external_url_formatter_test.py @@ -1,20 +1,9 @@ -from typing import Any -import os import pytest -import requests -from requests.auth import AuthBase, HTTPBasicAuth - from elastalert.opensearch_external_url_formatter import AbsoluteOpensearchExternalUrlFormatter from elastalert.opensearch_external_url_formatter import OpensearchExternalUrlFormatter from elastalert.opensearch_external_url_formatter import create_opensearch_external_url_formatter -from elastalert.auth import RefeshableAWSRequestsAuth -from elastalert.util import EAException - -from unittest import mock - - class AbsoluteFormatTestCase: def __init__( self, @@ -46,3 +35,13 @@ def test_absolute_opensearch_external_url_formatter( actualUrl = formatter.format(test_case.relative_url) print(actualUrl) assert actualUrl == test_case.expected_url + + +def test_create_opensearch_external_url_formatter_without_shortening(): + formatter = create_opensearch_external_url_formatter( + rule={ + 'opensearch_url': 'http://opensearch.test.org/' + }, + ) + assert type(formatter) is AbsoluteOpensearchExternalUrlFormatter + assert formatter.base_url == 'http://opensearch.test.org/' From 523f274d60e4752e7cfbc5dc8e94a218fbda70c6 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:19:16 +0100 Subject: [PATCH 56/65] Update opensearch_external_url_formatter.py --- elastalert/opensearch_external_url_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/opensearch_external_url_formatter.py b/elastalert/opensearch_external_url_formatter.py index dfc0fbbb..e0f190d5 100644 --- a/elastalert/opensearch_external_url_formatter.py +++ b/elastalert/opensearch_external_url_formatter.py @@ -32,4 +32,4 @@ def create_opensearch_external_url_formatter( base_url = rule.get('opensearch_url') - return AbsoluteKibanaExternalUrlFormatter(base_url) + return AbsoluteOpensearchExternalUrlFormatter(base_url) From 367ce02d62e796fb25965b638377a6d1f9831812 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:25:28 +0100 Subject: [PATCH 57/65] Update opensearch_external_url_formatter_test.py --- tests/opensearch_external_url_formatter_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/opensearch_external_url_formatter_test.py b/tests/opensearch_external_url_formatter_test.py index c484c00d..2ecbf54c 100644 --- a/tests/opensearch_external_url_formatter_test.py +++ b/tests/opensearch_external_url_formatter_test.py @@ -1,7 +1,6 @@ import pytest from elastalert.opensearch_external_url_formatter import AbsoluteOpensearchExternalUrlFormatter -from elastalert.opensearch_external_url_formatter import OpensearchExternalUrlFormatter from elastalert.opensearch_external_url_formatter import create_opensearch_external_url_formatter class AbsoluteFormatTestCase: From 49daa71b1c71047c01132099eea612dc768c6a4d Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:34:52 +0100 Subject: [PATCH 58/65] Update opensearch_external_url_formatter_test.py --- tests/opensearch_external_url_formatter_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/opensearch_external_url_formatter_test.py b/tests/opensearch_external_url_formatter_test.py index 2ecbf54c..c6c17eb9 100644 --- a/tests/opensearch_external_url_formatter_test.py +++ b/tests/opensearch_external_url_formatter_test.py @@ -3,6 +3,7 @@ from elastalert.opensearch_external_url_formatter import AbsoluteOpensearchExternalUrlFormatter from elastalert.opensearch_external_url_formatter import create_opensearch_external_url_formatter + class AbsoluteFormatTestCase: def __init__( self, @@ -23,8 +24,6 @@ def __init__( expected_url='http://opensearch.test.org/_dashboards/app/dev_tools#/console' ), ]) - - def test_absolute_opensearch_external_url_formatter( test_case: AbsoluteFormatTestCase ): From 0a79d85d6f0c52f63581a71fd119b615519cdd26 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:45:44 +0100 Subject: [PATCH 59/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 106 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index fec4e42c..403ecddf 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -3016,6 +3016,12 @@ Example mattermost_msg_fields:: ``mattermost_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. +``mattermost_attach_opensearch_discover_url``: Enables the attachment of the ``opensearch_discover_url`` to the mattermost notification. The config ``generate_opensearch_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``mattermost_opensearch_discover_color``: The color of the Opensearch Discover url attachment. Defaults to ``#ec4b98``. + +``mattermost_opensearch_discover_title``: The title of the Opensearch Discover url attachment. Defaults to ``Discover in opensearch``. + Example mattermost_attach_kibana_discover_url, mattermost_kibana_discover_color, mattermost_kibana_discover_title:: # (Required) @@ -3037,6 +3043,28 @@ Example mattermost_attach_kibana_discover_url, mattermost_kibana_discover_color, mattermost_kibana_discover_color: "#ec4b98" mattermost_kibana_discover_title: "Discover in Kibana" +Example mattermost_attach_opensearch_discover_url, mattermost_kibana_discover_color, mattermost_kibana_discover_title:: + + # (Required) + generate_opensearch_discover_url: True + opensearch_discover_app_url: "http://localhost:5601/app/discover#/" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "2.11" + + # (Optional) + opensearch_discover_from_timedelta: + minutes: 10 + opensearch_discover_to_timedelta: + minutes: 10 + + # (Required) + mattermost_attach_opensearch_discover_url: True + + # (Optional) + mattermost_opensearch_discover_color: "#ec4b98" + mattermost_opensearch_discover_title: "Discover in opensearch" + + Microsoft Teams ~~~~~~~~~~~~~~~ @@ -3073,6 +3101,10 @@ Example ms_teams_alert_facts:: ``ms_teams_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. +``ms_teams_attach_opensearch_discover_url``: Enables the attachment of the ``opensearch_discover_url`` to the MS Teams notification. The config ``generate_opensearch_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``ms_teams_opensearch_discover_title``: The title of the Opensearch Discover url attachment. Defaults to ``Discover in opensearch``. + Example ms_teams_attach_kibana_discover_url, ms_teams_kibana_discover_title:: # (Required) @@ -3093,6 +3125,26 @@ Example ms_teams_attach_kibana_discover_url, ms_teams_kibana_discover_title:: # (Optional) ms_teams_kibana_discover_title: "Discover in Kibana" +Example ms_teams_attach_opensearch_discover_url, ms_teams_opensearch_discover_title:: + + # (Required) + generate_opensearch_discover_url: True + opensearch_discover_app_url: "http://localhost:5601/app/discover#/" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "7.15" + + # (Optional) + opensearch_discover_from_timedelta: + minutes: 10 + opensearch_discover_to_timedelta: + minutes: 10 + + # (Required) + ms_teams_attach_opensearch_discover_url: True + + # (Optional) + ms_teams_opensearch_discover_title: "Discover in opensearch" + ``ms_teams_ca_certs``: Set this option to ``True`` or a path to a CA cert bundle or directory (eg: ``/etc/ssl/certs/ca-certificates.crt``) to validate the SSL certificate. ``ms_teams_ignore_ssl_errors``: By default ElastAlert 2 will verify SSL certificate. Set this option to ``True`` if you want to ignore SSL errors. @@ -3294,6 +3346,12 @@ ElastAlert 2 rule. Any Apple emoji can be used, see http://emojipedia.org/apple/ ``rocket_chat_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. +``rocket_chat_attach_opensearch_discover_url``: Enables the attachment of the ``opensearch_discover_url`` to the Rocket.Chat notification. The config ``generate_opensearch_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``rocket_chat_opensearch_discover_color``: The color of the Opensearch Discover url attachment. Defaults to ``#ec4b98``. + +``rocket_chat_opensearch_discover_title``: The title of the Opensearch Discover url attachment. Defaults to ``Discover in opensearch``. + Example rocket_chat_attach_kibana_discover_url, rocket_chat_kibana_discover_color, rocket_chat_kibana_discover_title:: # (Required) @@ -3315,6 +3373,27 @@ Example rocket_chat_attach_kibana_discover_url, rocket_chat_kibana_discover_colo rocket_chat_kibana_discover_color: "#ec4b98" rocket_chat_kibana_discover_title: "Discover in Kibana" +Example rocket_chat_attach_opensearch_discover_url, rocket_chat_opensearch_discover_color, rocket_chat_opensearch_discover_title:: + + # (Required) + generate_opensearch_discover_url: True + opensearch_discover_app_url: "http://localhost:5601/app/discover#/" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "2.11" + + # (Optional) + opensearch_discover_from_timedelta: + minutes: 10 + opensearch_discover_to_timedelta: + minutes: 10 + + # (Required) + rocket_chat_attach_opensearch_discover_url: True + + # (Optional) + rocket_chat_opensearch_discover_color: "#ec4b98" + rocket_chat_opensearch_discover_title: "Discover in opensearch" + ``rocket_chat_alert_fields``: You can add additional fields to your Rocket.Chat alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. Example rocket_chat_alert_fields:: @@ -3458,6 +3537,12 @@ Example slack_alert_fields:: ``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. +``slack_attach_opensearch_discover_url``: Enables the attachment of the ``opensearch_discover_url`` to the slack notification. The config ``generate_opensearch_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``slack_opensearch_discover_color``: The color of the Opensearch Discover url attachment. Defaults to ``#ec4b98``. + +``slack_opensearch_discover_title``: The title of the Opensearch Discover url attachment. Defaults to ``Discover in Opensearch``. + Example slack_attach_kibana_discover_url, slack_kibana_discover_color, slack_kibana_discover_title:: # (Required) @@ -3479,6 +3564,27 @@ Example slack_attach_kibana_discover_url, slack_kibana_discover_color, slack_kib slack_kibana_discover_color: "#ec4b98" slack_kibana_discover_title: "Discover in Kibana" +Example slack_attach_opensearch_discover_url, slack_opensearch_discover_color, slack_opensearch_discover_title:: + + # (Required) + generate_opensearch_discover_url: True + opensearch_discover_app_url: "http://localhost:5601/app/discover#/" + opensearch_discover_index_pattern_id: "4babf380-c3b1-11eb-b616-1b59c2feec54" + opensearch_discover_version: "7.15" + + # (Optional) + opensearch_discover_from_timedelta: + minutes: 10 + opensearch_discover_to_timedelta: + minutes: 10 + + # (Required) + slack_attach_opensearch_discover_url: True + + # (Optional) + slack_opensearch_discover_color: "#ec4b98" + slack_opensearch_discover_title: "Discover in opensearch" + ``slack_ca_certs``: Set this option to ``True`` or a path to a CA cert bundle or directory (eg: ``/etc/ssl/certs/ca-certificates.crt``) to validate the SSL certificate. ``slack_footer``: Add a static footer text for alert. Defaults to "". From 4cb537d7d25e03c34b5992d1ff50b02788744b9b Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:46:40 +0100 Subject: [PATCH 60/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 403ecddf..9232ff0a 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -96,8 +96,6 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``opensearch_discover_index_pattern_id`` (string, no default)| | +--------------------------------------------------------------+ | -| ``opensearch_discover_security_tenant`` (string, no default)| | -+--------------------------------------------------------------+ | |``opensearch_discover_columns`` (list of strs,default _source)| | +--------------------------------------------------------------+ | | ``opensearch_discover_from_timedelta`` (time,default: 10 min)| | From 8b4a381e48a01c61021a02fc9e6279211c3d67f9 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:56:15 +0100 Subject: [PATCH 61/65] Update kibana_external_url_formatter.py --- elastalert/kibana_external_url_formatter.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/elastalert/kibana_external_url_formatter.py b/elastalert/kibana_external_url_formatter.py index f95f2b3e..083bc8ac 100644 --- a/elastalert/kibana_external_url_formatter.py +++ b/elastalert/kibana_external_url_formatter.py @@ -157,15 +157,4 @@ def create_kibana_external_url_formatter( return ShortKibanaExternalUrlFormatter(base_url, auth, security_tenant, new_shortener, verify) return AbsoluteKibanaExternalUrlFormatter(base_url, security_tenant) - -def create_opensearch_external_url_formatter( - rule, - shorten: bool, - security_tenant: str, -) -> KibanaExternalUrlFormatter: - '''Creates a Kibana external url formatter''' - - base_url = rule.get('kibana_url') - - return AbsoluteKibanaExternalUrlFormatter(base_url, security_tenant) From c421445bafa6492222e47362fd9f3cffeda4fe6c Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:02:34 +0100 Subject: [PATCH 62/65] Update kibana_external_url_formatter.py --- elastalert/kibana_external_url_formatter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/kibana_external_url_formatter.py b/elastalert/kibana_external_url_formatter.py index 083bc8ac..5d739e72 100644 --- a/elastalert/kibana_external_url_formatter.py +++ b/elastalert/kibana_external_url_formatter.py @@ -157,4 +157,3 @@ def create_kibana_external_url_formatter( return ShortKibanaExternalUrlFormatter(base_url, auth, security_tenant, new_shortener, verify) return AbsoluteKibanaExternalUrlFormatter(base_url, security_tenant) - From 52b5db9778cadf52dc694a224fb5f9e14bdc4128 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:17:12 +0100 Subject: [PATCH 63/65] Update ruletypes.rst --- docs/source/ruletypes.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 9232ff0a..92ac80b2 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -70,8 +70,6 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``generate_kibana_discover_url`` (boolean, default False) | | +--------------------------------------------------------------+ | -| ``generate_opensearch_discover_url`` (boolean, default False)| | -+--------------------------------------------------------------+ | | ``shorten_kibana_discover_url`` (boolean, default False) | | +--------------------------------------------------------------+ | | ``kibana_discover_app_url`` (string, no default) | | @@ -88,7 +86,7 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana_discover_to_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | -| ``shorten_kibana_discover_url`` (boolean, default False) | | +| ``generate_opensearch_discover_url`` (boolean, default False)| | +--------------------------------------------------------------+ | | ``opensearch_discover_app_url`` (string, no default) | | +--------------------------------------------------------------+ | From 087f0e7e0a2e916d25f0d5c3ea166a6a14c21756 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:18:13 +0100 Subject: [PATCH 64/65] Update elastalert.py --- elastalert/elastalert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index e7fbb492..96b885b9 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1350,10 +1350,10 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): matches[0]['kibana_discover_url'] = kb_link_formatter.format(kb_link) if rule.get('generate_opensearch_discover_url'): - kb_link = generate_opensearch_discover_url(rule, matches[0]) - if kb_link: - kb_link_formatter = self.get_opensearch_discover_external_url_formatter(rule) - matches[0]['opensearch_discover_url'] = kb_link_formatter.format(kb_link) + opsh_link = generate_opensearch_discover_url(rule, matches[0]) + if opsh_link: + opsh_link_formatter = self.get_opensearch_discover_external_url_formatter(rule) + matches[0]['opensearch_discover_url'] = opsh_link_formatter.format(opsh_link) From c53c9c341b9074202cbcd0ad22d1362a52be0bd0 Mon Sep 17 00:00:00 2001 From: luffynextgen <34096485+luffynextgen@users.noreply.github.com> Date: Fri, 17 Nov 2023 15:07:03 +0100 Subject: [PATCH 65/65] Update opensearch_external_url_formatter_test.py --- tests/opensearch_external_url_formatter_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/opensearch_external_url_formatter_test.py b/tests/opensearch_external_url_formatter_test.py index c6c17eb9..f5f27b80 100644 --- a/tests/opensearch_external_url_formatter_test.py +++ b/tests/opensearch_external_url_formatter_test.py @@ -31,7 +31,6 @@ def test_absolute_opensearch_external_url_formatter( base_url=test_case.base_url ) actualUrl = formatter.format(test_case.relative_url) - print(actualUrl) assert actualUrl == test_case.expected_url