From 1dfe8e45fdec0f7049b4e2643b48e8154c5d6671 Mon Sep 17 00:00:00 2001 From: Johnny O'Neill <139136675+joneill-r7@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:20:27 +0000 Subject: [PATCH] [SOAR-17874] [Mimecast] - rate limiting improvements. (#2958) * SOAR=17874 - catch any exceptions when calculating new rate time * SOAR: check result from check_rate_limit and update unit tests. --- plugins/mimecast/.CHECKSUM | 6 +-- plugins/mimecast/bin/komand_mimecast | 2 +- plugins/mimecast/help.md | 1 + .../tasks/monitor_siem_logs/task.py | 31 +++++++----- plugins/mimecast/komand_mimecast/util/util.py | 2 +- plugins/mimecast/plugin.spec.yaml | 3 +- plugins/mimecast/setup.py | 2 +- .../unit_test/test_monitor_siem_logs.py | 47 ++++++++++++++++++- plugins/mimecast/unit_test/util.py | 10 +++- 9 files changed, 83 insertions(+), 21 deletions(-) diff --git a/plugins/mimecast/.CHECKSUM b/plugins/mimecast/.CHECKSUM index f6b7f4bbc6..d5b1c38deb 100644 --- a/plugins/mimecast/.CHECKSUM +++ b/plugins/mimecast/.CHECKSUM @@ -1,7 +1,7 @@ { - "spec": "ab629f58d383334a20d036678e45b9f2", - "manifest": "9f9a8a2c45095798625a37f3878b8dd4", - "setup": "5ba5d7927984907d1551077e7dde11ac", + "spec": "6da6579eda4ddc4e9480be4896f78c16", + "manifest": "66b9e9d783bc569c9c4af45fd1fe92e7", + "setup": "2cb4c47c6fd5fc6f70a966d2e601e04c", "schemas": [ { "identifier": "add_group_member/schema.py", diff --git a/plugins/mimecast/bin/komand_mimecast b/plugins/mimecast/bin/komand_mimecast index bc5d6faf0c..1b51e2a25c 100755 --- a/plugins/mimecast/bin/komand_mimecast +++ b/plugins/mimecast/bin/komand_mimecast @@ -6,7 +6,7 @@ from sys import argv Name = "Mimecast" Vendor = "rapid7" -Version = "5.3.19" +Version = "5.3.20" Description = "[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)" diff --git a/plugins/mimecast/help.md b/plugins/mimecast/help.md index 7d7463f16c..2b6d99850c 100644 --- a/plugins/mimecast/help.md +++ b/plugins/mimecast/help.md @@ -1014,6 +1014,7 @@ For the Create Managed URL action, the URL must include `http://` or `https://` # Version History +* 5.3.20 - Update Task `monitor_siem_logs` bump default rate limit period to 10 minutes and catch unexpected errors * 5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast | Update SDK to version 6.2.0 * 5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2 * 5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region diff --git a/plugins/mimecast/komand_mimecast/tasks/monitor_siem_logs/task.py b/plugins/mimecast/komand_mimecast/tasks/monitor_siem_logs/task.py index 612458013e..145a5911f5 100755 --- a/plugins/mimecast/komand_mimecast/tasks/monitor_siem_logs/task.py +++ b/plugins/mimecast/komand_mimecast/tasks/monitor_siem_logs/task.py @@ -45,7 +45,9 @@ def run( # pylint: disable=unused-argument # noqa: MC0001 self, params={}, state={}, custom_config={} ) -> (List[Dict], Dict, Dict): try: - self.check_rate_limit(state) + rate_limited = self.check_rate_limit(state) + if rate_limited: + return [], state, False, 429, rate_limited has_more_pages = False header_next_token = state.get(self.NEXT_TOKEN, "") normal_running_cutoff = state.get(self.NORMAL_RUNNING_CUTOFF, False) @@ -278,7 +280,7 @@ def check_rate_limit(self, state: Dict): cause=PluginException.causes.get(PluginException.Preset.RATE_LIMIT), assistance=RATE_LIMIT_ASSISTANCE, ) - return [], state, False, 429, error + return error log_msg += "However no longer in rate limiting period, so task can be executed..." del state[self.RATE_LIMIT_DATETIME] @@ -286,15 +288,22 @@ def check_rate_limit(self, state: Dict): def check_rate_limit_error( self, error: ApiClientException, response: Response, status_code: int, state: dict - ) -> Tuple[int, ApiClientException]: + ) -> Tuple[int, Any, bool]: if status_code == 429: - rate_limit_response_time = response.headers.get("X-RateLimit-Reset") - if rate_limit_response_time: - new_run_time = time() + (rate_limit_response_time / 1000) - else: - new_run_time = time() + 300 - new_run_time_string = Utils.convert_epoch_to_readable(new_run_time) - self.logger.error(f"A rate limit error has occurred, task will resume after {new_run_time_string}") - state[self.RATE_LIMIT_DATETIME] = new_run_time + new_run_time = time() + 600 # default to wait 10 minutes before the next run + try: + # This value should be the amount of time in milliseconds until the next call is allowed + rate_limit_response_time = response.headers.get("X-RateLimit-Reset") + if rate_limit_response_time: + self.logger.info(f"Got X-RateLimit-Reset value from headers: {rate_limit_response_time}") + new_run_time = time() + (int(rate_limit_response_time) / 1000) + new_run_time_string = Utils.convert_epoch_to_readable(new_run_time) + self.logger.error(f"A rate limit error has occurred, task will resume after {new_run_time_string}") + state[self.RATE_LIMIT_DATETIME] = new_run_time + except Exception as err: + self.logger.error( + f"Unable to calculate new run time, no rate limiting applied to the state. Error: {repr(err)}", + exc_info=True, + ) return 200, None, True return status_code, error, False diff --git a/plugins/mimecast/komand_mimecast/util/util.py b/plugins/mimecast/komand_mimecast/util/util.py index a9c8a4bf0b..2bb6da6f0b 100644 --- a/plugins/mimecast/komand_mimecast/util/util.py +++ b/plugins/mimecast/komand_mimecast/util/util.py @@ -6,7 +6,7 @@ class Utils: @staticmethod - def convert_epoch_to_readable(epoch_time: int) -> str: + def convert_epoch_to_readable(epoch_time: float) -> str: return datetime.utcfromtimestamp(epoch_time).strftime("%Y-%m-%d %H:%M:%S") @staticmethod diff --git a/plugins/mimecast/plugin.spec.yaml b/plugins/mimecast/plugin.spec.yaml index 96f8bc322a..305d294eb9 100644 --- a/plugins/mimecast/plugin.spec.yaml +++ b/plugins/mimecast/plugin.spec.yaml @@ -17,7 +17,7 @@ links: - "[Mimecast](http://mimecast.com)" references: - "[Mimecast API](https://www.mimecast.com/developer/documentation)" -version: 5.3.19 +version: 5.3.20 connection_version: 5 supported_versions: ["Mimecast API 2024-06-18"] vendor: rapid7 @@ -40,6 +40,7 @@ hub_tags: keywords: [mimecast, email, cloud_enabled] features: [] version_history: +- "5.3.20 - Update Task `monitor_siem_logs` bump default rate limit period to 10 minutes and catch unexpected errors" - "5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast | Update SDK to version 6.2.0" - "5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2" - "5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region" diff --git a/plugins/mimecast/setup.py b/plugins/mimecast/setup.py index e9aecb1996..fff3705f7b 100755 --- a/plugins/mimecast/setup.py +++ b/plugins/mimecast/setup.py @@ -3,7 +3,7 @@ setup(name="mimecast-rapid7-plugin", - version="5.3.19", + version="5.3.20", description="[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)", author="rapid7", author_email="", diff --git a/plugins/mimecast/unit_test/test_monitor_siem_logs.py b/plugins/mimecast/unit_test/test_monitor_siem_logs.py index cc1d70c42c..75a9f73c5e 100644 --- a/plugins/mimecast/unit_test/test_monitor_siem_logs.py +++ b/plugins/mimecast/unit_test/test_monitor_siem_logs.py @@ -3,9 +3,10 @@ import sys import logging from jsonschema import validate -from unittest import TestCase, skip +from unittest import TestCase from unittest.mock import patch from freezegun import freeze_time +from time import time sys.path.append(os.path.abspath("../")) @@ -67,7 +68,7 @@ def test_monitor_siem_logs_raises_401(self, _mock_data): def test_monitor_siem_logs_raises_429(self, _mock_data): state_params = {"next_token": "force_429", "last_log_line": 0} expected_state = state_params.copy() - expected_state.update({"rate_limit_datetime": 1641039000.0}) + expected_state.update({"rate_limit_datetime": 1641039600.0}) # current time + header value of 5 minutes response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params) self.assertEqual(status_code, 200) self.assertEqual(has_more_pages, True) @@ -75,6 +76,48 @@ def test_monitor_siem_logs_raises_429(self, _mock_data): self.assertEqual(new_state, expected_state) self.assertEqual(error, None) + # Second run should return 429 status code + _, new_state_2, has_more_pages, exp_status_code, error = self.task.run(params={}, state=expected_state) + self.assertEqual(429, exp_status_code) + self.assertEqual(new_state_2, expected_state) # state doesn't change until the time has passed + + @patch("logging.Logger.error") + def test_monitor_siem_logs_raises_429_and_errors(self, mocked_logger, _mock_data): + state_params = {"next_token": "force_429_error", "last_log_line": 0} + response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params) + self.assertEqual(status_code, 200) + self.assertEqual(has_more_pages, True) + self.assertEqual(response, []) + self.assertEqual(new_state, state_params) + self.assertEqual(error, None) + mocked_logger.assert_called() + self.assertIn( + "Unable to calculate new run time, no rate limiting applied to the state", mocked_logger.call_args[0][0] + ) + + def test_monitor_siem_logs_raises_429_no_header(self, _mock_data): + state_params = {"next_token": "force_429_no_header", "last_log_line": 0} + expected_state = state_params.copy() + expected_state.update({"rate_limit_datetime": 1641039000}) # uses current time + 10 minutes + response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params) + self.assertEqual(status_code, 200) + self.assertEqual(has_more_pages, True) + self.assertEqual(response, []) + self.assertEqual(new_state, expected_state) + self.assertEqual(error, None) + + @patch("logging.Logger.info") + def test_monitor_logs_rate_limit_has_passed(self, mock_logger, _mock_data): + # force the time to be the pasts then next run is able to run again + state = {"rate_limit_datetime": time()} + _, rate_limit_passed_state, _, status_code, _ = self.task.run(params={}, state=state) + self.assertEqual(200, status_code) + self.assertNotIn("rate_limit_datetime", rate_limit_passed_state.keys()) + + self.assertIn( + "However no longer in rate limiting period, so task can be executed...", mock_logger.call_args_list[0][0][0] + ) + @patch("logging.Logger.error") def test_monitor_siem_logs_stops_path_traversal(self, mock_logger, _mock_data): test_state = { diff --git a/plugins/mimecast/unit_test/util.py b/plugins/mimecast/unit_test/util.py index 0798c2b538..2768ac2f92 100644 --- a/plugins/mimecast/unit_test/util.py +++ b/plugins/mimecast/unit_test/util.py @@ -186,7 +186,15 @@ def json(self): } ], } - headers = {"X-RateLimit-Reset": 600000} + + # This value should be the amount of time in milliseconds until the next call is allowed + # in the mock make the customer wait 20 minutes. # todo - WHY IS THIS NOT WORKING + reset_value = "not an integer value" if "force_429_error" in data else "1200000" + headers = {"X-RateLimit-Reset": reset_value} + + # default to calculate a new time in the task based on no return value + if "force_429_no_header" in data: + del headers["X-RateLimit-Reset"] resp = MockResponseZip(429, b"", headers, json.dumps(json_value)) elif "force_json" in data: resp = MockResponseZip(200, b'{ "type" : "MTA", "data" : ', headers, '{"meta": {"status": 200}}')