[SOAR-17874] [Mimecast] - rate limiting improvements. (#2958)

* SOAR=17874 - catch any exceptions when calculating new rate time

* SOAR: check result from check_rate_limit and update unit tests.

plugins/mimecast/.CHECKSUM

@@ -1,7 +1,7 @@
 {
-	"spec": "ab629f58d383334a20d036678e45b9f2",
-	"manifest": "9f9a8a2c45095798625a37f3878b8dd4",
-	"setup": "5ba5d7927984907d1551077e7dde11ac",
+	"spec": "6da6579eda4ddc4e9480be4896f78c16",
+	"manifest": "66b9e9d783bc569c9c4af45fd1fe92e7",
+	"setup": "2cb4c47c6fd5fc6f70a966d2e601e04c",
 	"schemas": [
 		{
 			"identifier": "add_group_member/schema.py",

plugins/mimecast/bin/komand_mimecast

@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Mimecast"
 Vendor = "rapid7"
-Version = "5.3.19"
+Version = "5.3.20"
 Description = "[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)"
 
 

plugins/mimecast/help.md

@@ -1014,6 +1014,7 @@ For the Create Managed URL action, the URL must include `http://` or `https://`
 
 # Version History
 
+* 5.3.20 - Update Task `monitor_siem_logs` bump default rate limit period to 10 minutes and catch unexpected errors
 * 5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast | Update SDK to version 6.2.0
 * 5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2
 * 5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region

plugins/mimecast/komand_mimecast/tasks/monitor_siem_logs/task.py

@@ -45,7 +45,9 @@ def run(  # pylint: disable=unused-argument # noqa: MC0001
         self, params={}, state={}, custom_config={}
     ) -> (List[Dict], Dict, Dict):
         try:
-            self.check_rate_limit(state)
+            rate_limited = self.check_rate_limit(state)
+            if rate_limited:
+                return [], state, False, 429, rate_limited
             has_more_pages = False
             header_next_token = state.get(self.NEXT_TOKEN, "")
             normal_running_cutoff = state.get(self.NORMAL_RUNNING_CUTOFF, False)
@@ -278,23 +280,30 @@ def check_rate_limit(self, state: Dict):
                     cause=PluginException.causes.get(PluginException.Preset.RATE_LIMIT),
                     assistance=RATE_LIMIT_ASSISTANCE,
                 )
-                return [], state, False, 429, error
+                return error
 
             log_msg += "However no longer in rate limiting period, so task can be executed..."
             del state[self.RATE_LIMIT_DATETIME]
             self.logger.info(log_msg)
 
     def check_rate_limit_error(
         self, error: ApiClientException, response: Response, status_code: int, state: dict
-    ) -> Tuple[int, ApiClientException]:
+    ) -> Tuple[int, Any, bool]:
         if status_code == 429:
-            rate_limit_response_time = response.headers.get("X-RateLimit-Reset")
-            if rate_limit_response_time:
-                new_run_time = time() + (rate_limit_response_time / 1000)
-            else:
-                new_run_time = time() + 300
-            new_run_time_string = Utils.convert_epoch_to_readable(new_run_time)
-            self.logger.error(f"A rate limit error has occurred, task will resume after {new_run_time_string}")
-            state[self.RATE_LIMIT_DATETIME] = new_run_time
+            new_run_time = time() + 600  # default to wait 10 minutes before the next run
+            try:
+                # This value should be the amount of time in milliseconds until the next call is allowed
+                rate_limit_response_time = response.headers.get("X-RateLimit-Reset")
+                if rate_limit_response_time:
+                    self.logger.info(f"Got X-RateLimit-Reset value from headers: {rate_limit_response_time}")
+                    new_run_time = time() + (int(rate_limit_response_time) / 1000)
+                new_run_time_string = Utils.convert_epoch_to_readable(new_run_time)
+                self.logger.error(f"A rate limit error has occurred, task will resume after {new_run_time_string}")
+                state[self.RATE_LIMIT_DATETIME] = new_run_time
+            except Exception as err:
+                self.logger.error(
+                    f"Unable to calculate new run time, no rate limiting applied to the state. Error: {repr(err)}",
+                    exc_info=True,
+                )
             return 200, None, True
         return status_code, error, False

plugins/mimecast/komand_mimecast/util/util.py

@@ -6,7 +6,7 @@
 
 class Utils:
     @staticmethod
-    def convert_epoch_to_readable(epoch_time: int) -> str:
+    def convert_epoch_to_readable(epoch_time: float) -> str:
         return datetime.utcfromtimestamp(epoch_time).strftime("%Y-%m-%d %H:%M:%S")
 
     @staticmethod

plugins/mimecast/plugin.spec.yaml

@@ -17,7 +17,7 @@ links:
  - "[Mimecast](http://mimecast.com)"
 references:
  - "[Mimecast API](https://www.mimecast.com/developer/documentation)"
-version: 5.3.19
+version: 5.3.20
 connection_version: 5
 supported_versions: ["Mimecast API 2024-06-18"]
 vendor: rapid7
@@ -40,6 +40,7 @@ hub_tags:
   keywords: [mimecast, email, cloud_enabled]
   features: []
 version_history:
+- "5.3.20 - Update Task `monitor_siem_logs` bump default rate limit period to 10 minutes and catch unexpected errors"
 - "5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast | Update SDK to version 6.2.0"
 - "5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2"
 - "5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region"

plugins/mimecast/setup.py

@@ -3,7 +3,7 @@
 
 
 setup(name="mimecast-rapid7-plugin",
-      version="5.3.19",
+      version="5.3.20",
       description="[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)",
       author="rapid7",
       author_email="",

plugins/mimecast/unit_test/test_monitor_siem_logs.py

@@ -3,9 +3,10 @@
 import sys
 import logging
 from jsonschema import validate
-from unittest import TestCase, skip
+from unittest import TestCase
 from unittest.mock import patch
 from freezegun import freeze_time
+from time import time
 
 sys.path.append(os.path.abspath("../"))
 
@@ -67,14 +68,56 @@ def test_monitor_siem_logs_raises_401(self, _mock_data):
     def test_monitor_siem_logs_raises_429(self, _mock_data):
         state_params = {"next_token": "force_429", "last_log_line": 0}
         expected_state = state_params.copy()
-        expected_state.update({"rate_limit_datetime": 1641039000.0})
+        expected_state.update({"rate_limit_datetime": 1641039600.0})  # current time + header value of 5 minutes
         response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params)
         self.assertEqual(status_code, 200)
         self.assertEqual(has_more_pages, True)
         self.assertEqual(response, [])
         self.assertEqual(new_state, expected_state)
         self.assertEqual(error, None)
 
+        # Second run should return 429 status code
+        _, new_state_2, has_more_pages, exp_status_code, error = self.task.run(params={}, state=expected_state)
+        self.assertEqual(429, exp_status_code)
+        self.assertEqual(new_state_2, expected_state)  # state doesn't change until the time has passed
+
+    @patch("logging.Logger.error")
+    def test_monitor_siem_logs_raises_429_and_errors(self, mocked_logger, _mock_data):
+        state_params = {"next_token": "force_429_error", "last_log_line": 0}
+        response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params)
+        self.assertEqual(status_code, 200)
+        self.assertEqual(has_more_pages, True)
+        self.assertEqual(response, [])
+        self.assertEqual(new_state, state_params)
+        self.assertEqual(error, None)
+        mocked_logger.assert_called()
+        self.assertIn(
+            "Unable to calculate new run time, no rate limiting applied to the state", mocked_logger.call_args[0][0]
+        )
+
+    def test_monitor_siem_logs_raises_429_no_header(self, _mock_data):
+        state_params = {"next_token": "force_429_no_header", "last_log_line": 0}
+        expected_state = state_params.copy()
+        expected_state.update({"rate_limit_datetime": 1641039000})  # uses current time + 10 minutes
+        response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params)
+        self.assertEqual(status_code, 200)
+        self.assertEqual(has_more_pages, True)
+        self.assertEqual(response, [])
+        self.assertEqual(new_state, expected_state)
+        self.assertEqual(error, None)
+
+    @patch("logging.Logger.info")
+    def test_monitor_logs_rate_limit_has_passed(self, mock_logger, _mock_data):
+        # force the time to be the pasts then next run is able to run again
+        state = {"rate_limit_datetime": time()}
+        _, rate_limit_passed_state, _, status_code, _ = self.task.run(params={}, state=state)
+        self.assertEqual(200, status_code)
+        self.assertNotIn("rate_limit_datetime", rate_limit_passed_state.keys())
+
+        self.assertIn(
+            "However no longer in rate limiting period, so task can be executed...", mock_logger.call_args_list[0][0][0]
+        )
+
     @patch("logging.Logger.error")
     def test_monitor_siem_logs_stops_path_traversal(self, mock_logger, _mock_data):
         test_state = {

plugins/mimecast/unit_test/util.py

@@ -186,7 +186,15 @@ def json(self):
                         }
                     ],
                 }
-                headers = {"X-RateLimit-Reset": 600000}
+
+                # This value should be the amount of time in milliseconds until the next call is allowed
+                # in the mock make the customer wait 20 minutes. # todo - WHY IS THIS NOT WORKING
+                reset_value = "not an integer value" if "force_429_error" in data else "1200000"
+                headers = {"X-RateLimit-Reset": reset_value}
+
+                # default to calculate a new time in the task based on no return value
+                if "force_429_no_header" in data:
+                    del headers["X-RateLimit-Reset"]
                 resp = MockResponseZip(429, b"", headers, json.dumps(json_value))
             elif "force_json" in data:
                 resp = MockResponseZip(200, b'{ "type" : "MTA", "data" : ', headers, '{"meta": {"status": 200}}')