[SOAR-17874] Mimecast (#2897)

* Palo alto retry creds 401 (#2826)

* Add 401 exception handling

* Fix it up

* Rename response to new response

* Add 429 handler to task | update sdk | add unit test

* Update requirements werkzeug

* Update rate limit error

* Updare bandit

* Update error logging

* Update value returned by check rate limit status code

* Update SDK

* Update return 200 first 429

* Update has_more_pages

* Update task.py

---------

Co-authored-by: Conor <93926445+cmcnally-r7@users.noreply.github.com>

plugins/mimecast/.CHECKSUM

@@ -1,7 +1,7 @@
 {
-	"spec": "35db88201d0c437148d3a5851253f0e4",
-	"manifest": "8be151ad8e7d232ce62fded2340d6841",
-	"setup": "e585cdea9a7cf94ca339985c0f2bbc7d",
+	"spec": "ab629f58d383334a20d036678e45b9f2",
+	"manifest": "9f9a8a2c45095798625a37f3878b8dd4",
+	"setup": "5ba5d7927984907d1551077e7dde11ac",
 	"schemas": [
 		{
 			"identifier": "add_group_member/schema.py",

plugins/mimecast/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 rapid7/insightconnect-python-3-slim-plugin:6.1.2
+FROM --platform=linux/amd64 rapid7/insightconnect-python-3-slim-plugin:6.2.0
 
 LABEL organization=rapid7
 LABEL sdk=python

plugins/mimecast/bin/komand_mimecast

@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Mimecast"
 Vendor = "rapid7"
-Version = "5.3.18"
+Version = "5.3.19"
 Description = "[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)"
 
 

plugins/mimecast/help.md

@@ -1014,6 +1014,7 @@ For the Create Managed URL action, the URL must include `http://` or `https://`
 
 # Version History
 
+* 5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast | Update SDK to version 6.2.0
 * 5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2
 * 5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region
 * 5.3.16 - Task `monitor_siem_logs` Limit the number of events per run to 7500 | bump SDK to version 6.1.0

plugins/mimecast/komand_mimecast/tasks/monitor_siem_logs/task.py

@@ -8,11 +8,13 @@
 
 from insightconnect_plugin_runtime.exceptions import PluginException
 from insightconnect_plugin_runtime.helper import get_time_hours_ago
+from requests import Response
 
 from .schema import MonitorSiemLogsInput, MonitorSiemLogsOutput, MonitorSiemLogsState, Component
-from ...util.constants import IS_LAST_TOKEN_FIELD
-from ...util.event import EventLogs
-from ...util.exceptions import ApiClientException
+from komand_mimecast.util.constants import IS_LAST_TOKEN_FIELD, RATE_LIMIT_ASSISTANCE
+from komand_mimecast.util.event import EventLogs
+from komand_mimecast.util.exceptions import ApiClientException
+from komand_mimecast.util.util import Utils
 
 FIRST_RUN_CUTOFF = 24
 NORMAL_RUNNING_CUTOFF = 24 * 7
@@ -23,6 +25,7 @@ class MonitorSiemLogs(insightconnect_plugin_runtime.Task):
     NEXT_TOKEN = "next_token"  # nosec
     HEADER_NEXT_TOKEN = "mc-siem-token"  # nosec
     STATUS_CODE = "status_code"  # nosec
+    RATE_LIMIT_DATETIME = "rate_limit_datetime"
 
     NORMAL_RUNNING_CUTOFF = "normal_running_cutoff"  # nosec
     LAST_LOG_LINE = "last_log_line"  # nosec
@@ -42,6 +45,7 @@ def run(  # pylint: disable=unused-argument # noqa: MC0001
         self, params={}, state={}, custom_config={}
     ) -> (List[Dict], Dict, Dict):
         try:
+            self.check_rate_limit(state)
             has_more_pages = False
             header_next_token = state.get(self.NEXT_TOKEN, "")
             normal_running_cutoff = state.get(self.NORMAL_RUNNING_CUTOFF, False)
@@ -81,7 +85,10 @@ def run(  # pylint: disable=unused-argument # noqa: MC0001
                         f"An exception has been raised during retrieval of siem logs. Status code: {error.status_code} "
                         f"Error: {error}, returning state={state}, has_more_pages={has_more_pages}"
                     )
-                    return [], state, has_more_pages, error.status_code, error
+                    status_code, error, has_more_pages = self.check_rate_limit_error(
+                        error, error.response, error.status_code, state
+                    )
+                    return [], state, has_more_pages, status_code, error
 
                 # check if the hashed file list from the previous run is the same as this run
                 if len(output) > max_events_per_run:
@@ -203,7 +210,7 @@ def _filter_and_sort_recent_events(
 
     def _check_hash_of_file_names(self, file_name_list: List[str]) -> str:
         file_name_list = sorted(file_name_list)
-        return hashlib.md5(str(file_name_list).encode("utf-8")).hexdigest()  # nosec B303
+        return hashlib.md5(str(file_name_list).encode("utf-8")).hexdigest()  # nosec B324
 
     def _get_filter_time(self, custom_config: Dict[str, int], normal_running_cutoff: bool = False) -> int:
         """
@@ -257,3 +264,37 @@ def _get_filter_time(self, custom_config: Dict[str, int], normal_running_cutoff:
         self.logger.info(f"The following filter time will be used: {filter_time}")
 
         return filter_time
+
+    def check_rate_limit(self, state: Dict):
+        rate_limited = state.get(self.RATE_LIMIT_DATETIME)
+        now = time()
+        if rate_limited:
+            rate_limit_string = Utils.convert_epoch_to_readable(rate_limited)
+            log_msg = f"Rate limit value stored in state: {rate_limit_string}. "
+            if rate_limited > now:
+                log_msg += "Still within rate limiting period, skipping task execution..."
+                self.logger.info(log_msg)
+                error = PluginException(
+                    cause=PluginException.causes.get(PluginException.Preset.RATE_LIMIT),
+                    assistance=RATE_LIMIT_ASSISTANCE,
+                )
+                return [], state, False, 429, error
+
+            log_msg += "However no longer in rate limiting period, so task can be executed..."
+            del state[self.RATE_LIMIT_DATETIME]
+            self.logger.info(log_msg)
+
+    def check_rate_limit_error(
+        self, error: ApiClientException, response: Response, status_code: int, state: dict
+    ) -> Tuple[int, ApiClientException]:
+        if status_code == 429:
+            rate_limit_response_time = response.headers.get("X-RateLimit-Reset")
+            if rate_limit_response_time:
+                new_run_time = time() + (rate_limit_response_time / 1000)
+            else:
+                new_run_time = time() + 300
+            new_run_time_string = Utils.convert_epoch_to_readable(new_run_time)
+            self.logger.error(f"A rate limit error has occurred, task will resume after {new_run_time_string}")
+            state[self.RATE_LIMIT_DATETIME] = new_run_time
+            return 200, None, True
+        return status_code, error, False

plugins/mimecast/komand_mimecast/util/api.py

@@ -168,11 +168,14 @@ def _is_last_token(self, request: requests.Response) -> bool:
         except json.JSONDecodeError:
             return False
 
-    def _check_rate_limiting(self, request):
+    def _check_rate_limiting(self, response):
         rate_limit_status_code = 429
-        if request.status_code == rate_limit_status_code:
+        if response.status_code == rate_limit_status_code:
             raise ApiClientException(
-                preset=PluginException.Preset.RATE_LIMIT, status_code=rate_limit_status_code, data=request.text
+                preset=PluginException.Preset.RATE_LIMIT,
+                status_code=rate_limit_status_code,
+                data=response.text,
+                response=response,
             )
 
     def _handle_status_code_response(self, response: requests.request, status_code: int):

plugins/mimecast/komand_mimecast/util/constants.py

@@ -55,3 +55,4 @@
     VALIDATION_INVALID_EMAIL_ADDRESS_ERROR: "Email address is not valid.",
     INVALID_SIEM_TOKEN: "The SIEM token is invalid",
 }
+RATE_LIMIT_ASSISTANCE = "Task will resume collection of logs after the rate limiting period has expired."

plugins/mimecast/komand_mimecast/util/exceptions.py

@@ -2,6 +2,7 @@
 
 
 class ApiClientException(PluginException):
-    def __init__(self, cause=None, assistance=None, data=None, preset=None, status_code=None):
+    def __init__(self, cause=None, assistance=None, data=None, preset=None, status_code=None, response=None):
         super().__init__(cause, assistance, data, preset)
         self.status_code = status_code
+        self.response = response

plugins/mimecast/komand_mimecast/util/util.py

@@ -1,9 +1,14 @@
 from typing import Dict, Any, Union
+from datetime import datetime
 
 from komand_mimecast.util.constants import BASE_HOSTNAME_MAP, DEFAULT_REGION
 
 
 class Utils:
+    @staticmethod
+    def convert_epoch_to_readable(epoch_time: int) -> str:
+        return datetime.utcfromtimestamp(epoch_time).strftime("%Y-%m-%d %H:%M:%S")
+
     @staticmethod
     def prepare_base_url(region: str = DEFAULT_REGION) -> str:
         if region == "USBCOM":

plugins/mimecast/plugin.spec.yaml

@@ -17,15 +17,15 @@ links:
  - "[Mimecast](http://mimecast.com)"
 references:
  - "[Mimecast API](https://www.mimecast.com/developer/documentation)"
-version: 5.3.18
+version: 5.3.19
 connection_version: 5
 supported_versions: ["Mimecast API 2024-06-18"]
 vendor: rapid7
 support: rapid7
 cloud_ready: true
 sdk:
   type: slim
-  version: 6.1.2
+  version: 6.2.0
   user: nobody
 status: []
 resources:
@@ -40,6 +40,7 @@ hub_tags:
   keywords: [mimecast, email, cloud_enabled]
   features: []
 version_history:
+- "5.3.19 - Update Task `monitor_siem_logs` to delay retry if a rate limit error is returned from Mimecast | Update SDK to version 6.2.0"
 - "5.3.18 - Fix task connection test | Trim whitespace from connection inputs | bump SDK to version 6.1.2"
 - "5.3.17 - Task `monitor_siem_logs` update the mapping used for the USBCOM region"
 - "5.3.16 - Task `monitor_siem_logs` Limit the number of events per run to 7500 | bump SDK to version 6.1.0"

plugins/mimecast/requirements.txt

@@ -5,6 +5,6 @@ validators==0.21.0
 parameterized==0.8.1
 python-dateutil==2.7
 jsonschema==3.2.0
-werkzeug==3.0.3
+werkzeug==3.0.6
 setuptools==70.0.0
 freezegun==1.5.1

plugins/mimecast/setup.py

@@ -3,7 +3,7 @@
 
 
 setup(name="mimecast-rapid7-plugin",
-      version="5.3.18",
+      version="5.3.19",
       description="[Mimecast](https://www.mimecast.com) is a set of cloud services designed to provide next generation protection against advanced email-borne threats such as malicious URLs, malware, impersonation attacks, as well as internally generated threats, with a focus on email security. This plugin utilizes the [Mimecast API](https://www.mimecast.com/developer/documentation)",
       author="rapid7",
       author_email="",

plugins/mimecast/unit_test/test_monitor_siem_logs.py

@@ -64,6 +64,17 @@ def test_monitor_siem_logs_raises_401(self, _mock_data):
         self.assertEqual(new_state, state_params)  # we shouldn't change the state if we encounter an error
         self.assertEqual(type(error), ApiClientException)
 
+    def test_monitor_siem_logs_raises_429(self, _mock_data):
+        state_params = {"next_token": "force_429", "last_log_line": 0}
+        expected_state = state_params.copy()
+        expected_state.update({"rate_limit_datetime": 1641039000.0})
+        response, new_state, has_more_pages, status_code, error = self.task.run(params={}, state=state_params)
+        self.assertEqual(status_code, 200)
+        self.assertEqual(has_more_pages, True)
+        self.assertEqual(response, [])
+        self.assertEqual(new_state, expected_state)
+        self.assertEqual(error, None)
+
     @patch("logging.Logger.error")
     def test_monitor_siem_logs_stops_path_traversal(self, mock_logger, _mock_data):
         test_state = {

plugins/mimecast/unit_test/util.py

@@ -76,11 +76,12 @@ def status_code(self):
                 return 200
 
         class MockResponseZip:
-            def __init__(self, status_code, content, headers, json):
+            def __init__(self, status_code, content, headers, json, text=""):
                 self.status_code = status_code
                 self.content = content
                 self.headers_value = headers
                 self.json_value = json
+                self.text = text
 
             @property
             def headers(self):
@@ -170,6 +171,23 @@ def json(self):
                     ],
                 }
                 resp = MockResponseZip(401, b"", {}, json.dumps(json_value))
+            if "force_429" in data:
+                json_value = {
+                    "meta": {"isLastToken": False, "status": 429},
+                    "fail": [
+                        {
+                            "errors": [
+                                {
+                                    "code": "err_xdk_invalid_signature",
+                                    "message": "0004 Invalid Signature",
+                                    "retryable": False,
+                                }
+                            ]
+                        }
+                    ],
+                }
+                headers = {"X-RateLimit-Reset": 600000}
+                resp = MockResponseZip(429, b"", headers, json.dumps(json_value))
             elif "force_json" in data:
                 resp = MockResponseZip(200, b'{ "type" : "MTA", "data" : ', headers, '{"meta": {"status": 200}}')
             elif "force_single_json_error" in data: