[SOAR-15940] InsightIDR - advanced_query_on_log_set statistical resul…

…t bug fix (#2108) * SOAR-15940-Fixing issue where statistical calls to advanced_query_on_log_set was always coming back as 0.0 * SOAR-15940-updated unit tests for advanced_query_on_log_set to include schema validation * SOAR-15940-updated unit tests for advanced_query_on_log_set to include schema validation * SOAR-15940-updating count to better reflect the total items matched * SOAR-15940-updating count to better reflect the total items matched * SOAR-15940-updating unit test to reflect change to count * SOAR-15940-adding more specific error handling
rapid7 · Nov 10, 2023 · 34642e7 · 34642e7
1 parent b38a5d5
commit 34642e7
Show file tree

Hide file tree

Showing 13 changed files with 572 additions and 37 deletions.
diff --git a/plugins/rapid7_insightidr/.CHECKSUM b/plugins/rapid7_insightidr/.CHECKSUM
@@ -1,6 +1,6 @@
 {
-	"spec": "e88e03be591a773f6f06ab8f8ff72d75",
-	"manifest": "fcaf023d3d8e468094348803d3ecc5eb",
+	"spec": "0f975db262dadb06d0a90e39fc05434a",
+	"manifest": "f6c902be173f84e632cafb5b3093432e",
 	"setup": "3b83b99c77061338b639889ef7848b9b",
 	"schemas": [
 		{

diff --git a/plugins/rapid7_insightidr/bin/komand_rapid7_insightidr b/plugins/rapid7_insightidr/bin/komand_rapid7_insightidr
@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Rapid7 InsightIDR"
 Vendor = "rapid7"
-Version = "6.0.1"
+Version = "6.0.2"
 Description = "This plugin allows you to add indicators to a threat and see the status of investigations"
 
 

diff --git a/plugins/rapid7_insightidr/help.md b/plugins/rapid7_insightidr/help.md
@@ -2024,7 +2024,8 @@ Example output:
 
 # Version History
 
-* 6.0.1 - Action: `Advanced Query On Log Set` - Up the maximium events returned from 50 to 500
+* 6.0.2 - Action: `Advanced Query On Log set` - Fixed error where statistical queries would always return 0.0
+* 6.0.1 - Action: `Advanced Query On Log` - Increase the maximum results returned from 50 to 500
 * 6.0.0 - Action: `Advanced Query On Log Set` - Add new output type for statistical queries.
 * 5.1.2 - Action: `Advanced Query on Log Set` - Fix JSONDecoderError | Action: `Query` - Update spec and help.md to show it queries log IDs, not query IDs
 * 5.1.1 - Action: `List Investigations` - Now receiving size input | Actions: `Advanced Query On Log` & `Advanced Query On Log Set` - Acronym LQL has been updated to LEQL

diff --git a/...ns/rapid7_insightidr/komand_rapid7_insightidr/actions/advanced_query_on_log_set/action.py b/...ns/rapid7_insightidr/komand_rapid7_insightidr/actions/advanced_query_on_log_set/action.py
@@ -6,6 +6,7 @@
 from komand_rapid7_insightidr.util.resource_helper import ResourceHelper
 from insightconnect_plugin_runtime.exceptions import PluginException
 from komand_rapid7_insightidr.util.parse_dates import parse_dates
+from requests import HTTPError
 
 
 class AdvancedQueryOnLogSet(insightconnect_plugin_runtime.Action):
@@ -57,7 +58,10 @@ def run(self, params={}):
         if not statistical:
             return {Output.RESULTS_EVENTS: log_entries, Output.COUNT: len(log_entries)}
         else:
-            return {Output.RESULTS_STATISTICAL: log_entries, Output.COUNT: len(log_entries)}
+            return {
+                Output.RESULTS_STATISTICAL: log_entries,
+                Output.COUNT: log_entries.get("search_stats", {}).get("events_matched", 0),
+            }
 
     @staticmethod
     def parse_query_for_statistical(query: str) -> bool:
@@ -173,7 +177,18 @@ def maybe_get_log_entries(
 
         results_object = response.json()
         if statistical:
-            potential_results = results_object.get("partial")
+            stats_endpoint = f"{self.connection.url}log_search/query/{results_object.get('id', '')}"
+            self.logger.info(f"Getting statistical from: {stats_endpoint}")
+            stats_response = self.connection.session.get(stats_endpoint, params=params)
+            try:
+                stats_response.raise_for_status()
+            except HTTPError as error:
+                raise PluginException(
+                    cause="Failed to get log sets from InsightIDR\n",
+                    assistance=f"Could not get statistical info from: {stats_endpoint}\n",
+                    data=f"{stats_response.text}, {error}",
+                )
+            potential_results = stats_response.json()
         else:
             potential_results = results_object.get("events")
 

diff --git a/plugins/rapid7_insightidr/plugin.spec.yaml b/plugins/rapid7_insightidr/plugin.spec.yaml
@@ -4,7 +4,7 @@ products: [insightconnect]
 name: rapid7_insightidr
 title: "Rapid7 InsightIDR"
 description: "This plugin allows you to add indicators to a threat and see the status of investigations"
-version: 6.0.1
+version: 6.0.2
 connection_version: 5
 supported_versions: ["Latest release successfully tested on 2022-07-20."]
 vendor: rapid7

diff --git a/plugins/rapid7_insightidr/setup.py b/plugins/rapid7_insightidr/setup.py
@@ -3,7 +3,7 @@
 
 
 setup(name="rapid7_insightidr-rapid7-plugin",
-      version="6.0.1",
+      version="6.0.2",
       description="This plugin allows you to add indicators to a threat and see the status of investigations",
       author="rapid7",
       author_email="",

diff --git a/plugins/rapid7_insightidr/unit_test/payloads/log_id5.json.resp b/plugins/rapid7_insightidr/unit_test/payloads/log_id5.json.resp
@@ -0,0 +1,74 @@
+{
+  "logs": [
+    "553048ff-e6ab-4597-a3e0-2b24032c233e",
+    "3244ed07-c3af-4bee-90b5-905a38a034b4"
+  ],
+  "progress": 0,
+  "events": [],
+  "partial": {
+    "cardinality": 0,
+    "granularity": 4320000,
+    "from": 1699567413000,
+    "to": 1699610613000,
+    "type": "count",
+    "stats": {
+      "global_timeseries": {
+        "count": 0
+      }
+    },
+    "groups": [],
+    "others": {},
+    "status": 200,
+    "timeseries": {
+      "global_timeseries": [
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        },
+        {
+          "count": 0
+        }
+      ]
+    },
+    "groups_timeseries": [],
+    "all_exact_result": null,
+    "count": 0
+  },
+  "id": "7176face-f659-45a6-bd46-81fdc2b1f74b:1:85f959aecc0a7300f9ad93ddacf3454e36348348::bafefd2e9cf60c699529d5a8cf4493578b0b56dd:",
+  "links": [
+    {
+      "rel": "Self",
+      "href": "https://us.api.insight.rapid7.com/log_search/query/7176face-f659-45a6-bd46-81fdc2b1f74b:1:85f959aecc0a7300f9ad93ddacf3454e36348348::bafefd2e9cf60c699529d5a8cf4493578b0b56dd:"
+    }
+  ],
+  "leql": {
+    "statement": "where(hostname='WindowsX64') calculate(count)",
+    "during": {
+      "from": 1699567413000,
+      "to": 1699610613000
+    }
+  }
+}
diff --git a/plugins/rapid7_insightidr/unit_test/payloads/log_id6.json.resp b/plugins/rapid7_insightidr/unit_test/payloads/log_id6.json.resp
@@ -0,0 +1,74 @@
+{
+  "logs": [
+    "553048ff-e6ab-4597-a3e0-2b24032c233e",
+    "3244ed07-c3af-4bee-90b5-905a38a034b4"
+  ],
+  "statistics": {
+    "cardinality": 0,
+    "granularity": 4320000,
+    "from": 1699567413000,
+    "to": 1699610613000,
+    "type": "count",
+    "stats": {
+      "global_timeseries": {
+        "count": 462
+      }
+    },
+    "groups": [],
+    "others": {},
+    "status": 200,
+    "timeseries": {
+      "global_timeseries": [
+        {
+          "count": 38
+        },
+        {
+          "count": 47
+        },
+        {
+          "count": 36
+        },
+        {
+          "count": 56
+        },
+        {
+          "count": 60
+        },
+        {
+          "count": 40
+        },
+        {
+          "count": 39
+        },
+        {
+          "count": 41
+        },
+        {
+          "count": 61
+        },
+        {
+          "count": 44
+        }
+      ]
+    },
+    "groups_timeseries": [],
+    "all_exact_result": null,
+    "count": 462
+  },
+  "search_stats": {
+    "bytes_checked": 3589232,
+    "index_factor": 0.4294746,
+    "events_matched": 462,
+    "events_checked": 595,
+    "duration_ms": 19,
+    "events_all": 1025,
+    "bytes_all": 6291099
+  },
+  "leql": {
+    "statement": "where(hostname='WindowsX64') calculate(count)",
+    "during": {
+      "from": 1699567413000,
+      "to": 1699610613000
+    }
+  }
+}
diff --git a/plugins/rapid7_insightidr/unit_test/payloads/log_id7.json.resp b/plugins/rapid7_insightidr/unit_test/payloads/log_id7.json.resp
@@ -0,0 +1,36 @@
+{
+  "logs": [
+    "553048ff-e6ab-4597-a3e0-2b24032c233e",
+    "3244ed07-c3af-4bee-90b5-905a38a034b4"
+  ],
+  "progress": 0,
+  "events": [],
+  "partial": {
+    "cardinality": 0,
+    "granularity": 4320000,
+    "from": 1699569260000,
+    "to": 1699612460000,
+    "type": "count",
+    "stats": {},
+    "groups": [],
+    "others": {},
+    "status": 204,
+    "timeseries": {},
+    "groups_timeseries": [],
+    "all_exact_result": true
+  },
+  "id": "b55a768a-4c0e-449a-84ae-adc70e18eb20:1:8956245765c23b46f1d20322e5c076e53a7ab662::d56c37ab761ff5e65950abd93476f02ffbcb5a45:",
+  "links": [
+    {
+      "rel": "Self",
+      "href": "https://us.api.insight.rapid7.com/log_search/query/b55a768a-4c0e-449a-84ae-adc70e18eb20:1:8956245765c23b46f1d20322e5c076e53a7ab662::d56c37ab761ff5e65950abd93476f02ffbcb5a45:"
+    }
+  ],
+  "leql": {
+    "statement": "groupby(r7_context.asset.name)",
+    "during": {
+      "from": 1699569260000,
+      "to": 1699612460000
+    }
+  }
+}