Analyzing telemtry - for now

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py

@@ -44,6 +44,7 @@
 from loganalyze.log_analyzers.console_log_analyzer import ConsoleLogAnalyzer
 from loganalyze.log_analyzers.rest_api_log_analyzer import RestApiAnalyzer
 from loganalyze.log_analyzers.link_flapping_analyzer import LinkFlappingAnalyzer
+from loganalyze.log_analyzers.ibdiagnet2_port_counters_analyzer import Ibdiagnet2PortCountersAnalyzer
 
 from loganalyze.pdf_creator import PDFCreator
 from loganalyze.utils.common import delete_files_by_types
@@ -252,7 +253,7 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
     in the full report.
     Returns the created analyzer
     """
-    if log_name in full_extracted_logs_list:
+    if any(os.path.basename(log) == log_name for log in full_extracted_logs_list):
         log_csvs = get_files_in_dest_by_type(parsed_args.destination,
                                              log_name,
                                              parsed_args.extract_level)
@@ -305,7 +306,7 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
         log.LOGGER.debug("Starting analyzing the data")
         partial_create_analyzer = partial(create_analyzer,
                                           parsed_args=args,
-                                          full_extracted_logs_list=full_logs_list,
+                                          full_extracted_logs_list=logs_to_work_with,
                                           ufm_top_analyzer_obj=ufm_top_analyzer)
 
         # Creating the analyzer for each log
@@ -328,6 +329,12 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
 
         rest_api_log_analyzer = partial_create_analyzer(log_name="rest_api.log",
                                                         analyzer_clc=RestApiAnalyzer)
+
+        ibdianget_2_ports_primary_analyzer = partial_create_analyzer(log_name="ufm_logs_ibdiagnet2_port_counters.log",
+                                                                     analyzer_clc=Ibdiagnet2PortCountersAnalyzer)
+
+        ibdianget_2_ports_secondary_analyzer = partial_create_analyzer(log_name="ufm_logs_ibdiagnet2_port_counters.log",
+                                                                     analyzer_clc=Ibdiagnet2PortCountersAnalyzer)
         second_telemetry_samples = get_files_in_dest_by_type(args.destination,
                                                                  "secondary_",
                                                                  1000,
@@ -388,6 +395,11 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
         for image, title in images_and_title_to_present:
             log.LOGGER.info(f"{title}: {image}")
         log.LOGGER.info(f"Summary PDF was created! you can open here at {pdf_path}")
+
+        if args.interactive:
+            import IPython
+            IPython.embed()
+
         # Clean some unended files created during run
         files_types_to_delete = set()
         files_types_to_delete.add("png") #png images created for PDF report

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ibdiagnet2_port_counters_analyzer.py

@@ -0,0 +1,93 @@
+#
+# Copyright © 2013-2024 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# This software product is a proprietary product of Nvidia Corporation and its affiliates
+# (the "Company") and all right, title, and interest in and to the software
+# product, including all associated intellectual property rights, are and
+# shall remain exclusively with the Company.
+#
+# This software product is governed by the End User License Agreement
+# provided with the software product.
+#
+
+from typing import List
+
+import pandas as pd
+from loganalyze.log_analyzers.base_analyzer import BaseAnalyzer
+import loganalyze.logger as log
+
+class Ibdiagnet2PortCountersAnalyzer(BaseAnalyzer):
+    def __init__(self, logs_csvs: List[str], hours: int, dest_image_path: str, sort_timestamp=False):
+        super().__init__(logs_csvs, hours, dest_image_path, sort_timestamp)
+
+        # This will make all the extra colum are int
+        # Convert the 'extra' columns to integers if possible
+        extra_columns = ['extra1', 'extra2', 'extra3', 'extra4', 'extra5']
+
+        for col in extra_columns:
+            self._log_data_sorted[col] = pd.to_numeric(
+            self._log_data_sorted[col],
+            errors='coerce'
+            ).astype('Int64') 
+
+        # self._log_data_sorted['extra'] = (
+        # self._log_data_sorted['extra']
+        # .fillna(0)  # Replace NaN with 0
+        # .astype(int)  # Convert to integer
+        # )
+
+    def get_collectx_versions(self):
+            unique_collectx_versions = self._log_data_sorted[self._log_data_sorted['type'] == 'collectx_version']['data'].unique()
+            return unique_collectx_versions
+
+
+    def get_number_of_switches_and_ports(self):
+        """
+        Generate summary statistics for 'total_devices_ports' data.
+        This function calculates the average, maximum, minimum, and non-zero counts
+        for switches, CAs, routers, and ports.
+        """
+        # Step 1: Filter data for 'total_devices_ports'
+        filtered_data = self._log_data_sorted[self._log_data_sorted['type'] == 'total_devices_ports']
+
+        # Step 2: Create a combined column for 'extra1', 'extra3', and 'extra5'
+        combined_columns = ['extra1', 'extra3', 'extra5']
+        filtered_data['extra135'] = pd.to_numeric(
+            filtered_data[combined_columns].stack(), errors='coerce'
+        ).groupby(level=0).sum(min_count=1)
+
+        # Define columns of interest and their mapping to meaningful names
+        columns_of_interest = ['data', 'extra2', 'extra4', 'extra135']
+        column_mapping = {
+            'data': 'Number of Switches',
+            'extra2': 'CAs',
+            'extra4': 'Routers',
+            'extra135': 'Ports'
+        }
+
+        # Step 3: Initialize a list to store the summary statistics
+        summary_stats = []
+
+        # Step 4: Calculate statistics for each column
+        for col in columns_of_interest:
+            numeric_col = pd.to_numeric(filtered_data[col], errors='coerce')
+            non_zero_col = numeric_col[numeric_col != 0]
+
+            # Determine stats, defaulting to 0 if the column has no non-zero values
+            avg = int(round(non_zero_col.mean())) if not non_zero_col.empty else 0
+            max_val = int(non_zero_col.max()) if not non_zero_col.empty else 0
+            min_val = int(non_zero_col.min()) if not non_zero_col.empty else 0
+            count = int(non_zero_col.count())
+
+            summary_stats.append({
+                'Category': column_mapping.get(col, col),
+                'Average': avg,
+                'Maximum': max_val,
+                'Minimum': min_val,
+                'Total Rows (Non-Zero)': count
+            })
+
+        # Step 5: Convert the summary stats list into a DataFrame
+        summary_df = pd.DataFrame(summary_stats)
+
+        return summary_df

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/ibdiagnet2_port_counters_log_regex.py

@@ -20,28 +20,34 @@
 
 TIMEOUT_DUMP_CORE_REGEX = re.compile(r"^timeout: the monitored command dumped core$")
 
-TOTAL_SWITCH_PORTS_REGEX = re.compile(r"^.*Total switches\/ports \[(\d+)\/(\d+)\]\,.*$")
+TOTAL_SWITCH_PORTS_REGEX = re.compile(r"^.*Total switches\/ports \[(\d+)\/(\d+)\]\, CAs\/ports \[(\d+)\/(\d+)\]\, Routers\/ports \[(\d+)\/(\d+)\]\s*$")
 
 COLLECTX_VERSION_REGEX = re.compile(r"^\[ExportAPI\] Collectx version ([\d\.]+)$")
 
 def iteration_time(match: Match):
     iteration_time_sec = match.group(1)
     timestamp = match.group(2)
-    return ("iteration_time", timestamp, iteration_time_sec, None)
+    return ("iteration_time", timestamp, iteration_time_sec, None, None, None, None, None)
 
 def timeout_dump_core(_: Match):
-    return ("timeout_dump_core", None, None, None)
+    return ("timeout_dump_core", None, None, None, None, None, None, None)
 
 def total_switch_ports(match: Match):
     total_switches = match.group(1)
-    total_ports = match.group(2)
-    return ("total_switch_ports", None, total_switches, total_ports)
+    total_switch_ports = match.group(2)
+    total_cas = match.group(3)
+    total_cas_ports = match.group(4)
+    total_routers = match.group(5)
+    total_routers_ports = match.group(6)
+    return ("total_devices_ports", None, total_switches, total_switch_ports,\
+            total_cas, total_cas_ports,\
+            total_routers, total_routers_ports)
 
 def collectx_version(match:Match):
     collectx_version_str = match.group(1)
-    return ("collectx_version", None, collectx_version_str, None)
+    return ("collectx_version", None, collectx_version_str, None, None, None, None, None)
 
-ibdiagnet2_headers = ("type", "timestamp", "data", "extra")
+ibdiagnet2_headers = ("type", "timestamp", "data", "extra1", "extra2", "extra3", "extra4", "extra5")
 
 ibdiagnet2_primary_log_regex_cls = \
     RegexAndHandlers("ufm_logs_ibdiagnet2_port_counters.log", ibdiagnet2_headers)