issue:4050338 Add link flapping to log analyzer

.github/workflows/ufm_log_analyzer_ci_workflow.yml

@@ -2,23 +2,35 @@ name: Ufm log analyzer CI Workflow
 
 on: 
   push:
-     paths:
+    paths:
       - 'plugins/ufm_log_analyzer_plugin/**'
+      - '.github/workflows/ufm_log_analyzer_ci_workflow.yml'
+
 jobs:
   pylint:
     runs-on: ubuntu-latest
+
     steps:
     - name: Checkout code
       uses: actions/checkout@main
-    
+
     - name: Set up Python
       uses: actions/setup-python@main
       with:
-        python-version: 3.9  # Specify the Python version you want to use
-    
-    - name: Install dependencies
+        python-version: '3.9'
+
+    - name: Install dependencies and run PyLint
       run: |
-        pip install -r plugins/ufm_log_analyzer_plugin/src/loganalyze/requirements.txt
+        SCRIPT_DIR="plugins/ufm_log_analyzer_plugin"
+        # Set PYTHONPATH to include src directory and two levels up for utils
+        PYTHONPATH="$(realpath $SCRIPT_DIR/src):$(realpath $SCRIPT_DIR/../../)"
+        export PYTHONPATH
+
+        cd $SCRIPT_DIR
+
+        # Install dependencies
+        pip install -r src/loganalyze/requirements.txt
         pip install pylint
-    - name: Run PyLint
-      run: pylint --rcfile=plugins/ufm_log_analyzer_plugin/src/loganalyze/.pylintrc plugins/ufm_log_analyzer_plugin/src/loganalyze
+
+        # Run PyLint
+        pylint --rcfile=src/loganalyze/.pylintrc src/loganalyze

plugins/ufm_log_analyzer_plugin/log_analyzer.sh

@@ -16,6 +16,6 @@
 
 SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
 src_dir=$( realpath "${SCRIPT_DIR}/src" )
-export PYTHONPATH="${src_dir}"
+export PYTHONPATH="${src_dir}:../../" # The ../../ is to be able to use the utils
 
 python3 "${src_dir}/loganalyze/log_analyzer.py" "$@"

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py

@@ -32,7 +32,7 @@
 import matplotlib.pyplot as plt
 
 
-from loganalyze.log_analyzers.base_analyzer import BaseAnalyzer
+from loganalyze.log_analyzers.base_analyzer import BaseImageCreator
 from loganalyze.logs_extraction.directory_extractor import DirectoryExtractor
 from loganalyze.log_analyzers.ufm_top_analyzer import UFMTopAnalyzer
 from loganalyze.logs_extraction.tar_extractor import DumpFilesExtractor
@@ -46,6 +46,7 @@
 from loganalyze.log_analyzers.events_log_analyzer import EventsLogAnalyzer
 from loganalyze.log_analyzers.console_log_analyzer import ConsoleLogAnalyzer
 from loganalyze.log_analyzers.rest_api_log_analyzer import RestApiAnalyzer
+from loganalyze.log_analyzers.link_flapping_analyzer import LinkFlappingAnalyzer
 
 from loganalyze.pdf_creator import PDFCreator
 from loganalyze.utils.common import delete_files_by_types
@@ -61,6 +62,10 @@
     "rest_api.log"
 ]
 
+DIRECTORIES_TO_EXTRACT = [
+    "telemetry_samples"
+]
+
 def run_both_functions(parser_func, action_func, save_func):
     parser_func(action_func)
     save_func()
@@ -139,17 +144,20 @@ def sorting_logs(log_path):
     return count
 
 
-def get_csvs_in_dest(location: str, base_name: str, extraction_level: int):
+def get_files_in_dest_by_type(location: str,
+                              base_name: str,
+                              extraction_level: int,
+                              file_type="csv"):
     """
-    Return a list of all the CSV files that were parsed and part of the current
+    Return a list of all the files by type that were parsed and part of the current
     extraction level requested
     """
-    csv_files = glob.glob(os.path.join(location, "*.csv"))
-    matched_files = [file for file in csv_files if base_name in os.path.basename(file)]
+    files_by_type = glob.glob(os.path.join(location, f"*.{file_type}"))
+    matched_files = [file for file in files_by_type if base_name in os.path.basename(file)]
     full_paths = [os.path.abspath(file) for file in matched_files]
-    sorted_csvs = sorted(full_paths, key=sorting_logs)
-    sliced_csvs = sorted_csvs[: (extraction_level + 1)]
-    return sliced_csvs
+    sorted_files = sorted(full_paths, key=sorting_logs)
+    sliced_files = sorted_files[: (extraction_level + 1)]
+    return sliced_files
 
 
 def parse_args():
@@ -247,7 +255,9 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
     Returns the created analyzer
     """
     if log_name in full_extracted_logs_list:
-        log_csvs = get_csvs_in_dest(parsed_args.destination, log_name, parsed_args.extract_level)
+        log_csvs = get_files_in_dest_by_type(parsed_args.destination,
+                                             log_name,
+                                             parsed_args.extract_level)
         analyzer = analyzer_clc(log_csvs, parsed_args.hours, parsed_args.destination)
         ufm_top_analyzer_obj.add_analyzer(analyzer)
         return analyzer
@@ -275,7 +285,7 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
             extractor = DumpFilesExtractor(args.location)
 
         logs_to_work_with, failed_extract = extractor.extract_files(
-            full_logs_list, args.destination
+            full_logs_list, DIRECTORIES_TO_EXTRACT, args.destination
         )
 
         if len(failed_extract) > 0:
@@ -290,7 +300,7 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
 
 
         # Setting the time granularity for the graphs
-        BaseAnalyzer.time_interval = args.interval
+        BaseImageCreator.time_interval = args.interval
 
         # Analyze the CSV and be able to query the data
         start = time.perf_counter()
@@ -320,6 +330,13 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
 
         rest_api_log_analyzer = partial_create_analyzer(log_name="rest_api.log",
                                                         analyzer_clc=RestApiAnalyzer)
+        second_telemetry_sampled_csv = get_files_in_dest_by_type(args.destination,
+                                                                 "secondary_",
+                                                                 1000,
+                                                                 "gz")
+        links_flapping_analyzer = LinkFlappingAnalyzer(second_telemetry_sampled_csv,
+                                                       args.destination)
+        ufm_top_analyzer.add_analyzer(links_flapping_analyzer)
         end = time.perf_counter()
         log.LOGGER.debug(f"Took {end-start:.3f} to load the parsed data")
 
@@ -337,10 +354,14 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
         pdf_header = (
             f"Dump analysis for {os.path.basename(args.location)}, hours={args.hours}"
         )
-        fabric_info = ibdiagnet_analyzer.get_fabric_size() \
-                        if ibdiagnet_analyzer else "No Fabric Info found"
+        FABRIC_INFO = str(ibdiagnet_analyzer.get_fabric_size() \
+                        if ibdiagnet_analyzer else "No Fabric Info found")
+
+        LINK_FLAPPING = str(links_flapping_analyzer.get_link_flapping_last_week() \
+                            if links_flapping_analyzer else "No link flapping info")
         # PDF creator gets all the images and to add to the report
-        pdf = PDFCreator(pdf_path, pdf_header, png_images, fabric_info)
+        TEXT = FABRIC_INFO + os.linesep + "Link Flapping:" + os.linesep + LINK_FLAPPING
+        pdf = PDFCreator(pdf_path, pdf_header, png_images, TEXT)
         pdf.created_pdf()
         # Generated a report that can be located in the destination
         log.LOGGER.info("Analysis is done, please see the following outputs:")
@@ -351,7 +372,8 @@ def create_analyzer(parsed_args, full_extracted_logs_list,
         files_types_to_delete = set()
         files_types_to_delete.add("png") #png images created for PDF report
         files_types_to_delete.add("log") #logs taken from the logs
-        files_types_to_delete.add("gz") # Zipped logs taken from the logs
+        files_types_to_delete.add("csv") #tmp csv + telemetery samples
+        files_types_to_delete.add("gz") #gz files of logs and samples
         delete_files_by_types(args.destination, files_types_to_delete)
         if args.show_output:
             if platform.system() == "Windows":

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py

@@ -29,17 +29,96 @@
 warnings.filterwarnings("ignore")
 
 
-class BaseAnalyzer:
-    """
-    Analyzer class that gives all the logs the
-    ability to print/save images and filter data
-    """
-
+class BaseImageCreator:
     # Setting the graph time interval to 1 hour
     # This is out side of the constructor since
     # It is defined once for all graphic options
     # Per call/run.
+
     time_interval = "h"
+    _images_type = {"svg", "png"}
+    def __init__(self, dest_image_path):
+        self._dest_image_path = dest_image_path
+        self._images_created = []
+        self._funcs_for_analysis = set()
+
+    def _plot_and_save_data_based_on_timestamp(
+        self, data_to_plot, x_label, y_label, title
+    ):
+        with plt.ion():
+            log.LOGGER.debug(f"saving {title}")
+            plt.figure(figsize=(12, 6))
+            plt.plot(data_to_plot, marker="o", linestyle="-", color="b")
+            plt.title(title)
+            plt.xlabel(x_label)
+            plt.ylabel(y_label)
+            plt.grid(True)
+
+            # Set the locator to show ticks every hour and the formatter to
+            # include both date and time
+            ax = plt.gca()
+            ax.xaxis.set_major_locator(mdates.HourLocator())
+            ax.xaxis.set_minor_locator(
+                mdates.MinuteLocator(interval=15)
+            )  # Add minor ticks every 15 minutes
+            ax.xaxis.set_major_formatter(
+                mdates.DateFormatter("%Y-%m-%d %H:%M")
+            )  # Format major ticks to show date and time
+
+            plt.xticks(rotation=45)  # Rotate x-axis labels to make them readable
+            plt.tight_layout()
+
+            generic_file_name = f"{title}".replace(" ", "_").replace("/", "_")
+            images_created = []
+            for img_type in self._images_type:
+                cur_img = os.path.join(self._dest_image_path,f"{generic_file_name}.{img_type}")
+                log.LOGGER.debug(f"Saving {cur_img}")
+                plt.savefig(cur_img, format=img_type)
+                images_created.append(cur_img)
+            images_list_with_title = [(image, title) for image in images_created]
+            self._images_created.extend(images_list_with_title)
+            plt.close()
+
+    def _plot_and_save_pivot_data_in_bars(  # pylint: disable=# pylint: disable=too-many-arguments
+        self, pivoted_data, x_label, y_label, title, legend_title
+    ):
+        if pivoted_data.empty:
+            return
+        pivoted_data.plot(kind="bar", figsize=(14, 7))
+        # This allows for the image to keep open when we create another one
+        with plt.ion():
+            log.LOGGER.debug(f"saving {title}")
+            plt.title(title)
+            plt.xlabel(x_label)
+            plt.ylabel(y_label)
+            plt.legend(title=legend_title, bbox_to_anchor=(1.05, 1), loc="upper left")
+            plt.xticks(rotation=45)
+            plt.tight_layout()
+            generic_file_name = f"{title}".replace(" ", "_").replace("/", "_")
+            images_created = []
+            for img_type in self._images_type:
+                cur_img = os.path.join(self._dest_image_path,f"{generic_file_name}.{img_type}")
+                log.LOGGER.debug(f"Saving {cur_img}")
+                plt.savefig(cur_img, format=img_type)
+                images_created.append(cur_img)
+            images_list_with_title = [(image, title) for image in images_created]
+            self._images_created.extend(images_list_with_title)
+            plt.close()
+
+    def full_analysis(self):
+        """
+        Run all the analysis and returns a list of all the graphs created and their title
+        """
+        for func in self._funcs_for_analysis:
+            func()
+        return self._images_created if len(self._images_created) > 0 else []
+
+
+class BaseAnalyzer(BaseImageCreator):
+    """
+    Analyzer class that gives all the logs the
+    ability to print/save images and filter data
+    """
 
 
     def __init__(
@@ -50,7 +129,7 @@ def __init__(
         sort_timestamp=True
 
     ):
-        self._dest_image_path = dest_image_path
+        super().__init__(dest_image_path)
         dataframes = [pd.read_csv(ufm_log) for ufm_log in logs_csvs]
         df = pd.concat(dataframes, ignore_index=True)
         if sort_timestamp:
@@ -128,72 +207,3 @@ def fix_lines_with_no_timestamp(csvs):
 
                 # Replace the original file with the modified file
             os.replace(temp_file, csv_file)
-
-    def _plot_and_save_data_based_on_timestamp(
-        self, data_to_plot, x_label, y_label, title
-    ):
-        with plt.ion():
-            log.LOGGER.debug(f"saving {title}")
-            plt.figure(figsize=(12, 6))
-            plt.plot(data_to_plot, marker="o", linestyle="-", color="b")
-            plt.title(title)
-            plt.xlabel(x_label)
-            plt.ylabel(y_label)
-            plt.grid(True)
-
-            # Set the locator to show ticks every hour and the formatter to
-            # include both date and time
-            ax = plt.gca()
-            ax.xaxis.set_major_locator(mdates.HourLocator())
-            ax.xaxis.set_minor_locator(
-                mdates.MinuteLocator(interval=15)
-            )  # Add minor ticks every 15 minutes
-            ax.xaxis.set_major_formatter(
-                mdates.DateFormatter("%Y-%m-%d %H:%M")
-            )  # Format major ticks to show date and time
-
-            plt.xticks(rotation=45)  # Rotate x-axis labels to make them readable
-            plt.tight_layout()
-
-            generic_file_name = f"{title}".replace(" ", "_").replace("/", "_")
-            images_created = []
-            for img_type in self._images_type:
-                cur_img = os.path.join(self._dest_image_path,f"{generic_file_name}.{img_type}")
-                log.LOGGER.debug(f"Saving {cur_img}")
-                plt.savefig(cur_img, format=img_type)
-                images_created.append(cur_img)
-            images_list_with_title = [(image, title) for image in images_created]
-            plt.close()
-            return images_list_with_title
-
-    def _plot_and_save_pivot_data_in_bars(  # pylint: disable=# pylint: disable=too-many-arguments
-        self, pivoted_data, x_label, y_label, title, legend_title
-    ):
-        if pivoted_data.empty:
-            return []
-        pivoted_data.plot(kind="bar", figsize=(14, 7))
-        # This allows for the image to keep open when we create another one
-        with plt.ion():
-            log.LOGGER.debug(f"saving {title}")
-            plt.title(title)
-            plt.xlabel(x_label)
-            plt.ylabel(y_label)
-            plt.legend(title=legend_title, bbox_to_anchor=(1.05, 1), loc="upper left")
-            plt.xticks(rotation=45)
-            plt.tight_layout()
-            generic_file_name = f"{title}".replace(" ", "_").replace("/", "_")
-            images_created = []
-            for img_type in self._images_type:
-                cur_img = os.path.join(self._dest_image_path,f"{generic_file_name}.{img_type}")
-                log.LOGGER.debug(f"Saving {cur_img}")
-                plt.savefig(cur_img, format=img_type)
-                images_created.append(cur_img)
-            images_list_with_title = [(image, title) for image in images_created]
-            plt.close()
-            return images_list_with_title
-
-    def full_analysis(self):
-        """
-        Returns a list of all the graphs created and their title
-        """
-        log.LOGGER.debug("Missing implementation for full_analysis")

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/console_log_analyzer.py

@@ -25,6 +25,7 @@ def __init__(
         self.fix_lines_with_no_timestamp(logs_csvs)
         super().__init__(logs_csvs, hours, dest_image_path, sort_timestamp)
         self._log_data_sorted.dropna(subset=["data"], inplace=True)
+        self._funcs_for_analysis = {self.print_exceptions_per_time_count}
 
     def _get_exceptions(self):
         error_data = self._log_data_sorted[self._log_data_sorted["type"] == "Error"][
@@ -53,18 +54,16 @@ def print_exceptions(self):
     def print_exceptions_per_time_count(self):
         error_data = self._log_data_sorted[self._log_data_sorted["type"] == "Error"]
         errors_per_hour = error_data.groupby(DataConstants.AGGREGATIONTIME).size()
-        images_created = self._plot_and_save_data_based_on_timestamp(
+        self._plot_and_save_data_based_on_timestamp(
             errors_per_hour,
             "Time",
             "Amount of exceptions",
             "Exceptions count",
         )
-        return images_created
 
     def full_analysis(self):
         """
         Returns a list of all the graphs created and their title
         """
-        created_images = self.print_exceptions_per_time_count()
         self.print_exceptions()
-        return created_images if len(created_images) > 0 else []
+        return super().full_analysis()