supporting telemetry logs extraction

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py

@@ -51,18 +51,22 @@
 import loganalyze.logger as log
 
 LOGS_TO_EXTRACT = [
-    "event.log",
-    "ufmhealth.log",
-    "ufm.log",
-    "ibdiagnet2.log",
-    "console.log",
-    "rest_api.log"
+    # "event.log",
+    # "ufmhealth.log",
+    # "ufm.log",
+    # "ibdiagnet2.log",
+    # "console.log",
+    "rest_api.log",
+    "ufm_logs/ibdiagnet2_port_counters.log",
+    "secondary_telemetry/ibdiagnet2_port_counters.log"
 ]
 
 DIRECTORIES_TO_EXTRACT = [
     "telemetry_samples"
 ]
-
+DUPLICATE_LOGS_PATHS = {
+    "ibdiagnet2_port_counters.log":{"ufm_logs", "secondary_telemetry"}
+}
 def run_both_functions(parser_func, action_func, save_func):
     parser_func(action_func)
     save_func()

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py

@@ -125,7 +125,10 @@ def full_analysis(self):
         Run all the analysis and returns a list of all the graphs created and their title
         """
         for func in self._funcs_for_analysis:
-            func()
+            try:
+                func()
+            except:
+                print(":(")
         return self._images_created if len(self._images_created) > 0 else []
 
 

plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py

@@ -9,9 +9,10 @@
 # This software product is governed by the End User License Agreement
 # provided with the software product.
 #
+import os
 from pathlib import Path
 from abc import abstractmethod
-from typing import List
+from typing import List, Set
 
 
 class BaseExtractor:
@@ -27,6 +28,21 @@ def is_exists_get_as_path(self, location) -> Path:
             return location
         return None
 
+    @staticmethod
+    def _split_based_on_dir(files:Set[str]):
+        single_name_logs = set()
+        logs_with_dirs = {}
+        for log_name in files:
+            dir_name = os.path.dirname(log_name)
+            base_name = os.path.basename(log_name)
+            if dir_name:
+                if dir_name not in logs_with_dirs:
+                    logs_with_dirs[dir_name] = set()
+                logs_with_dirs[dir_name].add(base_name)
+            else:
+                single_name_logs.add(base_name)
+        return single_name_logs, logs_with_dirs
+
     @abstractmethod
     def extract_files(self, files_to_extract: List[str],
                       directories_to_extract: List[str],

plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py

@@ -30,20 +30,22 @@ def extract_files(self, files_to_extract: List[str],
         if not os.path.exists(destination):
             os.makedirs(destination)
 
-        # Convert the list to a set for faster lookup
         files_to_extract = set(files_to_extract)
         directories_to_extract = set(directories_to_extract)
         found_files = set()
         not_found_files = set(files_to_extract)
+        single_log_name, logs_with_dirs = self._split_based_on_dir(files_to_extract)
 
-        # Traverse the source directory and its subdirectories
         for root, _, files in os.walk(self.dir_path):
             for file_name in files:
                 full_dir_name = os.path.dirname(file_name)
                 last_dir_name = os.path.basename(full_dir_name)
-                if file_name in files_to_extract or last_dir_name in directories_to_extract:
+                is_logs_with_dir_flag = last_dir_name in logs_with_dirs and file_name in logs_with_dirs[last_dir_name]
+                if file_name in files_to_extract or last_dir_name in directories_to_extract or\
+                    is_logs_with_dir_flag:
                     src_file_path = os.path.join(root, file_name)
-                    dest_file_path = os.path.join(destination, file_name)
+                    new_file_name = f"{last_dir_name}_{file_name}" if is_logs_with_dir_flag else file_name
+                    dest_file_path = os.path.join(destination, new_file_name)
                     shutil.copy2(src_file_path, dest_file_path)
                     found_files.add(dest_file_path)
                     not_found_files.discard(file_name)

plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py

@@ -16,7 +16,7 @@
 from tarfile import TarFile
 import tarfile
 import os
-from typing import List, Set
+from typing import List, Set, Tuple
 
 from loganalyze.logs_extraction.base_extractor import BaseExtractor
 from loganalyze.utils.common import delete_folders
@@ -25,6 +25,9 @@
 LOGS_GZ_POSTFIX = ".gz"
 GZIP_MAGIC_NUMBER = b"\x1f\x8b"  # Magic number to understand if a file is really a gzip
 
+DUPLICATE_LOGS_PATHS = {
+    "ibdiagnet2_port_counters.log":{"ufm_logs", "secondary_telemetry"}
+}
 
 class DumpFilesExtractor(BaseExtractor):
     def __init__(self, dump_path: Path) -> None:
@@ -34,7 +37,7 @@ def __init__(self, dump_path: Path) -> None:
             self.directory = dump_path.parent
         else:
             raise FileNotFoundError(f"Could not use {dump_path}, make sure it exists and a tar")
-
+    
     def _get_files_from_tar(
         self, opened_file: TarFile,
         files_to_extract: Set[str],
@@ -44,12 +47,19 @@ def _get_files_from_tar(
         files_went_over = set()
         failed_extract = set()
         folders_to_remove = set()
+        single_log_name, logs_with_dirs = self._split_based_on_dir(files_to_extract)
         for member in opened_file:
             base_name = os.path.basename(member.name)
-            dir_name = os.path.dirname(member.name)
-            if base_name in files_to_extract or \
-                os.path.basename(dir_name) in directories_to_extract:
+            full_dir_path = os.path.dirname(member.name)
+            parent_dir_name = os.path.basename(full_dir_path)
+            original_base_name = base_name
+            is_logs_with_dir_flag = parent_dir_name in logs_with_dirs and base_name in logs_with_dirs[parent_dir_name]
+            if base_name in single_log_name or \
+                parent_dir_name in directories_to_extract or \
+                is_logs_with_dir_flag:
                 try:
+                    if is_logs_with_dir_flag:
+                        base_name = f"{parent_dir_name}_{base_name}"
                     opened_file.extract(member, path=destination)
                     extracted_file_path = os.path.join(destination, str(member.path))
                     log.LOGGER.debug(f"Extracted {base_name}")
@@ -63,6 +73,11 @@ def _get_files_from_tar(
                     files_went_over.add(base_name)
                     if base_name in files_to_extract:
                         files_to_extract.remove(base_name)
+                    elif is_logs_with_dir_flag:
+                        logs_with_dirs[parent_dir_name].discard(original_base_name)
+                        if len(logs_with_dirs[parent_dir_name]) == 0:
+                            del logs_with_dirs[parent_dir_name]
+
         files_extracted = files_went_over.difference(failed_extract)
         # When extracting the files from the tar, they are also taken with their
         # directories from inside the tar, there is no way to only take the file