From 4ce7184a0cfc44ff89004c236613e4968f3aa678 Mon Sep 17 00:00:00 2001 From: Boaz Haim Date: Tue, 5 Nov 2024 21:35:23 +0200 Subject: [PATCH] supporting telemetry logs extraction --- .../src/loganalyze/log_analyzer.py | 18 +++++++------ .../loganalyze/log_analyzers/base_analyzer.py | 5 +++- .../logs_extraction/base_extractor.py | 18 ++++++++++++- .../logs_extraction/directory_extractor.py | 10 +++++--- .../logs_extraction/tar_extractor.py | 25 +++++++++++++++---- 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py index 1cdece7b..52b17b11 100755 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py @@ -51,18 +51,22 @@ import loganalyze.logger as log LOGS_TO_EXTRACT = [ - "event.log", - "ufmhealth.log", - "ufm.log", - "ibdiagnet2.log", - "console.log", - "rest_api.log" + # "event.log", + # "ufmhealth.log", + # "ufm.log", + # "ibdiagnet2.log", + # "console.log", + "rest_api.log", + "ufm_logs/ibdiagnet2_port_counters.log", + "secondary_telemetry/ibdiagnet2_port_counters.log" ] DIRECTORIES_TO_EXTRACT = [ "telemetry_samples" ] - +DUPLICATE_LOGS_PATHS = { + "ibdiagnet2_port_counters.log":{"ufm_logs", "secondary_telemetry"} +} def run_both_functions(parser_func, action_func, save_func): parser_func(action_func) save_func() diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py index ecffb517..f186c653 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py @@ -125,7 +125,10 @@ def full_analysis(self): Run all the analysis and returns a list of all the graphs created and their title """ for func in self._funcs_for_analysis: - func() + try: + func() + except: + print(":(") return self._images_created if len(self._images_created) > 0 else [] diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py index d778d054..eb8c36d3 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py @@ -9,9 +9,10 @@ # This software product is governed by the End User License Agreement # provided with the software product. # +import os from pathlib import Path from abc import abstractmethod -from typing import List +from typing import List, Set class BaseExtractor: @@ -27,6 +28,21 @@ def is_exists_get_as_path(self, location) -> Path: return location return None + @staticmethod + def _split_based_on_dir(files:Set[str]): + single_name_logs = set() + logs_with_dirs = {} + for log_name in files: + dir_name = os.path.dirname(log_name) + base_name = os.path.basename(log_name) + if dir_name: + if dir_name not in logs_with_dirs: + logs_with_dirs[dir_name] = set() + logs_with_dirs[dir_name].add(base_name) + else: + single_name_logs.add(base_name) + return single_name_logs, logs_with_dirs + @abstractmethod def extract_files(self, files_to_extract: List[str], directories_to_extract: List[str], diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py index 08604515..d7f916c6 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py @@ -30,20 +30,22 @@ def extract_files(self, files_to_extract: List[str], if not os.path.exists(destination): os.makedirs(destination) - # Convert the list to a set for faster lookup files_to_extract = set(files_to_extract) directories_to_extract = set(directories_to_extract) found_files = set() not_found_files = set(files_to_extract) + single_log_name, logs_with_dirs = self._split_based_on_dir(files_to_extract) - # Traverse the source directory and its subdirectories for root, _, files in os.walk(self.dir_path): for file_name in files: full_dir_name = os.path.dirname(file_name) last_dir_name = os.path.basename(full_dir_name) - if file_name in files_to_extract or last_dir_name in directories_to_extract: + is_logs_with_dir_flag = last_dir_name in logs_with_dirs and file_name in logs_with_dirs[last_dir_name] + if file_name in files_to_extract or last_dir_name in directories_to_extract or\ + is_logs_with_dir_flag: src_file_path = os.path.join(root, file_name) - dest_file_path = os.path.join(destination, file_name) + new_file_name = f"{last_dir_name}_{file_name}" if is_logs_with_dir_flag else file_name + dest_file_path = os.path.join(destination, new_file_name) shutil.copy2(src_file_path, dest_file_path) found_files.add(dest_file_path) not_found_files.discard(file_name) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py index a0730a8b..f2640112 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py @@ -16,7 +16,7 @@ from tarfile import TarFile import tarfile import os -from typing import List, Set +from typing import List, Set, Tuple from loganalyze.logs_extraction.base_extractor import BaseExtractor from loganalyze.utils.common import delete_folders @@ -25,6 +25,9 @@ LOGS_GZ_POSTFIX = ".gz" GZIP_MAGIC_NUMBER = b"\x1f\x8b" # Magic number to understand if a file is really a gzip +DUPLICATE_LOGS_PATHS = { + "ibdiagnet2_port_counters.log":{"ufm_logs", "secondary_telemetry"} +} class DumpFilesExtractor(BaseExtractor): def __init__(self, dump_path: Path) -> None: @@ -34,7 +37,7 @@ def __init__(self, dump_path: Path) -> None: self.directory = dump_path.parent else: raise FileNotFoundError(f"Could not use {dump_path}, make sure it exists and a tar") - + def _get_files_from_tar( self, opened_file: TarFile, files_to_extract: Set[str], @@ -44,12 +47,19 @@ def _get_files_from_tar( files_went_over = set() failed_extract = set() folders_to_remove = set() + single_log_name, logs_with_dirs = self._split_based_on_dir(files_to_extract) for member in opened_file: base_name = os.path.basename(member.name) - dir_name = os.path.dirname(member.name) - if base_name in files_to_extract or \ - os.path.basename(dir_name) in directories_to_extract: + full_dir_path = os.path.dirname(member.name) + parent_dir_name = os.path.basename(full_dir_path) + original_base_name = base_name + is_logs_with_dir_flag = parent_dir_name in logs_with_dirs and base_name in logs_with_dirs[parent_dir_name] + if base_name in single_log_name or \ + parent_dir_name in directories_to_extract or \ + is_logs_with_dir_flag: try: + if is_logs_with_dir_flag: + base_name = f"{parent_dir_name}_{base_name}" opened_file.extract(member, path=destination) extracted_file_path = os.path.join(destination, str(member.path)) log.LOGGER.debug(f"Extracted {base_name}") @@ -63,6 +73,11 @@ def _get_files_from_tar( files_went_over.add(base_name) if base_name in files_to_extract: files_to_extract.remove(base_name) + elif is_logs_with_dir_flag: + logs_with_dirs[parent_dir_name].discard(original_base_name) + if len(logs_with_dirs[parent_dir_name]) == 0: + del logs_with_dirs[parent_dir_name] + files_extracted = files_went_over.difference(failed_extract) # When extracting the files from the tar, they are also taken with their # directories from inside the tar, there is no way to only take the file