From 22cf5b87913965b7117fcd7fcb92c2dd7db2fe0f Mon Sep 17 00:00:00 2001 From: boazhaim <160493207+boazhaim@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:42:35 +0200 Subject: [PATCH] task:4171696 -Log analyzer - Adding ruff to the ci (#282) * for-now * ruff-ci * Making ruff and pylint parllel --- .../ufm_log_analyzer_ci_workflow.yml | 23 +- .../src/loganalyze/log_analyzer.py | 232 +++++++++++------- .../loganalyze/log_analyzers/base_analyzer.py | 49 ++-- .../log_analyzers/console_log_analyzer.py | 17 +- .../src/loganalyze/log_analyzers/constants.py | 1 + .../log_analyzers/events_log_analyzer.py | 65 ++--- .../ibdiagnet2_port_counters_analyzer.py | 158 ++++++------ .../log_analyzers/link_flapping_analyzer.py | 89 ++++--- .../log_analyzers/rest_api_log_analyzer.py | 59 +++-- .../log_analyzers/ufm_health_analyzer.py | 6 +- .../log_analyzers/ufm_log_analyzer.py | 37 ++- .../log_parsing/console_log_regex.py | 6 +- .../ibdiagnet2_port_counters_log_regex.py | 87 +++++-- .../src/loganalyze/log_parsing/log_parser.py | 3 +- .../src/loganalyze/log_parsing/logs_regex.py | 8 +- .../log_parsing/rest_api_log_regex.py | 4 +- .../src/loganalyze/logger.py | 5 +- .../src/loganalyze/logs_csv/csv_handler.py | 1 + .../logs_extraction/base_extractor.py | 11 +- .../logs_extraction/directory_extractor.py | 35 ++- .../logs_extraction/tar_extractor.py | 47 ++-- .../src/loganalyze/pdf_creator.py | 18 +- .../src/loganalyze/utils/common.py | 11 +- 23 files changed, 602 insertions(+), 370 deletions(-) diff --git a/.github/workflows/ufm_log_analyzer_ci_workflow.yml b/.github/workflows/ufm_log_analyzer_ci_workflow.yml index 9311ee4d..e1e71243 100644 --- a/.github/workflows/ufm_log_analyzer_ci_workflow.yml +++ b/.github/workflows/ufm_log_analyzer_ci_workflow.yml @@ -28,9 +28,28 @@ jobs: cd $SCRIPT_DIR - # Install dependencies pip install -r src/loganalyze/requirements.txt pip install pylint==3.2.6 - # Run PyLint pylint --rcfile=src/loganalyze/.pylintrc src/loganalyze + + ruff: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@main + + - name: Set up Python + uses: actions/setup-python@main + with: + python-version: 3.9 + + - name: Install and run Ruff + run: | + SCRIPT_DIR="plugins/ufm_log_analyzer_plugin" + cd $SCRIPT_DIR + + pip install ruff + + ruff format --diff --check src/loganalyze diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py index 40a13290..bcdbcbfb 100755 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py @@ -44,8 +44,9 @@ from loganalyze.log_analyzers.console_log_analyzer import ConsoleLogAnalyzer from loganalyze.log_analyzers.rest_api_log_analyzer import RestApiAnalyzer from loganalyze.log_analyzers.link_flapping_analyzer import LinkFlappingAnalyzer -from loganalyze.log_analyzers.ibdiagnet2_port_counters_analyzer \ - import Ibdiagnet2PortCountersAnalyzer +from loganalyze.log_analyzers.ibdiagnet2_port_counters_analyzer import ( + Ibdiagnet2PortCountersAnalyzer, +) from loganalyze.pdf_creator import PDFCreator from loganalyze.utils.common import delete_files_by_types @@ -65,23 +66,22 @@ "console.log", "rest_api.log", "ufm_logs/ibdiagnet2_port_counters.log", - "secondary_telemetry/ibdiagnet2_port_counters.log" + "secondary_telemetry/ibdiagnet2_port_counters.log", ] -DIRECTORIES_TO_EXTRACT = [ - "telemetry_samples" -] +DIRECTORIES_TO_EXTRACT = ["telemetry_samples"] + def run_both_functions(parser_func, action_func, save_func): parser_func(action_func) save_func() -def create_parsers_processes(log_files_and_regex: List[ - Tuple[str, List[str], Callable[[Tuple], None], Callable] - ],): +def create_parsers_processes( + log_files_and_regex: List[Tuple[str, List[str], Callable[[Tuple], None], Callable]], +): """ - Per log files, creates the log parser process class that will + Per log files, creates the log parser process class that will handle that log. """ processes = [] @@ -99,7 +99,7 @@ def create_parsers_processes(log_files_and_regex: List[ return processes -def run_parsers_processes(processes:List[Process]): +def run_parsers_processes(processes: List[Process]): """ Runs all the parsing process and waits for the to finish """ @@ -132,8 +132,9 @@ def create_logs_regex_csv_handler_list(logs: Set[str]): else: csv_path = file_as_path.with_suffix(cur_path_suffix + ".csv") csv_handler = CsvHandler(csv_headers, csv_path) - result_list.append((path, patterns_and_fn, - csv_handler.add_line, csv_handler.save_file)) + result_list.append( + (path, patterns_and_fn, csv_handler.add_line, csv_handler.save_file) + ) return result_list @@ -150,16 +151,17 @@ def sorting_logs(log_path): return count -def get_files_in_dest_by_type(location: str, - base_name: str, - extraction_level: int, - file_type="csv"): +def get_files_in_dest_by_type( + location: str, base_name: str, extraction_level: int, file_type="csv" +): """ Return a list of all the files by type that were parsed and part of the current extraction level requested """ files_by_type = glob.glob(os.path.join(location, f"*.{file_type}")) - matched_files = [file for file in files_by_type if base_name in os.path.basename(file)] + matched_files = [ + file for file in files_by_type if base_name in os.path.basename(file) + ] full_paths = [os.path.abspath(file) for file in matched_files] sorted_files = sorted(full_paths, key=sorting_logs) sliced_files = sorted_files[: (extraction_level + 1)] @@ -206,24 +208,24 @@ def parse_args(): parser.add_argument( "--skip-tar-extract", action="store_true", - help="If the location is to an existing extracted tar, skip the " \ - "tar extraction and only copy the needed logs. Default is False" + help="If the location is to an existing extracted tar, skip the " + "tar extraction and only copy the needed logs. Default is False", ) parser.add_argument( - '--interval', + "--interval", type=str, - nargs='?', - default='1h', - choices=['1min', '10min', '1h', '24h'], + nargs="?", + default="1h", + choices=["1min", "10min", "1h", "24h"], help="Time interval for the graphs. Choices are: '1min'- Every minute, " - "'10min'- Every ten minutes, '1h'- Every one hour, " - "'24h'- Every 24 hours. Default is '1H'." + "'10min'- Every ten minutes, '1h'- Every one hour, " + "'24h'- Every 24 hours. Default is '1H'.", ) parser.add_argument( - '--log-level', + "--log-level", help="Tool log level, default is CRITICAL", - default='INFO', - choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], ) return parser.parse_args() @@ -246,8 +248,9 @@ def add_extraction_levels_to_files_set( return combined_logs_named -def create_analyzer(parsed_args, full_extracted_logs_list, - ufm_top_analyzer_obj, log_name, analyzer_clc): +def create_analyzer( + parsed_args, full_extracted_logs_list, ufm_top_analyzer_obj, log_name, analyzer_clc +): """ Create the analyzer based on the given inputs. Also adds it to the top_analyzer so it can be used @@ -256,9 +259,9 @@ def create_analyzer(parsed_args, full_extracted_logs_list, """ # Checking the base name since some logs in the list are with a directory name if any(os.path.basename(log) == log_name for log in full_extracted_logs_list): - log_csvs = get_files_in_dest_by_type(parsed_args.destination, - log_name, - parsed_args.extract_level) + log_csvs = get_files_in_dest_by_type( + parsed_args.destination, log_name, parsed_args.extract_level + ) analyzer = analyzer_clc(log_csvs, parsed_args.hours, parsed_args.destination) ufm_top_analyzer_obj.add_analyzer(analyzer) return analyzer @@ -268,8 +271,10 @@ def create_analyzer(parsed_args, full_extracted_logs_list, if __name__ == "__main__": args = parse_args() log.setup_logger("Logs_analyzer", args.log_level) - log.LOGGER.info("Starting analysis, this might take a few minutes" - " depending on the amount of data in the logs") + log.LOGGER.info( + "Starting analysis, this might take a few minutes" + " depending on the amount of data in the logs" + ) if not os.path.exists(args.location): log.LOGGER.critical(f"-E- Cannot find dump file at {args.location}") sys.exit(1) @@ -290,7 +295,9 @@ def create_analyzer(parsed_args, full_extracted_logs_list, ) if len(failed_extract) > 0: - log.LOGGER.debug(f"Failed to get some logs - {failed_extract}, skipping them") + log.LOGGER.debug( + f"Failed to get some logs - {failed_extract}, skipping them" + ) logs_regex_csv_handler_list = create_logs_regex_csv_handler_list( logs_to_work_with ) @@ -299,63 +306,71 @@ def create_analyzer(parsed_args, full_extracted_logs_list, run_parsers_processes(parsers_processes) log.LOGGER.debug("Done saving all CSV files") - # Setting the time granularity for the graphs BaseImageCreator.time_interval = args.interval # Analyze the CSV and be able to query the data start = time.perf_counter() log.LOGGER.debug("Starting analyzing the data") - partial_create_analyzer = partial(create_analyzer, - parsed_args=args, - full_extracted_logs_list=logs_to_work_with, - ufm_top_analyzer_obj=ufm_top_analyzer) + partial_create_analyzer = partial( + create_analyzer, + parsed_args=args, + full_extracted_logs_list=logs_to_work_with, + ufm_top_analyzer_obj=ufm_top_analyzer, + ) # Creating the analyzer for each log # By assigning them, a user can query the data via # the interactive session - ibdiagnet_analyzer = partial_create_analyzer(log_name="ibdiagnet2.log", - analyzer_clc=IBDIAGNETLogAnalyzer) + ibdiagnet_analyzer = partial_create_analyzer( + log_name="ibdiagnet2.log", analyzer_clc=IBDIAGNETLogAnalyzer + ) - event_log_analyzer = partial_create_analyzer(log_name="event.log", - analyzer_clc=EventsLogAnalyzer) + event_log_analyzer = partial_create_analyzer( + log_name="event.log", analyzer_clc=EventsLogAnalyzer + ) - ufm_health_analyzer = partial_create_analyzer(log_name="ufmhealth.log", - analyzer_clc=UFMHealthAnalyzer) + ufm_health_analyzer = partial_create_analyzer( + log_name="ufmhealth.log", analyzer_clc=UFMHealthAnalyzer + ) - ufm_log_analyzer = partial_create_analyzer(log_name="ufm.log", - analyzer_clc=UFMLogAnalyzer) + ufm_log_analyzer = partial_create_analyzer( + log_name="ufm.log", analyzer_clc=UFMLogAnalyzer + ) - console_log_analyzer = partial_create_analyzer(log_name="console.log", - analyzer_clc=ConsoleLogAnalyzer) + console_log_analyzer = partial_create_analyzer( + log_name="console.log", analyzer_clc=ConsoleLogAnalyzer + ) - rest_api_log_analyzer = partial_create_analyzer(log_name="rest_api.log", - analyzer_clc=RestApiAnalyzer) + rest_api_log_analyzer = partial_create_analyzer( + log_name="rest_api.log", analyzer_clc=RestApiAnalyzer + ) ibdianget_2_ports_primary_analyzer = partial_create_analyzer( log_name="ufm_logs_ibdiagnet2_port_counters.log", - analyzer_clc=Ibdiagnet2PortCountersAnalyzer) + analyzer_clc=Ibdiagnet2PortCountersAnalyzer, + ) ibdianget_2_ports_secondary_analyzer = partial_create_analyzer( log_name="secondary_telemetry_ibdiagnet2_port_counters.log", - analyzer_clc=Ibdiagnet2PortCountersAnalyzer) + analyzer_clc=Ibdiagnet2PortCountersAnalyzer, + ) - second_telemetry_samples = get_files_in_dest_by_type(args.destination, - "secondary_", - 1000, - "gz") + second_telemetry_samples = get_files_in_dest_by_type( + args.destination, "secondary_", 1000, "gz" + ) if len(second_telemetry_samples): - - links_flapping_analyzer = LinkFlappingAnalyzer(second_telemetry_samples, - args.destination) + links_flapping_analyzer = LinkFlappingAnalyzer( + second_telemetry_samples, args.destination + ) ufm_top_analyzer.add_analyzer(links_flapping_analyzer) else: - links_flapping_analyzer = None # pylint: disable=invalid-name + links_flapping_analyzer = None # pylint: disable=invalid-name end = time.perf_counter() log.LOGGER.debug(f"Took {end-start:.3f} to load the parsed data") all_images_outputs_and_title = ufm_top_analyzer.full_analysis() - png_images =[] + png_images = [] images_and_title_to_present = [] for image_title in all_images_outputs_and_title: image, title = image_title @@ -365,46 +380,74 @@ def create_analyzer(parsed_args, full_extracted_logs_list, images_and_title_to_present.append((image, title)) # Next section is to create a summary PDF pdf_path = os.path.join(args.destination, "UFM_Dump_analysis.pdf") - pdf_header = ( - f"{os.path.basename(args.location)}, hours={args.hours}" - ) + pdf_header = f"{os.path.basename(args.location)}, hours={args.hours}" used_ufm_version = console_log_analyzer.ufm_versions - text_to_show_in_pdf = f"Used ufm version in console log {used_ufm_version}{os.linesep}" + text_to_show_in_pdf = ( + f"Used ufm version in console log {used_ufm_version}{os.linesep}" + ) pdf = PDFCreator(pdf_path, pdf_header, png_images, text_to_show_in_pdf) dataframes_for_pdf = [] - fabric_info = ibdiagnet_analyzer.get_fabric_size() \ - if ibdiagnet_analyzer else "No Fabric Info found" + fabric_info = ( + ibdiagnet_analyzer.get_fabric_size() + if ibdiagnet_analyzer + else "No Fabric Info found" + ) dataframes_for_pdf.append(("Fabric info", fabric_info)) if links_flapping_analyzer: - dataframes_for_pdf.append(("Link Flapping past week", - links_flapping_analyzer.get_link_flapping_last_week())) + dataframes_for_pdf.append( + ( + "Link Flapping past week", + links_flapping_analyzer.get_link_flapping_last_week(), + ) + ) lists_to_add = [] critical_events_headers = ["timestamp", "event_type", "event", "count"] - lists_to_add.append((event_log_analyzer.get_critical_event_bursts(), - "More than 5 events burst over a minute", - critical_events_headers)) + lists_to_add.append( + ( + event_log_analyzer.get_critical_event_bursts(), + "More than 5 events burst over a minute", + critical_events_headers, + ) + ) existing_telemetry_analyzers = [] - for telemetry_analyzer in \ - [ibdianget_2_ports_primary_analyzer, ibdianget_2_ports_secondary_analyzer]: + for telemetry_analyzer in [ + ibdianget_2_ports_primary_analyzer, + ibdianget_2_ports_secondary_analyzer, + ]: if telemetry_analyzer: existing_telemetry_analyzers.append(telemetry_analyzer) for cur_telemetry in existing_telemetry_analyzers: - dataframes_for_pdf.append((f"{cur_telemetry.telemetry_type} Telemetry iteration time", - cur_telemetry.get_last_iterations_time_stats())) - dataframes_for_pdf.append((f"{cur_telemetry.telemetry_type} " - "Telemetry iteration first and last timestamps", - cur_telemetry.get_first_last_iteration_timestamp())) - dataframes_for_pdf.append((f"{cur_telemetry.telemetry_type} Telemetry fabric size", - cur_telemetry.get_number_of_switches_and_ports())) - lists_to_add.append(([cur_telemetry.get_number_of_core_dumps()], - f"{cur_telemetry.telemetry_type} " - "number of core dumps found in the logs", - ["Amount"])) - + dataframes_for_pdf.append( + ( + f"{cur_telemetry.telemetry_type} Telemetry iteration time", + cur_telemetry.get_last_iterations_time_stats(), + ) + ) + dataframes_for_pdf.append( + ( + f"{cur_telemetry.telemetry_type} " + "Telemetry iteration first and last timestamps", + cur_telemetry.get_first_last_iteration_timestamp(), + ) + ) + dataframes_for_pdf.append( + ( + f"{cur_telemetry.telemetry_type} Telemetry fabric size", + cur_telemetry.get_number_of_switches_and_ports(), + ) + ) + lists_to_add.append( + ( + [cur_telemetry.get_number_of_core_dumps()], + f"{cur_telemetry.telemetry_type} " + "number of core dumps found in the logs", + ["Amount"], + ) + ) # PDF creator gets all the images and to add to the report pdf.create_pdf(dataframes_for_pdf, lists_to_add) @@ -416,14 +459,15 @@ def create_analyzer(parsed_args, full_extracted_logs_list, if args.interactive: import IPython + IPython.embed() # Clean some unended files created during run files_types_to_delete = set() - files_types_to_delete.add("png") #png images created for PDF report - files_types_to_delete.add("log") #logs taken from the logs - files_types_to_delete.add("csv") #tmp csv + telemetery samples - files_types_to_delete.add("gz") #gz files of logs and samples + files_types_to_delete.add("png") # png images created for PDF report + files_types_to_delete.add("log") # logs taken from the logs + files_types_to_delete.add("csv") # tmp csv + telemetery samples + files_types_to_delete.add("gz") # gz files of logs and samples delete_files_by_types(args.destination, files_types_to_delete) except Exception as exc: diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py index e5b2da63..d45d7394 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py @@ -27,13 +27,17 @@ from loganalyze.log_analyzers.constants import DataConstants import loganalyze.logger as log + # This makes sure the user does not see the warning from plotting -logging.getLogger('matplotlib').setLevel(logging.ERROR) -matplotlib.use('Agg') # This allows to run the tool on servers without graphic card/headless +logging.getLogger("matplotlib").setLevel(logging.ERROR) +matplotlib.use( + "Agg" +) # This allows to run the tool on servers without graphic card/headless pd.set_option("display.max_colwidth", None) warnings.filterwarnings("ignore") + class BaseImageCreator: # Setting the graph time interval to 1 hour # This is out side of the constructor since @@ -89,11 +93,12 @@ def _save_data_based_on_timestamp( bbox={"facecolor": "white", "alpha": 0.5}, ) - generic_file_name = f"{title}".replace(" ", "_").replace("/", "_") images_created = [] for img_type in self._images_type: - cur_img = os.path.join(self._dest_image_path,f"{generic_file_name}.{img_type}") + cur_img = os.path.join( + self._dest_image_path, f"{generic_file_name}.{img_type}" + ) log.LOGGER.debug(f"Saving {cur_img}") plt.savefig(cur_img, format=img_type) images_created.append(cur_img) @@ -119,7 +124,9 @@ def _save_pivot_data_in_bars( generic_file_name = f"{title}".replace(" ", "_").replace("/", "_") images_created = [] for img_type in self._images_type: - cur_img = os.path.join(self._dest_image_path,f"{generic_file_name}.{img_type}") + cur_img = os.path.join( + self._dest_image_path, f"{generic_file_name}.{img_type}" + ) log.LOGGER.debug(f"Saving {cur_img}") plt.savefig(cur_img, format=img_type) images_created.append(cur_img) @@ -137,14 +144,16 @@ def full_analysis(self): # In case a function is raising an exception. try: func() - except: # pylint: disable=bare-except + except: # pylint: disable=bare-except function_name = func.__name__ try: class_name = "" if "." in func.__qualname__: - class_name = func.__qualname__.split('.')[0] - log.LOGGER.debug(f"Error when calling {function_name} {class_name}, skipping") - except: # pylint: disable=bare-except + class_name = func.__qualname__.split(".")[0] + log.LOGGER.debug( + f"Error when calling {function_name} {class_name}, skipping" + ) + except: # pylint: disable=bare-except pass return self._images_created if len(self._images_created) > 0 else [] @@ -156,14 +165,12 @@ class BaseAnalyzer(BaseImageCreator): ability to print/save images and filter data """ - def __init__( self, logs_csvs: List[str], hours: int, dest_image_path: str, - sort_timestamp=True - + sort_timestamp=True, ): super().__init__(dest_image_path) dataframes = [pd.read_csv(ufm_log) for ufm_log in logs_csvs] @@ -175,8 +182,9 @@ def __init__( # Filter logs to include only those within the last 'hours' from the max timestamp filtered_logs = df[df[DataConstants.TIMESTAMP] >= start_time] data_sorted = filtered_logs.sort_values(by=DataConstants.TIMESTAMP) - data_sorted[DataConstants.AGGREGATIONTIME] = \ - data_sorted[DataConstants.TIMESTAMP].dt.floor(self.time_interval) + data_sorted[DataConstants.AGGREGATIONTIME] = data_sorted[ + DataConstants.TIMESTAMP + ].dt.floor(self.time_interval) self._log_data_sorted = data_sorted else: self._log_data_sorted = df @@ -186,7 +194,9 @@ def __init__( def _remove_empty_lines_from_csv(input_file): temp_file = input_file + ".temp" - with open(input_file, "r", newline="", encoding=DataConstants.UTF8ENCODING) as infile, open( + with open( + input_file, "r", newline="", encoding=DataConstants.UTF8ENCODING + ) as infile, open( temp_file, "w", newline="", encoding=DataConstants.UTF8ENCODING ) as outfile: reader = csv.reader(infile) @@ -212,10 +222,11 @@ def fix_lines_with_no_timestamp(csvs): temp_file = csv_file + ".temp" BaseAnalyzer._remove_empty_lines_from_csv(csv_file) fixed_lines = 0 - with open(csv_file, "r", newline="", encoding=DataConstants.UTF8ENCODING) \ - as infile, open( - temp_file, "w", newline="", encoding=DataConstants.UTF8ENCODING - ) as outfile: + with open( + csv_file, "r", newline="", encoding=DataConstants.UTF8ENCODING + ) as infile, open( + temp_file, "w", newline="", encoding=DataConstants.UTF8ENCODING + ) as outfile: reader = csv.reader(infile) writer = csv.writer(outfile) current_line = None diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/console_log_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/console_log_analyzer.py index d8406f07..dec942f6 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/console_log_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/console_log_analyzer.py @@ -43,14 +43,11 @@ def _extract_ufm_version(logs_csvs): temp_file = csv_file + ".temp" # Open the input CSV file for reading - with open(csv_file, - mode='r', - newline='', - encoding=DataConstants.UTF8ENCODING) as infile, \ - open(temp_file, - mode='w', - newline='', - encoding=DataConstants.UTF8ENCODING) as outfile: + with open( + csv_file, mode="r", newline="", encoding=DataConstants.UTF8ENCODING + ) as infile, open( + temp_file, mode="w", newline="", encoding=DataConstants.UTF8ENCODING + ) as outfile: reader = csv.DictReader(infile) fieldnames = reader.fieldnames # Get the header from the CSV writer = csv.DictWriter(outfile, fieldnames=fieldnames) @@ -60,10 +57,10 @@ def _extract_ufm_version(logs_csvs): # Iterate through each row in the input CSV for row in reader: - if row['type'] == 'ufm_version': + if row["type"] == "ufm_version": # If the type is 'ufm_version', # save the row and don't write it to the new file - ufm_versions.add(row['data']) + ufm_versions.add(row["data"]) else: # Write the row to the new CSV file writer.writerow(row) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/constants.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/constants.py index 0273a66f..07744e38 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/constants.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/constants.py @@ -10,6 +10,7 @@ # provided with the software product. # + class DataConstants: AGGREGATIONTIME = "aggregated_by_time" TIMESTAMP = "timestamp" diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/events_log_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/events_log_analyzer.py index 5b71125f..f888d88c 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/events_log_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/events_log_analyzer.py @@ -24,9 +24,11 @@ class EventsLogAnalyzer(BaseAnalyzer): def __init__(self, logs_csvs: List[str], hours: int, dest_image_path): super().__init__(logs_csvs, hours, dest_image_path) self._supported_log_levels = ["CRITICAL", "WARNING", "INFO", "MINOR"] - self._funcs_for_analysis = {self.plot_critical_events_per_aggregation_time, - self.plot_link_up_down_count_per_aggregation_time, - self.plot_top_n_critical_events_over_time} + self._funcs_for_analysis = { + self.plot_critical_events_per_aggregation_time, + self.plot_link_up_down_count_per_aggregation_time, + self.plot_top_n_critical_events_over_time, + } # Function to split "object_id" into "device" and "description" def _split_switch_object_id(self, row): @@ -48,16 +50,20 @@ def get_events_by_log_level(self, log_level="CRITICAL"): def plot_top_n_critical_events_over_time(self, n=10): critical_events = self.get_events_by_log_level("CRITICAL") - total_critical_events = critical_events.groupby("event").size().reset_index(name="count") + total_critical_events = ( + critical_events.groupby("event").size().reset_index(name="count") + ) # Get the top n events with the highest count overall - top_n_events = total_critical_events.nlargest(n, 'count') + top_n_events = total_critical_events.nlargest(n, "count") # Group the top 5 events by time interval - critical_events_grouped_by_time = \ - critical_events[critical_events["event"].isin(top_n_events["event"])]\ - .groupby([DataConstants.AGGREGATIONTIME, "event"])\ - .size().reset_index(name="count") + critical_events_grouped_by_time = ( + critical_events[critical_events["event"].isin(top_n_events["event"])] + .groupby([DataConstants.AGGREGATIONTIME, "event"]) + .size() + .reset_index(name="count") + ) pivot_top_n_events_by_hour = critical_events_grouped_by_time.pivot( index=DataConstants.AGGREGATIONTIME, columns="event", values="count" @@ -78,38 +84,42 @@ def get_critical_event_bursts(self, n=5): critical_events = self.get_events_by_log_level("CRITICAL") # Round timestamps to the nearest minute - critical_events['minute'] = critical_events['timestamp'].dt.floor('T') + critical_events["minute"] = critical_events["timestamp"].dt.floor("T") # Group by minute and event type, then count the number of events in each group - event_counts = (critical_events - .groupby(['minute', 'event', 'event_type']) - .size() - .reset_index(name='count')) + event_counts = ( + critical_events.groupby(["minute", "event", "event_type"]) + .size() + .reset_index(name="count") + ) # Filter for bursts where the count exceeds or equals 'n' - bursts = event_counts[event_counts['count'] >= n] + bursts = event_counts[event_counts["count"] >= n] # Create a Series with 'minute' as index and 'count' as values - bursts_series = bursts.set_index('minute')['count'] + bursts_series = bursts.set_index("minute")["count"] # Save the plot using the series self._save_data_based_on_timestamp( bursts_series, # Pass the Series instead of separate lists "Time", "Number of Critical Events in the burst", - "Critical Event Bursts" + "Critical Event Bursts", ) # Convert the result to a list of dictionaries for returning - burst_list = bursts.rename(columns={'minute': 'timestamp'}).to_dict(orient='records') + burst_list = bursts.rename(columns={"minute": "timestamp"}).to_dict( + orient="records" + ) return burst_list def plot_critical_events_per_aggregation_time(self): critical_events = self.get_events_by_log_level("CRITICAL") critical_events_grouped_by_time = ( - critical_events.groupby([DataConstants.AGGREGATIONTIME, "event"])\ - .size().reset_index(name="count") + critical_events.groupby([DataConstants.AGGREGATIONTIME, "event"]) + .size() + .reset_index(name="count") ) pivot_critical_events_by_hour = critical_events_grouped_by_time.pivot( @@ -124,15 +134,14 @@ def plot_critical_events_per_aggregation_time(self): "Events", ) - - def plot_link_up_down_count_per_aggregation_time(self): links_events = self._log_data_sorted[ (self._log_data_sorted["event"] == "Link is up") - | - (self._log_data_sorted["event"] == "Link went down") + | (self._log_data_sorted["event"] == "Link went down") ] - grouped_links_events = links_events.groupby([DataConstants.AGGREGATIONTIME, "event"]) + grouped_links_events = links_events.groupby( + [DataConstants.AGGREGATIONTIME, "event"] + ) counted_links_events_by_time = grouped_links_events.size().reset_index( name="count" ) @@ -141,9 +150,5 @@ def plot_link_up_down_count_per_aggregation_time(self): index=DataConstants.AGGREGATIONTIME, columns="event", values="count" ).fillna(0) self._save_pivot_data_in_bars( - pivot_links_data, - "Time", - "Number of Events", - "Link up/down events", - "Event" + pivot_links_data, "Time", "Number of Events", "Link up/down events", "Event" ) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ibdiagnet2_port_counters_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ibdiagnet2_port_counters_analyzer.py index 40872c0a..2b58d4d8 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ibdiagnet2_port_counters_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ibdiagnet2_port_counters_analyzer.py @@ -17,22 +17,23 @@ class Ibdiagnet2PortCountersAnalyzer(BaseAnalyzer): - def __init__(self, - logs_csvs: List[str], - hours: int, - dest_image_path: str, - sort_timestamp=False): + def __init__( + self, + logs_csvs: List[str], + hours: int, + dest_image_path: str, + sort_timestamp=False, + ): super().__init__(logs_csvs, hours, dest_image_path, sort_timestamp) self._iteration_time_data = None self._iteration_time_stats = None self.text_to_show_in_pdf = "" # This will make sure all the extra columns are int - extra_columns = ['extra1', 'extra2', 'extra3', 'extra4', 'extra5'] + extra_columns = ["extra1", "extra2", "extra3", "extra4", "extra5"] for col in extra_columns: self._log_data_sorted[col] = pd.to_numeric( - self._log_data_sorted[col], - errors='coerce' - ).astype('Int64') + self._log_data_sorted[col], errors="coerce" + ).astype("Int64") self._funcs_for_analysis = {self.plot_iteration_time_over_time} # Based on the log path, decided if this is primary or secondary if "ufm_logs" in logs_csvs[0]: @@ -46,8 +47,9 @@ def __init__(self, self._last_timestamp_of_logs = None def get_collectx_versions(self): - unique_collectx_versions = self._log_data_sorted[\ - self._log_data_sorted['type'] == 'collectx_version']['data'].unique() + unique_collectx_versions = self._log_data_sorted[ + self._log_data_sorted["type"] == "collectx_version" + ]["data"].unique() return unique_collectx_versions def get_number_of_switches_and_ports(self): @@ -56,26 +58,29 @@ def get_number_of_switches_and_ports(self): This function calculates the average, maximum, minimum for switches, CAs, routers, and ports. """ - filtered_data = self._log_data_sorted[\ - self._log_data_sorted['type'] == 'total_devices_ports'] - - ports_numbers_columns = ['extra1', 'extra3', 'extra5'] - filtered_data['extra135'] = pd.to_numeric( - filtered_data[ports_numbers_columns].stack(), errors='coerce' - ).groupby(level=0).sum(min_count=1) - - columns_of_interest = ['data', 'extra2', 'extra4', 'extra135'] + filtered_data = self._log_data_sorted[ + self._log_data_sorted["type"] == "total_devices_ports" + ] + + ports_numbers_columns = ["extra1", "extra3", "extra5"] + filtered_data["extra135"] = ( + pd.to_numeric(filtered_data[ports_numbers_columns].stack(), errors="coerce") + .groupby(level=0) + .sum(min_count=1) + ) + + columns_of_interest = ["data", "extra2", "extra4", "extra135"] column_mapping = { - 'data': '# of Switches', - 'extra2': 'CAs', - 'extra4': 'Routers', - 'extra135': 'Ports' + "data": "# of Switches", + "extra2": "CAs", + "extra4": "Routers", + "extra135": "Ports", } summary_stats = [] for col in columns_of_interest: - numeric_col = pd.to_numeric(filtered_data[col], errors='coerce') + numeric_col = pd.to_numeric(filtered_data[col], errors="coerce") non_zero_col = numeric_col[numeric_col != 0] avg = round(non_zero_col.mean()) if not non_zero_col.empty else 0 @@ -83,13 +88,15 @@ def get_number_of_switches_and_ports(self): min_val = int(non_zero_col.min()) if not non_zero_col.empty else 0 count = int(non_zero_col.count()) - summary_stats.append({ - 'Category': column_mapping.get(col, col), - 'Average': avg, - 'Maximum': max_val, - 'Minimum': min_val, - 'Total Rows (Non-Zero)': count - }) + summary_stats.append( + { + "Category": column_mapping.get(col, col), + "Average": avg, + "Maximum": max_val, + "Minimum": min_val, + "Total Rows (Non-Zero)": count, + } + ) summary_df = pd.DataFrame(summary_stats) @@ -101,31 +108,37 @@ def analyze_iteration_time(self, threshold=0.15): Keep only 'type', 'timestamp', and 'data' columns. Calculate statistics for the 'data' column, including timestamps for max and min. Also, find gaps of at least 2 minutes with no data and allow filtering by a threshold. - + Parameters: - threshold (float): Minimum value to consider for analysis. Default is 0.5 seconds. """ - filtered_data = self._log_data_sorted[self._log_data_sorted['type'] == 'iteration_time'] - filtered_data = filtered_data[['type', 'timestamp', 'data']] - filtered_data['data'] = pd.to_numeric(filtered_data['data'], errors='coerce') - - filtered_data = filtered_data[filtered_data['data'] >= threshold] - filtered_data['timestamp'] = pd.to_datetime(filtered_data['timestamp'], errors='coerce') - filtered_data = filtered_data.dropna(subset=['timestamp']) - - filtered_data = filtered_data.sort_values(by='timestamp').reset_index(drop=True) - - if not filtered_data['data'].empty: - average = filtered_data['data'].mean() - max_value = filtered_data['data'].max() - min_value = filtered_data['data'].min() - - max_timestamp = filtered_data.loc[filtered_data['data'] \ - == max_value, 'timestamp'].iloc[0] - min_timestamp = filtered_data.loc[filtered_data['data'] \ - == min_value, 'timestamp'].iloc[0] - first_timestamp = filtered_data['timestamp'].iloc[0] - last_timestamp = filtered_data['timestamp'].iloc[-1] + filtered_data = self._log_data_sorted[ + self._log_data_sorted["type"] == "iteration_time" + ] + filtered_data = filtered_data[["type", "timestamp", "data"]] + filtered_data["data"] = pd.to_numeric(filtered_data["data"], errors="coerce") + + filtered_data = filtered_data[filtered_data["data"] >= threshold] + filtered_data["timestamp"] = pd.to_datetime( + filtered_data["timestamp"], errors="coerce" + ) + filtered_data = filtered_data.dropna(subset=["timestamp"]) + + filtered_data = filtered_data.sort_values(by="timestamp").reset_index(drop=True) + + if not filtered_data["data"].empty: + average = filtered_data["data"].mean() + max_value = filtered_data["data"].max() + min_value = filtered_data["data"].min() + + max_timestamp = filtered_data.loc[ + filtered_data["data"] == max_value, "timestamp" + ].iloc[0] + min_timestamp = filtered_data.loc[ + filtered_data["data"] == min_value, "timestamp" + ].iloc[0] + first_timestamp = filtered_data["timestamp"].iloc[0] + last_timestamp = filtered_data["timestamp"].iloc[-1] else: average = max_value = min_value = 0.0 @@ -133,12 +146,12 @@ def analyze_iteration_time(self, threshold=0.15): first_timestamp = last_timestamp = None stats = { - 'Average': average, - 'Maximum': max_value, - 'Max Timestamp': max_timestamp, - 'Minimum': min_value, - 'Min Timestamp': min_timestamp, - 'Total Rows': filtered_data['data'].count() + "Average": average, + "Maximum": max_value, + "Max Timestamp": max_timestamp, + "Minimum": min_value, + "Min Timestamp": min_timestamp, + "Total Rows": filtered_data["data"].count(), } stats_df = pd.DataFrame([stats]) self._iteration_time_data = filtered_data @@ -150,9 +163,9 @@ def analyze_iteration_time(self, threshold=0.15): def get_first_last_iteration_timestamp(self): if not self._first_timestamp_of_logs or not self._last_timestamp_of_logs: self.analyze_iteration_time() - times ={ - 'first': str(self._first_timestamp_of_logs), - 'last': str(self._last_timestamp_of_logs) + times = { + "first": str(self._first_timestamp_of_logs), + "last": str(self._last_timestamp_of_logs), } return pd.DataFrame([times]) @@ -163,17 +176,20 @@ def plot_iteration_time_over_time(self): if self._iteration_time_data is None: self.analyze_iteration_time() - self._iteration_time_data.set_index('timestamp', inplace=True) + self._iteration_time_data.set_index("timestamp", inplace=True) with warnings.catch_warnings(): warnings.filterwarnings("ignore", ".*Locator attempting to generate.*") self._save_data_based_on_timestamp( - data_to_plot=self._iteration_time_data['data'], - x_label='Timestamp', - y_label='Iteration Time (s)', - title=f'{self.telemetry_type} Iteration Time', - large_sample=True) + data_to_plot=self._iteration_time_data["data"], + x_label="Timestamp", + y_label="Iteration Time (s)", + title=f"{self.telemetry_type} Iteration Time", + large_sample=True, + ) def get_number_of_core_dumps(self): - core_dumps = self._log_data_sorted[self._log_data_sorted['type'] == 'timeout_dump_core'] - return {"Amount":len(core_dumps)} + core_dumps = self._log_data_sorted[ + self._log_data_sorted["type"] == "timeout_dump_core" + ] + return {"Amount": len(core_dumps)} diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/link_flapping_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/link_flapping_analyzer.py index 82c04a4a..a415ef62 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/link_flapping_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/link_flapping_analyzer.py @@ -21,16 +21,18 @@ from utils.netfix.link_flapping import get_link_flapping from loganalyze.log_analyzers.base_analyzer import BaseImageCreator import loganalyze.logger as log -FILE_NAME_PATTERN=r"^secondary_(5m|1h|1d|1w)_(\d{14})\.gz$" -TIME_PATTERN="%Y%m%d%H%M%S" + +FILE_NAME_PATTERN = r"^secondary_(5m|1h|1d|1w)_(\d{14})\.gz$" +TIME_PATTERN = "%Y%m%d%H%M%S" + class LinkFlappingAnalyzer(BaseImageCreator): - def __init__(self, telemetry_samples_csv:List[str], dest_image_path:str): + def __init__(self, telemetry_samples_csv: List[str], dest_image_path: str): super().__init__(dest_image_path) self._telemetry_samples_csv = telemetry_samples_csv - self._mapped_samples = {"5m":{}, "1h":{}, "1d":{}, "1w":{}} + self._mapped_samples = {"5m": {}, "1h": {}, "1d": {}, "1w": {}} self._map_files_by_sampling_rate_and_date() - self._funcs_for_analysis={self.plot_link_flapping_last_week} + self._funcs_for_analysis = {self.plot_link_flapping_last_week} def _map_files_by_sampling_rate_and_date(self): for sample_file in self._telemetry_samples_csv: @@ -45,40 +47,46 @@ def _map_files_by_sampling_rate_and_date(self): date_time = datetime.strptime(date_time_str, TIME_PATTERN) self._mapped_samples[interval_time][date_time] = sample_file else: - log.LOGGER.debug(f"File {file_name} cannot be used for links flapping") + log.LOGGER.debug( + f"File {file_name} cannot be used for links flapping" + ) except FileNotFoundError: log.LOGGER.debug(f"File {sample_file} does not exist") except re.error: - log.LOGGER.debug(f"Invalid regular expression pattern {FILE_NAME_PATTERN}") + log.LOGGER.debug( + f"Invalid regular expression pattern {FILE_NAME_PATTERN}" + ) except ValueError: log.LOGGER.debug(f"Invalid date/time format {date_time_str}") except KeyError: log.LOGGER.debug(f"Invalid interval time {interval_time}") - #Sorting the inner dict so the first item would be the last timestamp - sorted_data = {key: sorted(value.items(), - key=lambda x: x[0], - reverse=True) for key, value in self._mapped_samples.items()} + # Sorting the inner dict so the first item would be the last timestamp + sorted_data = { + key: sorted(value.items(), key=lambda x: x[0], reverse=True) + for key, value in self._mapped_samples.items() + } self._mapped_samples = {key: dict(value) for key, value in sorted_data.items()} def _ungz_to_csv(self, file_path): # Ensure the input file has a .gz extension - if not file_path.endswith('.gz'): + if not file_path.endswith(".gz"): return "" # Create the output file path by replacing .gz with .csv output_file_path = f"{Path(file_path).stem}.csv" # Open the .gz file and write its content to a new .csv file - with gzip.open(file_path, 'rb') as f_in: - with open(output_file_path, 'wb') as f_out: + with gzip.open(file_path, "rb") as f_in: + with open(output_file_path, "wb") as f_out: shutil.copyfileobj(f_in, f_out) return output_file_path def _get_link_flapping_by_gz_files(self, file1_gz, file2_gz): - link_flapping = get_link_flapping(self._ungz_to_csv(file1_gz), - self._ungz_to_csv(file2_gz)) - columns_to_keep =['link_hash_id', 'estimated_time'] + link_flapping = get_link_flapping( + self._ungz_to_csv(file1_gz), self._ungz_to_csv(file2_gz) + ) + columns_to_keep = ["link_hash_id", "estimated_time"] link_flapping = link_flapping.loc[:, columns_to_keep] # Drop duplicate columns link_flapping = link_flapping.loc[:, ~link_flapping.columns.duplicated()] @@ -87,38 +95,51 @@ def _get_link_flapping_by_gz_files(self, file1_gz, file2_gz): return link_flapping def get_link_flapping_last_week(self): - five_m_samples = self._mapped_samples.get('5m') - week_samples = self._mapped_samples.get('1w') + five_m_samples = self._mapped_samples.get("5m") + week_samples = self._mapped_samples.get("1w") if len(five_m_samples) <= 0 or len(week_samples) <= 0: return pd.DataFrame() latest_sample_gz = list(islice(five_m_samples.values(), 1))[0] older_sample_gz = list(islice(week_samples.values(), 1))[0] - link_flapping = self._get_link_flapping_by_gz_files(older_sample_gz, latest_sample_gz) - data_sorted = link_flapping.sort_values(by='estimated_time').reset_index(drop=True) + link_flapping = self._get_link_flapping_by_gz_files( + older_sample_gz, latest_sample_gz + ) + data_sorted = link_flapping.sort_values(by="estimated_time").reset_index( + drop=True + ) return data_sorted def plot_link_flapping_last_week(self): link_flapping = self.get_link_flapping_last_week() # Convert "estimated_time" column to datetime object - link_flapping['estimated_time'] = pd.to_datetime(link_flapping['estimated_time']) - link_flapping['aggregated_by_time'] = link_flapping['estimated_time'].\ - dt.floor(self.time_interval) + link_flapping["estimated_time"] = pd.to_datetime( + link_flapping["estimated_time"] + ) + link_flapping["aggregated_by_time"] = link_flapping["estimated_time"].dt.floor( + self.time_interval + ) # Create pivot table with 'aggregated_by_time' as index and count of records as value - pivot_table = link_flapping.groupby('aggregated_by_time').size().reset_index(name='counts') - pivot_table['aggregated_by_time'] = pd.to_datetime(pivot_table['aggregated_by_time']) + pivot_table = ( + link_flapping.groupby("aggregated_by_time") + .size() + .reset_index(name="counts") + ) + pivot_table["aggregated_by_time"] = pd.to_datetime( + pivot_table["aggregated_by_time"] + ) # Reset index to create a new column for time intervals - pivot_table = pivot_table.set_index('aggregated_by_time').\ - rename_axis(None).rename(columns={'counts': 'Count'}) + pivot_table = ( + pivot_table.set_index("aggregated_by_time") + .rename_axis(None) + .rename(columns={"counts": "Count"}) + ) # Plot pivot table using _plot_and_save_pivot_data_in_bars method - self._save_pivot_data_in_bars(pivot_table, - 'Time', - 'Count', - 'Link Flapping Count', - None) - + self._save_pivot_data_in_bars( + pivot_table, "Time", "Count", "Link Flapping Count", None + ) def full_analysis(self): self.get_link_flapping_last_week() diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/rest_api_log_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/rest_api_log_analyzer.py index 0665d0e1..8b195e15 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/rest_api_log_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/rest_api_log_analyzer.py @@ -14,19 +14,29 @@ import numpy as np from loganalyze.log_analyzers.base_analyzer import BaseAnalyzer -class RestApiAnalyzer(BaseAnalyzer): - def __init__(self, logs_csvs: List[str], hours: int, dest_image_path: str, sort_timestamp=True): +class RestApiAnalyzer(BaseAnalyzer): + def __init__( + self, + logs_csvs: List[str], + hours: int, + dest_image_path: str, + sort_timestamp=True, + ): super().__init__(logs_csvs, hours, dest_image_path, sort_timestamp) - #Removing all the request coming from the ufm itself - self._log_data_sorted = self._log_data_sorted.loc\ - [self._log_data_sorted['user'] != 'ufmsystem'] - #Splitting the URL for better analysis + # Removing all the request coming from the ufm itself + self._log_data_sorted = self._log_data_sorted.loc[ + self._log_data_sorted["user"] != "ufmsystem" + ] + # Splitting the URL for better analysis if not self._log_data_sorted.empty: - self._log_data_sorted[['uri', 'query_params']] = self._log_data_sorted['url']\ - .apply(self.split_url_to_uri_and_query_params).apply(pd.Series) - self._have_duration = self._have_data_in_column('duration') - self._have_user = self._have_data_in_column('user') + self._log_data_sorted[["uri", "query_params"]] = ( + self._log_data_sorted["url"] + .apply(self.split_url_to_uri_and_query_params) + .apply(pd.Series) + ) + self._have_duration = self._have_data_in_column("duration") + self._have_user = self._have_data_in_column("user") self._funcs_for_analysis = {self.analyze_endpoints_freq} @staticmethod @@ -50,19 +60,22 @@ def _have_data_in_column(self, column): return self._log_data_sorted[column].notna().all() def analyze_endpoints_freq(self, endpoints_count_to_show=10): - by_uri_per_time = self._log_data_sorted.groupby(['uri', - 'aggregated_by_time']).size().reset_index( - name='amount_per_uri') - total_amount_per_uri = by_uri_per_time.groupby('uri')['amount_per_uri'].sum() + by_uri_per_time = ( + self._log_data_sorted.groupby(["uri", "aggregated_by_time"]) + .size() + .reset_index(name="amount_per_uri") + ) + total_amount_per_uri = by_uri_per_time.groupby("uri")["amount_per_uri"].sum() top_x_uris = total_amount_per_uri.nlargest(endpoints_count_to_show).index - data_to_show = by_uri_per_time.pivot(index='aggregated_by_time', - columns='uri', - values='amount_per_uri').fillna(0) + data_to_show = by_uri_per_time.pivot( + index="aggregated_by_time", columns="uri", values="amount_per_uri" + ).fillna(0) data_to_show = data_to_show[top_x_uris] - return self._save_pivot_data_in_bars(data_to_show, - "time", - "requests count", - f"Top {endpoints_count_to_show} "\ - "requests count over time", - "legend") + return self._save_pivot_data_in_bars( + data_to_show, + "time", + "requests count", + f"Top {endpoints_count_to_show} " "requests count over time", + "legend", + ) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_health_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_health_analyzer.py index eadf0d22..c98918e2 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_health_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_health_analyzer.py @@ -23,11 +23,13 @@ def __init__(self, logs_csvs: List[str], hours: int, dest_image_path): self._failed_tests_data = self._log_data_sorted[ self._log_data_sorted["test_status"] != "succeeded" ] - self._funcs_for_analysis={self.print_failed_tests_per_hour} + self._funcs_for_analysis = {self.print_failed_tests_per_hour} def print_failed_tests_per_hour(self): grouped_failed_by_time = ( - self._failed_tests_data.groupby([DataConstants.AGGREGATIONTIME, "test_name"]) + self._failed_tests_data.groupby( + [DataConstants.AGGREGATIONTIME, "test_name"] + ) .size() .reset_index(name="count") ) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_log_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_log_analyzer.py index 463c4d35..8cd94d64 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_log_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/ufm_log_analyzer.py @@ -25,9 +25,11 @@ class UFMLogAnalyzer(BaseAnalyzer): def __init__(self, logs_csvs: List[str], hours: int, dest_image_path): super().__init__(logs_csvs, hours, dest_image_path) - self._funcs_for_analysis = {self.full_analyze_ufm_loading_time, - self.full_telemetry_processing_time_report, - self.full_analyze_fabric_analysis_time} + self._funcs_for_analysis = { + self.full_analyze_ufm_loading_time, + self.full_telemetry_processing_time_report, + self.full_analyze_fabric_analysis_time, + } def full_analyze_ufm_loading_time(self): ufm_started_logs = self._log_data_sorted[ @@ -87,8 +89,12 @@ def full_analyze_ufm_loading_time(self): # Print statistics log.LOGGER.debug(f"Mean Time: {mean_time:.2f} seconds") - log.LOGGER.debug(f"Maximum Time: {max_time:.2f} seconds at {max_time_occurrence}") - log.LOGGER.debug(f"Minimum Time: {min_time:.2f} seconds at {min_time_occurrence}") + log.LOGGER.debug( + f"Maximum Time: {max_time:.2f} seconds at {max_time_occurrence}" + ) + log.LOGGER.debug( + f"Minimum Time: {min_time:.2f} seconds at {min_time_occurrence}" + ) # Print timestamps with missing 'Completed' logs if len(missing_completed_logs) > 0: @@ -225,11 +231,15 @@ def full_telemetry_processing_time_report(self): ] # Sort the telemetry logs by timestamp - telemetry_logs_sorted = telemetry_logs.sort_values(by=DataConstants.TIMESTAMP).copy() + telemetry_logs_sorted = telemetry_logs.sort_values( + by=DataConstants.TIMESTAMP + ).copy() # Extract the processing time and convert to float telemetry_logs_sorted["processing_time"] = ( - telemetry_logs_sorted[DataConstants.DATA].str.extract(r"(\d+(\.\d+)?)")[0].astype(float) + telemetry_logs_sorted[DataConstants.DATA] + .str.extract(r"(\d+(\.\d+)?)")[0] + .astype(float) ) # Calculate statistics @@ -250,8 +260,12 @@ def full_telemetry_processing_time_report(self): # Print the statistics log.LOGGER.debug(f"Mean Processing Time: {mean_time}") - log.LOGGER.debug(f"Maximum Processing Time: {max_time} at {max_time_occurrence}") - log.LOGGER.debug(f"Minimum Processing Time: {min_time} at {min_time_occurrence}") + log.LOGGER.debug( + f"Maximum Processing Time: {max_time} at {max_time_occurrence}" + ) + log.LOGGER.debug( + f"Minimum Processing Time: {min_time} at {min_time_occurrence}" + ) # Set the index to timestamp for resampling, converting to DateTimeIndex telemetry_logs_sorted[DataConstants.TIMESTAMP] = pd.to_datetime( @@ -270,8 +284,5 @@ def full_telemetry_processing_time_report(self): # Plot the data within the filtered time range title = "Telemetry processing time" self._save_data_based_on_timestamp( - minutely_mean_processing_time, - "Time", - "Processing Time (s)", - title + minutely_mean_processing_time, "Time", "Processing Time (s)", title ) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/console_log_regex.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/console_log_regex.py index 315ac3ee..58ef59e0 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/console_log_regex.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/console_log_regex.py @@ -38,14 +38,14 @@ UFM_STATE_CHANGE = re.compile(r"^.*UFM daemon.*$") -UFM_VERSION_REGEX = re.compile( - r"^\d+\.\d+\.\d+ build \d+$" -) +UFM_VERSION_REGEX = re.compile(r"^\d+\.\d+\.\d+ build \d+$") + def ufm_version(match: Match): ufm_version_text = match.group() return (None, "ufm_version", ufm_version_text) + def console_log_exception(match: Match): timestamp = match.group(1) line = match.group(3) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/ibdiagnet2_port_counters_log_regex.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/ibdiagnet2_port_counters_log_regex.py index 97f30d5e..68177aa6 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/ibdiagnet2_port_counters_log_regex.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/ibdiagnet2_port_counters_log_regex.py @@ -15,25 +15,41 @@ from loganalyze.log_parsing.base_regex import RegexAndHandlers -ITERATION_TIME_REGEX = re.compile(r"^\[ExportAPI_STATS\] Iteration time" - r"\: ([\d\.]+) sec \@ \[([\d\-]+ [\d\:\.]+)\]$") +ITERATION_TIME_REGEX = re.compile( + r"^\[ExportAPI_STATS\] Iteration time" + r"\: ([\d\.]+) sec \@ \[([\d\-]+ [\d\:\.]+)\]$" +) TIMEOUT_DUMP_CORE_REGEX = re.compile(r"^timeout: the monitored command dumped core$") -TOTAL_SWITCH_PORTS_REGEX = re.compile(r"^.*Total switches\/ports \[(\d+)\/(\d+)\]\, " - r"CAs\/ports \[(\d+)\/(\d+)\]\, Routers\/ports " - r"\[(\d+)\/(\d+)\]\s*$") +TOTAL_SWITCH_PORTS_REGEX = re.compile( + r"^.*Total switches\/ports \[(\d+)\/(\d+)\]\, " + r"CAs\/ports \[(\d+)\/(\d+)\]\, Routers\/ports " + r"\[(\d+)\/(\d+)\]\s*$" +) COLLECTX_VERSION_REGEX = re.compile(r"^\[ExportAPI\] Collectx version ([\d\.]+)$") + def iteration_time(match: Match): iteration_time_sec = match.group(1) timestamp = match.group(2) - return ("iteration_time", timestamp, iteration_time_sec, None, None, None, None, None) + return ( + "iteration_time", + timestamp, + iteration_time_sec, + None, + None, + None, + None, + None, + ) + def timeout_dump_core(_: Match): return ("timeout_dump_core", None, None, None, None, None, None, None) + def total_switch_ports(match: Match): total_switches = match.group(1) total_switch_ports_count = match.group(2) @@ -41,25 +57,56 @@ def total_switch_ports(match: Match): total_cas_ports = match.group(4) total_routers = match.group(5) total_routers_ports = match.group(6) - return ("total_devices_ports", None, total_switches, total_switch_ports_count,\ - total_cas, total_cas_ports,\ - total_routers, total_routers_ports) + return ( + "total_devices_ports", + None, + total_switches, + total_switch_ports_count, + total_cas, + total_cas_ports, + total_routers, + total_routers_ports, + ) -def collectx_version(match:Match): + +def collectx_version(match: Match): collectx_version_str = match.group(1) - return ("collectx_version", None, collectx_version_str, None, None, None, None, None) + return ( + "collectx_version", + None, + collectx_version_str, + None, + None, + None, + None, + None, + ) + -ibdiagnet2_headers = ("type", "timestamp", "data", "extra1", "extra2", "extra3", "extra4", "extra5") +ibdiagnet2_headers = ( + "type", + "timestamp", + "data", + "extra1", + "extra2", + "extra3", + "extra4", + "extra5", +) -ibdiagnet2_primary_log_regex_cls = \ - RegexAndHandlers("ufm_logs_ibdiagnet2_port_counters.log", ibdiagnet2_headers) -ibdiagnet2_secondary_log_regex_cls = \ - RegexAndHandlers("secondary_telemetry_ibdiagnet2_port_counters.log", ibdiagnet2_headers) +ibdiagnet2_primary_log_regex_cls = RegexAndHandlers( + "ufm_logs_ibdiagnet2_port_counters.log", ibdiagnet2_headers +) +ibdiagnet2_secondary_log_regex_cls = RegexAndHandlers( + "secondary_telemetry_ibdiagnet2_port_counters.log", ibdiagnet2_headers +) -regex_funcs_map = {ITERATION_TIME_REGEX: iteration_time, - TIMEOUT_DUMP_CORE_REGEX:timeout_dump_core, - TOTAL_SWITCH_PORTS_REGEX: total_switch_ports, - COLLECTX_VERSION_REGEX: collectx_version} +regex_funcs_map = { + ITERATION_TIME_REGEX: iteration_time, + TIMEOUT_DUMP_CORE_REGEX: timeout_dump_core, + TOTAL_SWITCH_PORTS_REGEX: total_switch_ports, + COLLECTX_VERSION_REGEX: collectx_version, +} for regex, regex_func in regex_funcs_map.items(): ibdiagnet2_primary_log_regex_cls.add_regex(regex, regex_func) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/log_parser.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/log_parser.py index ee79f83f..61613147 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/log_parser.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/log_parser.py @@ -16,7 +16,8 @@ from typing import Callable, Tuple, List import gzip -class LogParser: # pylint: disable=too-few-public-methods + +class LogParser: # pylint: disable=too-few-public-methods """ Basic class for parsing logs """ diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/logs_regex.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/logs_regex.py index 22ad76a8..e1b9035b 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/logs_regex.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/logs_regex.py @@ -17,8 +17,10 @@ from loganalyze.log_parsing.ibdiagnet_log_regex import ibdiagnet_log_regex_cls from loganalyze.log_parsing.console_log_regex import console_log_regex_cls from loganalyze.log_parsing.rest_api_log_regex import rest_api_log_regex_cls -from loganalyze.log_parsing.ibdiagnet2_port_counters_log_regex import \ - ibdiagnet2_primary_log_regex_cls, ibdiagnet2_secondary_log_regex_cls +from loganalyze.log_parsing.ibdiagnet2_port_counters_log_regex import ( + ibdiagnet2_primary_log_regex_cls, + ibdiagnet2_secondary_log_regex_cls, +) logs = [ event_log_regex_cls, @@ -28,7 +30,7 @@ console_log_regex_cls, rest_api_log_regex_cls, ibdiagnet2_secondary_log_regex_cls, - ibdiagnet2_primary_log_regex_cls + ibdiagnet2_primary_log_regex_cls, ] logs_regex_csv_headers_list = [] diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/rest_api_log_regex.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/rest_api_log_regex.py index 186625c6..f8408ced 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/rest_api_log_regex.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_parsing/rest_api_log_regex.py @@ -27,6 +27,7 @@ r"(?: status_code: \((?P[\d]+)\),?)?(?: duration: (?P[\d\.]+) seconds)?$" ) + def rest_api_log(match: Match): """ The rest api log line had serval changes in the last releases. @@ -43,6 +44,7 @@ def rest_api_log(match: Match): duration = match.group("duration") return (timestamp, severity, client_ip, user, url, method, status_code, duration) + REST_API_LOG_HEADERS = ( "timestamp", "severity", @@ -51,7 +53,7 @@ def rest_api_log(match: Match): "url", "method", "status_code", - "duration" + "duration", ) rest_api_log_regex_cls = RegexAndHandlers("rest_api.log", REST_API_LOG_HEADERS) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logger.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logger.py index c26cd115..c15d95a2 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logger.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logger.py @@ -13,6 +13,7 @@ LOGGER = None + def setup_logger(name, level=logging.INFO): """ Function to set up a logger that outputs to the console. @@ -20,9 +21,9 @@ def setup_logger(name, level=logging.INFO): :param level: Logging level (default is INFO). :return: Configured logger instance. """ - global LOGGER # pylint: disable=global-statement + global LOGGER # pylint: disable=global-statement - formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") # Create console handler console_handler = logging.StreamHandler() diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_csv/csv_handler.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_csv/csv_handler.py index d265ad77..b4a3c00d 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_csv/csv_handler.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_csv/csv_handler.py @@ -18,6 +18,7 @@ from typing import List, Tuple import loganalyze.logger as log + class CsvHandler: def __init__( self, diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py index eb8c36d3..526d8fb9 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py @@ -29,7 +29,7 @@ def is_exists_get_as_path(self, location) -> Path: return None @staticmethod - def _split_based_on_dir(files:Set[str]): + def _split_based_on_dir(files: Set[str]): single_name_logs = set() logs_with_dirs = {} for log_name in files: @@ -44,7 +44,10 @@ def _split_based_on_dir(files:Set[str]): return single_name_logs, logs_with_dirs @abstractmethod - def extract_files(self, files_to_extract: List[str], - directories_to_extract: List[str], - destination: str): + def extract_files( + self, + files_to_extract: List[str], + directories_to_extract: List[str], + destination: str, + ): pass diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py index 8adcc790..12df164f 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py @@ -15,18 +15,23 @@ from typing import List from loganalyze.logs_extraction.base_extractor import BaseExtractor + class DirectoryExtractor(BaseExtractor): - def __init__(self, dir_path:Path): + def __init__(self, dir_path: Path): dir_path = self.is_exists_get_as_path(dir_path) if dir_path and dir_path.is_dir(): self.dir_path = dir_path else: - raise FileNotFoundError(f"Could not use {dir_path}, " - "make sure it exists and is a directory") + raise FileNotFoundError( + f"Could not use {dir_path}, " "make sure it exists and is a directory" + ) - def extract_files(self, files_to_extract: List[str], - directories_to_extract: List[str], - destination: str): + def extract_files( + self, + files_to_extract: List[str], + directories_to_extract: List[str], + destination: str, + ): if not os.path.exists(destination): os.makedirs(destination) @@ -39,13 +44,21 @@ def extract_files(self, files_to_extract: List[str], for root, _, files in os.walk(self.dir_path): for file_name in files: last_dir_name = os.path.basename(root) - is_logs_with_dir_flag = last_dir_name in logs_with_dirs and \ - file_name in logs_with_dirs[last_dir_name] - if file_name in files_to_extract or last_dir_name in directories_to_extract or\ - is_logs_with_dir_flag: + is_logs_with_dir_flag = ( + last_dir_name in logs_with_dirs + and file_name in logs_with_dirs[last_dir_name] + ) + if ( + file_name in files_to_extract + or last_dir_name in directories_to_extract + or is_logs_with_dir_flag + ): src_file_path = os.path.join(root, file_name) - new_file_name = f"{last_dir_name}_{file_name}" if is_logs_with_dir_flag \ + new_file_name = ( + f"{last_dir_name}_{file_name}" + if is_logs_with_dir_flag else file_name + ) dest_file_path = os.path.join(destination, new_file_name) shutil.copy2(src_file_path, dest_file_path) found_files.add(dest_file_path) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py index c001e11e..d2eea420 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py @@ -25,6 +25,7 @@ LOGS_GZ_POSTFIX = ".gz" GZIP_MAGIC_NUMBER = b"\x1f\x8b" # Magic number to understand if a file is really a gzip + class DumpFilesExtractor(BaseExtractor): def __init__(self, dump_path: Path) -> None: dump_path = self.is_exists_get_as_path(dump_path) @@ -32,13 +33,16 @@ def __init__(self, dump_path: Path) -> None: self.dump_path = dump_path self.directory = dump_path.parent else: - raise FileNotFoundError(f"Could not use {dump_path}, make sure it exists and a tar") + raise FileNotFoundError( + f"Could not use {dump_path}, make sure it exists and a tar" + ) def _get_files_from_tar( - self, opened_file: TarFile, + self, + opened_file: TarFile, files_to_extract: Set[str], - directories_to_extract:Set[str], - destination: str + directories_to_extract: Set[str], + destination: str, ): files_went_over = set() failed_extract = set() @@ -49,11 +53,15 @@ def _get_files_from_tar( full_dir_path = os.path.dirname(member.name) parent_dir_name = os.path.basename(full_dir_path) original_base_name = base_name - is_logs_with_dir_flag = parent_dir_name in logs_with_dirs and \ - base_name in logs_with_dirs[parent_dir_name] - if base_name in single_log_name or \ - parent_dir_name in directories_to_extract or \ - is_logs_with_dir_flag: + is_logs_with_dir_flag = ( + parent_dir_name in logs_with_dirs + and base_name in logs_with_dirs[parent_dir_name] + ) + if ( + base_name in single_log_name + or parent_dir_name in directories_to_extract + or is_logs_with_dir_flag + ): try: if is_logs_with_dir_flag: base_name = f"{parent_dir_name}_{base_name}" @@ -98,9 +106,12 @@ def is_gzip_file_obj(file_obj) -> bool: file_obj.seek(position) # Reset the stream position return magic_number == GZIP_MAGIC_NUMBER - def extract_files(self, files_to_extract: List[str], - directories_to_extract: List[str], - destination: str): + def extract_files( + self, + files_to_extract: List[str], + directories_to_extract: List[str], + destination: str, + ): """Since we do not know the type of dump, we search the files in the nested tars""" os.makedirs(destination, exist_ok=True) files_to_extract = set(files_to_extract) @@ -121,12 +132,14 @@ def extract_files(self, files_to_extract: List[str], fileobj=inner_tar_stream, mode=inner_file_open_mode ) as inner_tar: extracted_files, failed_files = self._get_files_from_tar( - inner_tar, files_to_extract, directories_to_extract, destination + inner_tar, + files_to_extract, + directories_to_extract, + destination, ) if len(extracted_files) > 0: return extracted_files, failed_files # If we got to this point, we might have a simple tar, try to extract from it - return self._get_files_from_tar(outer_tar, - files_to_extract, - directories_to_extract, - destination) + return self._get_files_from_tar( + outer_tar, files_to_extract, directories_to_extract, destination + ) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/pdf_creator.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/pdf_creator.py index e038cdb9..63774fe9 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/pdf_creator.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/pdf_creator.py @@ -48,7 +48,9 @@ def add_images(self): x, y = x_start, y_start for image_path in self._images_path: if os.path.exists(image_path): - self.image(image_path, x=x, y=y, w=image_width, h=image_height, type="PNG") + self.image( + image_path, x=x, y=y, w=image_width, h=image_height, type="PNG" + ) y += image_height + spacing if y > self.h - image_height - 20: self.add_page() @@ -68,13 +70,15 @@ def add_list_of_dicts_as_text(self, data_list, title=None, headers=None): if title: self.set_font("Arial", "B", 12) - self.cell(0, 10, title, 0, 1, 'C') + self.cell(0, 10, title, 0, 1, "C") self.ln(5) self.set_font("Arial", "", 10) - table_data = [[str(item.get(header, '')) for header in headers] for item in data_list] - table_str = tabulate(table_data, headers=headers, tablefmt='plain') + table_data = [ + [str(item.get(header, "")) for header in headers] for item in data_list + ] + table_str = tabulate(table_data, headers=headers, tablefmt="plain") self.multi_cell(0, 10, table_str) self.ln(10) @@ -86,7 +90,7 @@ def add_dataframe_as_text(self, data_frame, title=None): if title: self.set_font("Arial", "B", 12) - self.cell(0, 10, title, 0, 1, 'C') + self.cell(0, 10, title, 0, 1, "C") self.ln(5) num_columns = len(data_frame.columns) @@ -98,7 +102,9 @@ def add_dataframe_as_text(self, data_frame, title=None): self.set_font("Arial", "", 12) # Converting and removing the row number as it is not needed - table_str = tabulate(data_frame.values, headers=data_frame.columns, tablefmt='plain') + table_str = tabulate( + data_frame.values, headers=data_frame.columns, tablefmt="plain" + ) self.multi_cell(0, 10, table_str) self.ln(10) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/utils/common.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/utils/common.py index 318d24b6..06a94da8 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/utils/common.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/utils/common.py @@ -17,7 +17,8 @@ from typing import Set import loganalyze.logger as log -def delete_folders(folders: Set[str], until_folder:str): + +def delete_folders(folders: Set[str], until_folder: str): """ Delete all the folders in the folders param, and their parents until reaching the until_folder folder @@ -31,7 +32,9 @@ def delete_folders(folders: Set[str], until_folder:str): except FileNotFoundError: log.LOGGER.debug(f"Error: {cur_dir} does not exist.") except PermissionError: - log.LOGGER.debug(f"Error: Insufficient permissions to delete {cur_dir}.") + log.LOGGER.debug( + f"Error: Insufficient permissions to delete {cur_dir}." + ) except NotADirectoryError: log.LOGGER.debug(f"Error: {cur_dir} is not a directory.") except OSError as e: @@ -41,12 +44,12 @@ def delete_folders(folders: Set[str], until_folder:str): cur_dir = os.path.dirname(cur_dir) -def delete_files_by_types(folder:str, file_types:set[str]): +def delete_files_by_types(folder: str, file_types: set[str]): """ Given a folder and file types, for example svg and png, it will delete all files that end with .svg and .png in the given folder. """ - + for file_type in file_types: search_pattern = f"*.{file_type}" all_files_in_dir_by_type = glob.glob(os.path.join(folder, search_pattern))