From 14917aa2afef954a872c715494f45a68e387bab7 Mon Sep 17 00:00:00 2001 From: Vitaly Gavensky Date: Sun, 29 Sep 2024 17:12:50 +0300 Subject: [PATCH 1/7] issue:4091669 PDR plugin is not isolate based on LinkDownedCounter measurements --- .../telemetry_collector.py | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py index 566b7608..82e22be2 100644 --- a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py +++ b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py @@ -23,34 +23,46 @@ def get_telemetry(self): get the telemetry from secondary telemetry, if it in test mode it get from the simulation return DataFrame of the telemetry """ + # get telemetry data if self.test_mode: url = "http://127.0.0.1:9090/csv/xcset/simulated_telemetry" else: url = f"http://127.0.0.1:{self.SECONDARY_TELEMETRY_PORT}/csv/xcset/{self.SECONDARY_INSTANCE}" try: - self.logger.info(f"collecting telemetry from {url}.") + self.logger.info(f"Collecting telemetry from {url}.") telemetry_data = pd.read_csv(url) except (pd.errors.ParserError, pd.errors.EmptyDataError, urllib.error.URLError) as connection_error: - self.logger.error("failed to get telemetry data from UFM, fetched url=%s. Error: %s",url,connection_error) + self.logger.error("Failed to get telemetry data from UFM, fetched url=%s. Error: %s",url,connection_error) telemetry_data = None - if self.previous_telemetry_data is not None and telemetry_data is not None: - delta = self._get_delta(self.previous_telemetry_data,telemetry_data) - # when we want to keep only delta - if len(delta) > 0: - self.data_store.save(delta,self.data_store.get_filename_delta()) - elif telemetry_data is not None: - # when we want to keep the abs + + # store telemetry data + try: + if self.previous_telemetry_data is not None and telemetry_data is not None: + delta = self._get_delta(self.previous_telemetry_data,telemetry_data) + # when we want to keep only delta + if len(delta) > 0: + self.data_store.save(delta,self.data_store.get_filename_delta()) + elif telemetry_data is not None: + # when we want to keep the abs + self.data_store.save(telemetry_data,self.data_store.get_filename_abs()) + except Exception as exception_error: + self.logger.error(f"Failed to store telemetry data with error {exception_error}") + + # update previous telemetry data + if telemetry_data is not None: self.previous_telemetry_data = telemetry_data - self.data_store.save(telemetry_data,self.data_store.get_filename_abs()) return telemetry_data def _get_delta(self, first_df: pd.DataFrame, second_df:pd.DataFrame): merged_df = pd.merge(second_df, first_df, on=self.BASED_COLUMNS, how='inner', suffixes=('', '_x')) delta_dataframe = pd.DataFrame() - for index,col in enumerate(second_df.columns): - if col not in self.KEY and not isinstance(merged_df.iat[0,index],str): + for _,col in enumerate(second_df.columns): + if col not in self.KEY: col_x = col + "_x" - delta_dataframe[col] = merged_df[col] - merged_df[col_x] + try: + delta_dataframe[col] = merged_df[col] - merged_df[col_x] + except TypeError: + delta_dataframe[col] = second_df[col] else: delta_dataframe[col] = second_df[col] return delta_dataframe From 0ea2c43f9eafbc1c7f3e2b4a3edd484897445b74 Mon Sep 17 00:00:00 2001 From: Vitaly Gavensky Date: Sun, 29 Sep 2024 17:22:56 +0300 Subject: [PATCH 2/7] pylint fixes --- .../ufm_sim_web_service/telemetry_collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py index 82e22be2..74debe51 100644 --- a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py +++ b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py @@ -45,8 +45,8 @@ def get_telemetry(self): elif telemetry_data is not None: # when we want to keep the abs self.data_store.save(telemetry_data,self.data_store.get_filename_abs()) - except Exception as exception_error: - self.logger.error(f"Failed to store telemetry data with error {exception_error}") + except Exception as exception_error: # pylint: disable=broad-exception-caught + self.logger.error(f"Failed to store telemetry data with error {exception_error}") # update previous telemetry data if telemetry_data is not None: From bf9c01cad15035cc7d650e7c748ed04acb906b5b Mon Sep 17 00:00:00 2001 From: Vitaly Gavensky Date: Sun, 29 Sep 2024 17:26:06 +0300 Subject: [PATCH 3/7] pylint fixes --- .../ufm_sim_web_service/pdr_algorithm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py index f18e9e7f..9650714a 100644 --- a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py +++ b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py @@ -24,7 +24,7 @@ class PortData: """ Represents the port data. """ - #pylint: disable=too-many-arguments + #pylint: disable=too-many-positional-arguments def __init__(self, port_name=None, port_num=None, peer=None, node_type=None, active_speed=None, port_width=None, port_guid=None): """ Initialize a new instance of the PortData class. @@ -470,7 +470,7 @@ def check_deisolation_conditions(self, isolated_port): return True - #pylint: disable=too-many-arguments,too-many-locals + #pylint: disable=too-many-positional-arguments,too-many-locals def calc_symbol_ber_rate(self, port_name, port_speed, port_width, col_name, time_delta): """ calculate the symbol BER rate for a given port given the time delta From e6fd9428f99255085b479c942899c3a817dc092f Mon Sep 17 00:00:00 2001 From: Vitaly Gavensky Date: Sun, 29 Sep 2024 17:28:45 +0300 Subject: [PATCH 4/7] pylint fixes --- .../ufm_sim_web_service/pdr_algorithm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py index 9650714a..b6c0b011 100644 --- a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py +++ b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/pdr_algorithm.py @@ -24,7 +24,7 @@ class PortData: """ Represents the port data. """ - #pylint: disable=too-many-positional-arguments + #pylint: disable=too-many-positional-arguments,too-many-arguments def __init__(self, port_name=None, port_num=None, peer=None, node_type=None, active_speed=None, port_width=None, port_guid=None): """ Initialize a new instance of the PortData class. @@ -470,7 +470,7 @@ def check_deisolation_conditions(self, isolated_port): return True - #pylint: disable=too-many-positional-arguments,too-many-locals + #pylint: disable=too-many-positional-arguments,too-many-arguments,too-many-locals def calc_symbol_ber_rate(self, port_name, port_speed, port_width, col_name, time_delta): """ calculate the symbol BER rate for a given port given the time delta From e3c6f98fa5411d6e3d2b2a8b6f3ca55829decdbe Mon Sep 17 00:00:00 2001 From: Vitaly Gavensky Date: Mon, 30 Sep 2024 10:21:48 +0300 Subject: [PATCH 5/7] Types check in delta calculation --- .../ufm_sim_web_service/telemetry_collector.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py index 74debe51..11fcfce2 100644 --- a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py +++ b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py @@ -57,8 +57,10 @@ def _get_delta(self, first_df: pd.DataFrame, second_df:pd.DataFrame): merged_df = pd.merge(second_df, first_df, on=self.BASED_COLUMNS, how='inner', suffixes=('', '_x')) delta_dataframe = pd.DataFrame() for _,col in enumerate(second_df.columns): - if col not in self.KEY: - col_x = col + "_x" + col_x = col + "_x" + if col not in self.KEY\ + and not merged_df[col].apply(lambda x: isinstance(x, str)).any()\ + and not merged_df[col_x].apply(lambda x: isinstance(x, str)).any(): try: delta_dataframe[col] = merged_df[col] - merged_df[col_x] except TypeError: From edb23583efb2ddf8a31e43eb7affc6943efd7442 Mon Sep 17 00:00:00 2001 From: Vitaly Gavensky Date: Mon, 30 Sep 2024 10:30:23 +0300 Subject: [PATCH 6/7] Types check in delta calculation --- .../ufm_sim_web_service/telemetry_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py index 11fcfce2..d7f2dc2c 100644 --- a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py +++ b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py @@ -58,9 +58,9 @@ def _get_delta(self, first_df: pd.DataFrame, second_df:pd.DataFrame): delta_dataframe = pd.DataFrame() for _,col in enumerate(second_df.columns): col_x = col + "_x" - if col not in self.KEY\ - and not merged_df[col].apply(lambda x: isinstance(x, str)).any()\ - and not merged_df[col_x].apply(lambda x: isinstance(x, str)).any(): + if col not in self.KEY \ + and merged_df[col].dtype != 'object' \ + and merged_df[col_x].dtype != 'object': try: delta_dataframe[col] = merged_df[col] - merged_df[col_x] except TypeError: From 58891e9338467ba2e42e35c50f07fe48d353d49a Mon Sep 17 00:00:00 2001 From: Vitaly Gavensky Date: Mon, 30 Sep 2024 13:57:05 +0300 Subject: [PATCH 7/7] Separate function for storing data --- .../ufm_sim_web_service/telemetry_collector.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py index d7f2dc2c..7da64509 100644 --- a/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py +++ b/plugins/pdr_deterministic_plugin/ufm_sim_web_service/telemetry_collector.py @@ -36,6 +36,15 @@ def get_telemetry(self): telemetry_data = None # store telemetry data + self.store_telemetry_data(telemetry_data) + + # return retrieved data + return telemetry_data + + def store_telemetry_data(self, telemetry_data): + """ + Store telemetry data into the file + """ try: if self.previous_telemetry_data is not None and telemetry_data is not None: delta = self._get_delta(self.previous_telemetry_data,telemetry_data) @@ -48,10 +57,9 @@ def get_telemetry(self): except Exception as exception_error: # pylint: disable=broad-exception-caught self.logger.error(f"Failed to store telemetry data with error {exception_error}") - # update previous telemetry data + # keep telemetry data for next delta calculation if telemetry_data is not None: self.previous_telemetry_data = telemetry_data - return telemetry_data def _get_delta(self, first_df: pd.DataFrame, second_df:pd.DataFrame): merged_df = pd.merge(second_df, first_df, on=self.BASED_COLUMNS, how='inner', suffixes=('', '_x'))