Merge branch 'main' into pdr_telemetry

.ci/cidemo-init.sh

@@ -7,7 +7,7 @@ cd .ci
 changed_files=$(git diff --name-only remotes/origin/$ghprbTargetBranch)
 
 # Check for changes excluding .gitmodules and root .ci directory
-changes_excluding_gitmodules_and_root_ci=$(echo "$changed_files" | grep -v -e '.gitmodules' -e '.gitignore' -e '^\.ci/')
+changes_excluding_gitmodules_and_root_ci=$(echo "$changed_files" | grep -v -e '.gitmodules' -e '.gitignore' -e '^\.ci/' -e '^\.github/workflows' -e '\utils' -e '\plugins/ufm_log_analyzer_plugin') #Removing ufm_log_analyzer_plugin as for now it does not need a formal build
 
 # Check if changes exist and only in a single plugin directory (including its .ci directory)
 if [ -n "$changes_excluding_gitmodules_and_root_ci" ] && [ $(echo "$changes_excluding_gitmodules_and_root_ci" | cut -d '/' -f1,2 | uniq | wc -l) -eq 1 ]; then

.github/workflows/pdr_plugin_ci_workflow.yml

@@ -28,7 +28,7 @@ jobs:
         pip install pytest-cov
 
     - name: Run PyLint
-      run: pylint --rcfile=$PDRPATH/.pylintrc $PDRPATH
+      run: pylint --rcfile=$PDRPATH/.pylintrc $PDRPATH/ufm_sim_web_service/
 
     - name: Run exclusion list class test
       timeout-minutes: 5

.github/workflows/ufm_log_analyzer_ci_workflow.yml

@@ -0,0 +1,24 @@
+name: Ufm log analyzer CI Workflow
+
+on: 
+  push:
+     paths:
+      - 'plugins/ufm_log_analyzer_plugin/**'
+jobs:
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@main
+
+    - name: Set up Python
+      uses: actions/setup-python@main
+      with:
+        python-version: 3.9  # Specify the Python version you want to use
+
+    - name: Install dependencies
+      run: |
+        pip install -r plugins/ufm_log_analyzer_plugin/src/loganalyze/requirements.txt
+        pip install pylint
+    - name: Run PyLint
+      run: pylint --rcfile=plugins/ufm_log_analyzer_plugin/src/loganalyze/.pylintrc plugins/ufm_log_analyzer_plugin/src/loganalyze

plugins/pdr_deterministic_plugin/README.md

@@ -90,7 +90,7 @@ The ports are added or removed to blacklist via PDR plugin Rest API.
 Add ports to exclude list (to be excluded from the analysis):
     curl -k -i -X PUT 'http://<host_ip>/excluded' -d '[<formatted_ports_list>]'
     TTL (time to live in blacklist) can optionally follow the port after the comma (if zero or not specified, then port is excluded forever)
-    Example: curl -k -i -X PUT 'http://127.0.0.1:8977/excluded' -d '[["9c0591030085ac80_45"],["9c0591030085ac80_46",300]' (first port is added forever, second - just for 300 seconds)
+    Example: curl -k -i -X PUT 'http://127.0.0.1:8977/excluded' -d '[["9c0591030085ac80_45"],["9c0591030085ac80_46",300]]' (first port is added forever, second - just for 300 seconds)
 
 Remove ports from exclude list
     curl -k -i -X DELETE 'http://<host_ip>/excluded' -d '[<comma_separated_port_mames>]'

plugins/pdr_deterministic_plugin/build/Dockerfile

@@ -21,7 +21,7 @@ EXPOSE 9007
 
 RUN apt-get update && apt-get -y install supervisor python3 python3-pip rsyslog vim curl sudo
 
-RUN python3 -m pip install flask flask_restful requests twisted jsonschema pandas numpy
+RUN python3 -m pip install requests jsonschema pandas numpy aiohttp
 
 # remove an unused library that caused a high CVE vulnerability issue https://redmine.mellanox.com/issues/3837452
 RUN apt-get remove -y linux-libc-dev

plugins/pdr_deterministic_plugin/requirements.txt

@@ -1,9 +1,7 @@
-flask<=3.0.3
 numpy<=1.26.4
 pandas<=2.2.2
 pytest<=8.2.0
 requests<=2.31.0
-twisted<=22.1.0
-flask_restful<=0.3.10
 tzlocal<=4.2
 jsonschema<=4.5.1
+aiohttp<=3.9.1

plugins/pdr_deterministic_plugin/ufm_sim_web_service/api/__init__.py

@@ -11,6 +11,9 @@
 #
 
 class InvalidRequest(Exception):
+    """
+    exception of invalid request
+    """
 
     def __init__(self, message):
-        Exception.__init__(self,message)
+        Exception.__init__(self,message)

plugins/pdr_deterministic_plugin/ufm_sim_web_service/api/base_aiohttp_api.py

@@ -0,0 +1,81 @@
+#
+# Copyright © 2013-2024 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# This software product is a proprietary product of Nvidia Corporation and its affiliates
+# (the "Company") and all right, title, and interest in and to the software
+# product, including all associated intellectual property rights, are and
+# shall remain exclusively with the Company.
+#
+# This software product is governed by the End User License Agreement
+# provided with the software product.
+#
+
+import asyncio
+from aiohttp import web
+
+class BaseAiohttpAPI:
+    """
+    Base class for API implemented with aiohttp
+    """
+    def __init__(self):
+        """
+        Initialize a new instance of the BaseAiohttpAPI class.
+        """
+        self.app = web.Application()
+
+    @property
+    def application(self):
+        """
+        Read-only property for the application instance.
+        """
+        return self.app
+
+    def add_route(self, method, path, handler):
+        """
+        Add route to API.
+        """
+        self.app.router.add_route(method, path, handler)
+
+    def web_response(self, text, status):
+        """
+        Create response object.
+        """
+        return web.json_response(text=text, status=status)
+
+
+class BaseAiohttpServer:
+    """
+    Base class for HTTP server implemented with aiohttp
+    """
+    def __init__(self, logger):
+        """
+        Initialize a new instance of the BaseAiohttpAPI class.
+        """
+        self.logger = logger
+
+    def run(self, app, host, port):
+        """
+        Run the server on the specified host and port.
+        """
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(self._run_server(app, host, port))
+
+    async def _run_server(self, app, host, port):
+        """
+        Asynchronously run the server and handle shutdown.
+        """
+        # Run server
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, host, port)
+        await site.start()
+        self.logger.info(f"Server started at {host}:{port}")
+
+        # Wait for shutdown signal
+        shutdown_event = asyncio.Event()
+        try:
+            await shutdown_event.wait()
+        except KeyboardInterrupt:
+            self.logger.info(f"Shutting down server {host}:{port}...")
+        finally:
+            await runner.cleanup()

plugins/pdr_deterministic_plugin/ufm_sim_web_service/api/pdr_plugin_api.py

@@ -10,16 +10,16 @@
 # provided with the software product.
 #
 
+import json
 import time
 from http import HTTPStatus
 from json import JSONDecodeError
-from flask import json, request
-from utils.flask_server.base_flask_api_server import BaseAPIApplication
+from api.base_aiohttp_api import BaseAiohttpAPI
 
 ERROR_INCORRECT_INPUT_FORMAT = "Incorrect input format"
 EOL = '\n'
 
-class PDRPluginAPI(BaseAPIApplication):
+class PDRPluginAPI(BaseAiohttpAPI):
     '''
     class PDRPluginAPI
     '''
@@ -28,45 +28,40 @@ def __init__(self, isolation_mgr):
         """
         Initialize a new instance of the PDRPluginAPI class.
         """
-        super(PDRPluginAPI, self).__init__()
+        super().__init__()
         self.isolation_mgr = isolation_mgr
 
+        # Define routes using the base class's method
+        self.add_route("GET",    "/excluded", self.get_excluded_ports)
+        self.add_route("PUT",    "/excluded", self.exclude_ports)
+        self.add_route("DELETE", "/excluded", self.include_ports)
 
-    def _get_routes(self):
-        """
-        Map URLs to function calls
-        """
-        return {
-            self.get_excluded_ports: dict(urls=["/excluded"], methods=["GET"]),
-            self.exclude_ports: dict(urls=["/excluded"], methods=["PUT"]),
-            self.include_ports: dict(urls=["/excluded"], methods=["DELETE"])
-        }
 
-
-    def get_excluded_ports(self):
+    async def get_excluded_ports(self, request): # pylint: disable=unused-argument
         """
         Return ports from exclude list as comma separated port names
         """
         items = self.isolation_mgr.exclude_list.items()
-        formatted_items = [f"{item.port_name}: {'infinite' if item.ttl_seconds == 0 else int(max(0, item.remove_time - time.time()))}" for item in items]
+        formatted_items = [f"{item.port_name}: {'infinite' if item.ttl_seconds == 0 else int(max(0, item.remove_time - time.time()))}"
+                            for item in items]
         response = EOL.join(formatted_items) + ('' if not formatted_items else EOL)
-        return response, HTTPStatus.OK
+        return self.web_response(response, HTTPStatus.OK)
 
 
-    def exclude_ports(self):
+    async def exclude_ports(self, request):
         """
         Parse input ports and add them to exclude list (or just update TTL)
         Input string example: [["0c42a10300756a04_1"],["98039b03006c73ba_2",300]]
         TTL that follows port name after the colon is optional
         """
 
         try:
-            pairs = self.get_request_data()
+            pairs = await self.get_request_data(request)
         except (JSONDecodeError, ValueError):
-            return ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST
+            return self.web_response(ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST)
 
         if not isinstance(pairs, list) or not all(isinstance(pair, list) for pair in pairs):
-            return ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST
+            return self.web_response(ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST)
 
         response = ""
         for pair in pairs:
@@ -80,23 +75,23 @@ def exclude_ports(self):
                     response += f"Port {port_name} added to exclude list for {ttl} seconds"
 
                 response += self.get_port_warning(port_name) + EOL
-    
-        return response, HTTPStatus.OK
+
+        return self.web_response(response, HTTPStatus.OK)
 
 
-    def include_ports(self):
+    async def include_ports(self, request):
         """
         Remove ports from exclude list
         Input string: comma separated port names list
         Example: ["0c42a10300756a04_1","98039b03006c73ba_2"]
         """
         try:
-            port_names = self.get_request_data()
+            port_names = await self.get_request_data(request)
         except (JSONDecodeError, ValueError):
-            return ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST
+            return self.web_response(ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST)
 
         if not isinstance(port_names, list):
-            return ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST
+            return self.web_response(ERROR_INCORRECT_INPUT_FORMAT + EOL, HTTPStatus.BAD_REQUEST)
 
         response = ""
         for port_name in port_names:
@@ -108,22 +103,26 @@ def include_ports(self):
 
             response += self.get_port_warning(port_name) + EOL
 
-        return response, HTTPStatus.OK
+        return self.web_response(response, HTTPStatus.OK)
 
-
-    def get_request_data(self):
+    async def get_request_data(self, request):
         """
-        Deserialize request json data into object
+        Deserialize request data into object for aiohttp
         """
-        if request.is_json:
-            # Directly convert JSON data into Python object
-            return request.get_json()
-        else:
-            # Attempt to load plain data text as JSON
-            return json.loads(request.get_data(as_text=True))
-
-
-    def fix_port_name(self, port_name):
+        try:
+            # Try to get JSON data
+            return await request.json()
+        except json.JSONDecodeError:
+            # Try to get plain text data
+            text = await request.text()
+            try:
+                # Try to parse the text as JSON
+                return json.loads(text)
+            except json.JSONDecodeError:
+                # Return the raw text data
+                return text
+
+    def fix_port_name(self,port_name):
         """
         Try to fix common user mistakes for input port names
         Return fixed port name

plugins/pdr_deterministic_plugin/ufm_sim_web_service/constants.py

@@ -12,7 +12,10 @@
 
 import logging
 
-class PDRConstants(object):
+class PDRConstants():
+    """
+    The constants of the PDR plugin.
+    """
 
     CONF_FILE = "/config/pdr_deterministic.conf"
     LOG_FILE = '/log/pdr_deterministic_plugin.log'
@@ -55,6 +58,8 @@ class PDRConstants(object):
     API_ISOLATED_PORTS = "isolated_ports"
     SECONDARY_INSTANCE = "low_freq_debug"
 
+    TIMEOUT = 60
+
     EXTERNAL_EVENT_ERROR = 554
     EXTERNAL_EVENT_ALERT = 553
     EXTERNAL_EVENT_NOTICE = 552

plugins/pdr_deterministic_plugin/ufm_sim_web_service/exclude_list.py

@@ -13,7 +13,7 @@
 import time
 import threading
 
-class ExcludeListItem(object):
+class ExcludeListItem():
     """
     Represents details of excluded port.
 
@@ -33,7 +33,7 @@ def __init__(self, port_name, ttl_seconds):
         self.remove_time = 0 if ttl_seconds == 0 else time.time() + ttl_seconds
 
 
-class ExcludeList(object):
+class ExcludeList():
     """
     Implements list for excluded ports.
 
@@ -71,7 +71,7 @@ def add(self, port_name, ttl_seconds = 0):
     def contains(self, port_name):
         """
         Check if port exists.
-        Remove the port if its remove time is reached. 
+        Remove the port if its remove time is reached.
         :param port_name: The name of the port.
         :return: True if the port still exists, False otherwise.
         """
@@ -81,10 +81,10 @@ def contains(self, port_name):
                 if data.remove_time == 0 or time.time() < data.remove_time:
                     # Excluded port
                     return True
-                else:
-                    # The time is expired, so remove port from the list
-                    self.__dict.pop(port_name)
-                    self.__logger.info(f"Port {port_name} automatically removed from exclude list after {data.ttl_seconds} seconds")
+
+                # The time is expired, so remove port from the list
+                self.__dict.pop(port_name)
+                self.__logger.info(f"Port {port_name} automatically removed from exclude list after {data.ttl_seconds} seconds")
         return False
 
 
@@ -98,8 +98,7 @@ def remove(self, port_name):
                 self.__dict.pop(port_name)
                 self.__logger.info(f"Port {port_name} removed from exclude list")
                 return True
-            else:
-                return False
+            return False
 
 
     def refresh(self):