issue: 3799995: Integrate PEP8/ pylint into TFS CI process (#210)

.github/workflows/tfs_plugin_ci_workflow.yml

@@ -0,0 +1,24 @@
+name: TFS Plugin CI Workflow
+
+on:
+  push:
+     paths:
+      - 'plugins/fluentd_telemetry_plugin/**'
+jobs:
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@main
+
+    - name: Set up Python
+      uses: actions/setup-python@main
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      run: |
+        pip install -r plugins/fluentd_telemetry_plugin/requirements.txt
+        pip install pylint
+    - name: Run PyLint
+      run: pylint --rcfile=plugins/fluentd_telemetry_plugin/.pylintrc plugins/fluentd_telemetry_plugin

plugins/fluentd_telemetry_plugin/.pylintrc

@@ -0,0 +1,17 @@
+[MASTER]
+init-hook="import os, sys; sys.path.append(os.path.join(os.getcwd(), 'plugins', 'fluentd_telemetry_plugin', 'src')); sys.path.append(os.path.join(os.getcwd(), 'utils'))"
+
+[MAIN]
+max-public-methods=100
+
+[DESIGN]
+max-attributes=10
+
+[MESSAGES CONTROL]
+disable=missing-module-docstring,missing-function-docstring,fixme
+
+[FORMAT]
+max-line-length=140
+
+[BASIC]
+min-public-methods=0

plugins/fluentd_telemetry_plugin/requirements.txt

@@ -8,4 +8,4 @@ twisted
 tzlocal<3.0
 jsonschema
 prometheus_client
-msgpack
+msgpack

plugins/fluentd_telemetry_plugin/src/__init__.py

@@ -0,0 +1,15 @@
+"""
+@copyright:
+    Copyright (C) Mellanox Technologies Ltd. 2014-2020.  ALL RIGHTS RESERVED.
+
+    This software product is a proprietary product of Mellanox Technologies
+    Ltd. (the "Company") and all right, title, and interest in and to the
+    software product, including all associated intellectual property rights,
+    are and shall remain exclusively with the Company.
+
+    This software product is governed by the End User License Agreement
+    provided with the software product.
+
+@author: Anan Al-Aghbar
+@date:   May 9, 2024
+"""

plugins/fluentd_telemetry_plugin/src/api/__init__.py

@@ -1,4 +1,5 @@
 class InvalidConfRequest(Exception):
+    """InvalidConfRequest Exception class"""
 
     def __init__(self, message):
-        Exception.__init__(self,message)
+        Exception.__init__(self, message)

plugins/fluentd_telemetry_plugin/src/api/base_api.py

@@ -1,20 +1,23 @@
-from flask import Flask
-from flask_restful import Api
 from http import HTTPStatus
 from functools import partial
-from utils.json_schema_validator import ValidationError, SchemaValidationError
+from flask import Flask
+from flask_restful import Api
 from web_service_error_messages import \
-    no_running_streaming_instance,\
-    streaming_already_running
+    NO_RUNNING_STREAMING_INSTANCE,\
+    STREAMING_ALREADY_RUNNING
 
 from streaming_scheduler import \
     NoRunningStreamingInstance,\
     StreamingAlreadyRunning
 
 from api import InvalidConfRequest
 
+# pylint: disable=no-name-in-module,import-error
+from utils.json_schema_validator import ValidationError, SchemaValidationError
+
 
 class BaseAPIApplication:
+    """BaseAPIApplication API Flask Class Base"""
 
     def __init__(self):
         self.app = Flask(__name__)
@@ -26,9 +29,9 @@ def __init__(self):
     def _get_error_handlers(self):
         return [
             (NoRunningStreamingInstance,
-             lambda e: (no_running_streaming_instance, HTTPStatus.BAD_REQUEST)),
+             lambda e: (NO_RUNNING_STREAMING_INSTANCE, HTTPStatus.BAD_REQUEST)),
             (StreamingAlreadyRunning,
-             lambda e: (streaming_already_running, HTTPStatus.BAD_REQUEST)),
+             lambda e: (STREAMING_ALREADY_RUNNING, HTTPStatus.BAD_REQUEST)),
             (InvalidConfRequest,
              lambda e: (str(e), HTTPStatus.BAD_REQUEST)),
             (ValidationError,
@@ -41,8 +44,8 @@ def _get_error_handlers(self):
 
     def _add_error_handlers(self):
         hdlrs = self._get_error_handlers()
-        for code_or_exception, f in hdlrs:
-            self.app.register_error_handler(code_or_exception, f)
+        for code_or_exception, callback in hdlrs:
+            self.app.register_error_handler(code_or_exception, callback)
 
     @property
     def application(self):

plugins/fluentd_telemetry_plugin/src/api/conf_api.py

@@ -1,19 +1,23 @@
 import re
 import json
 import logging
+from http import HTTPStatus
 from flask import make_response, request
 from api import InvalidConfRequest
 from api.base_api import BaseAPIApplication
 from streamer import UFMTelemetryStreaming
 from streaming_scheduler import StreamingScheduler
+
+# pylint: disable=no-name-in-module,import-error
 from utils.json_schema_validator import validate_schema
 from utils.utils import Utils
 
 
 class StreamingConfigurationsAPI(BaseAPIApplication):
+    """StreamingConfigurationsAPI class"""
 
     def __init__(self, conf):
-        super(StreamingConfigurationsAPI, self).__init__()
+        super(StreamingConfigurationsAPI, self).__init__()  # pylint: disable=super-with-arguments
         self.conf = conf
         self.scheduler = StreamingScheduler.getInstance()
         self.streamer = UFMTelemetryStreaming.getInstance()
@@ -26,10 +30,10 @@ def __init__(self, conf):
 
     def _get_routes(self):
         return {
-            self.get: dict(urls=["/"], methods=["GET"]),
-            self.post: dict(urls=["/"], methods=["POST"]),
-            self.get_streaming_attributes: dict(urls=["/attributes"], methods=["GET"]),
-            self.update_streaming_attributes: dict(urls=["/attributes"], methods=["POST"])
+            self.get: {'urls': ["/"], 'methods': ["GET"]},
+            self.post: {'urls': ["/"], 'methods': ["POST"]},
+            self.get_streaming_attributes: {'urls': ["/attributes"], 'methods': ["GET"]},
+            self.update_streaming_attributes: {'urls': ["/attributes"], 'methods': ["POST"]}
         }
 
     def _set_new_conf(self):
@@ -68,7 +72,7 @@ def _set_new_conf(self):
     def _validate_required_configurations_on_enable(self):
         # just checking the required attributes
         # if one of the attributes below was missing it will throw an exception
-        fluentd_host = self.conf.get_fluentd_host()
+        return self.conf.get_fluentd_host()
 
     def post(self):
         # validate the new conf json
@@ -89,15 +93,15 @@ def post(self):
             self.conf.update_config_file(self.conf.config_file)
             raise ex
 
-    def get(self):
+    def get(self):  # pylint: disable=too-many-locals, too-many-branches
         try:
-            with open(Utils.get_absolute_path(self.conf_schema_path)) as json_data:
+            with open(Utils.get_absolute_path(self.conf_schema_path), encoding='utf-8') as json_data:
                 schema = json.load(json_data)
                 properties = schema.get('properties', None)
                 if properties is None:
                     raise InvalidConfRequest("Failed to get the configurations schema properties")
                 conf_dict = {}
-                for section in self.conf.get_conf_sections():
+                for section in self.conf.get_conf_sections():  # pylint: disable=too-many-nested-blocks
                     section_properties = properties.get(section, None)
                     if section_properties is None:
                         raise InvalidConfRequest("Failed to get the configurations schema for the section: " + section)
@@ -116,13 +120,12 @@ def get(self):
                                                              f'under the section: {section}')
                                 item_type = item_type.get('type', None)
                                 item_values = item_value.split(",")
-                                for i in range(len(item_values)):
+                                for i, value in enumerate(item_values):
                                     try:
                                         arr_element_obj = conf_dict[section][i]
                                     except IndexError:
                                         arr_element_obj = {}
                                         conf_dict[section].append(arr_element_obj)
-                                    value = item_values[i]
                                     arr_element_obj[item_key] = Utils.convert_str_to_type(value, item_type)
                                     conf_dict[section][i] = arr_element_obj
                     elif section_type == "object":
@@ -143,8 +146,10 @@ def get(self):
                         raise InvalidConfRequest(f'Failed to get the configurations, unsupported type '
                                                  f'{section_type}: for the section: {section}')
                 return make_response(conf_dict)
-        except InvalidConfRequest as e:
-            logging.error("Error occurred while getting the current streaming configurations: " + str(e))
+        except InvalidConfRequest as ex:
+            err_msg = f"Error occurred while getting the current streaming configurations: {str(ex)}"
+            logging.error(err_msg)
+            return make_response(err_msg, HTTPStatus.INTERNAL_SERVER_ERROR)
 
     def get_streaming_attributes(self):
         return make_response(self.streamer.streaming_attributes)

plugins/fluentd_telemetry_plugin/src/api/streaming_monitoring_stats_api.py

@@ -18,13 +18,15 @@
 
 
 class StreamingMonitoringStatsAPI(BaseAPIApplication):
+    """StreamingMonitoringStatsAPI API class"""
+
     def __init__(self):
-        super(StreamingMonitoringStatsAPI, self).__init__()
+        super(StreamingMonitoringStatsAPI, self).__init__()  # pylint: disable=super-with-arguments
         self.streamer = UFMTelemetryStreaming.getInstance()
 
     def _get_routes(self):
         return {
-            self.get: dict(urls=["/"], methods=["GET"])
+            self.get: {'urls': ["/"], 'methods': ["GET"]}
         }
 
     def get(self):

plugins/fluentd_telemetry_plugin/src/app.py

@@ -17,18 +17,21 @@
 import sys
 sys.path.append(os.getcwd())
 
+# pylint: disable=wrong-import-position
 import logging
-from utils.flask_server import run_api
-from utils.args_parser import ArgsParser
-from utils.logger import Logger
-from utils.utils import Utils
 from web_service import UFMTelemetryFluentdStreamingAPI
 from streamer import \
     UFMTelemetryStreaming,\
     UFMTelemetryStreamingConfigParser,\
     UFMTelemetryConstants
 from streaming_scheduler import StreamingScheduler
 
+# pylint: disable=no-name-in-module,import-error
+from utils.flask_server import run_api
+from utils.args_parser import ArgsParser
+from utils.logger import Logger
+from utils.utils import Utils
+
 
 def _init_logs(config_parser):
     # init logs configs
@@ -44,31 +47,32 @@ def _init_logs(config_parser):
 
     # init app config parser & load config files
     args = ArgsParser.parse_args("UFM Telemetry Streaming to fluentd", UFMTelemetryConstants.args_list)
-    config_parser = UFMTelemetryStreamingConfigParser(args)
+    _config_parser = UFMTelemetryStreamingConfigParser(args)
 
-    _init_logs(config_parser)
+    _init_logs(_config_parser)
 
-    streamer = None
+    STREAMER = None
     try:
-        streamer = UFMTelemetryStreaming.getInstance(config_parser)
-        if config_parser.get_enable_streaming_flag():
+        STREAMER = UFMTelemetryStreaming.getInstance(_config_parser)
+        if _config_parser.get_enable_streaming_flag():
             scheduler = StreamingScheduler.getInstance()
             job_id = scheduler.start_streaming()
             logging.info("Streaming has been started successfully")
         else:
             logging.warning("Streaming was not started, need to enable the streaming & set the required configurations")
 
     except ValueError as ex:
-        logging.warning("Streaming was not started, need to enable the streaming & set the required configurations. "+ str(ex))
-    except Exception as ex:
-        logging.error(f'Streaming was not started due to the following error: {str(ex)}')
+        logging.warning("Streaming was not started, need to enable the streaming & set the required configurations. %s"
+                        , str(ex))
+    except Exception as ex:  # pylint: disable=broad-except
+        logging.error('Streaming was not started due to the following error: %s', str(ex))
 
-    if streamer:
+    if STREAMER:
         try:
-            app = UFMTelemetryFluentdStreamingAPI(config_parser)
+            app = UFMTelemetryFluentdStreamingAPI(_config_parser)
             port = Utils.get_plugin_port('/config/tfs_httpd_proxy.conf', 8981)
             run_api(app=app, port_number=int(port))
-        except Exception as ex:
-            logging.error(f'Streaming server was not started due to the following error: {str(ex)}')
+        except Exception as ex:  # pylint: disable=broad-except
+            logging.error('Streaming server was not started due to the following error: %s', str(ex))
     else:
-        logging.error(f'Streaming server was not started.')
+        logging.error('Streaming server was not started.')