Merge pull request apel#358 from apel/release-1.9.3

Release 1.9.3 to master

.github/workflows/build-pkgs.yml

@@ -19,7 +19,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install rpmlint
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         # Get all branches and tags so the latest tag can be found for VERSION
         fetch-depth: 0
@@ -48,7 +48,7 @@ jobs:
       run: rpmlint ${{ steps.rpm.outputs.rpm_dir_path }}
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v3.1.2
+      uses: actions/upload-artifact@v4.3
       with:
         name: Binary and Source RPMs
         path: |

.github/workflows/codeql-analysis.yml

@@ -32,19 +32,19 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
-        
+
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         queries: security-extended,security-and-quality
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/unit-test.yml

@@ -0,0 +1,69 @@
+name: Run Unit Tests
+
+on: [push, pull_request]
+
+jobs:
+  unit-test:
+    runs-on: ubuntu-20.04
+    services:
+      mariadb:
+        image: mariadb:10.4
+        env:
+          MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 'yes'
+          MARIADB_DATABASE: apel_unittest
+        ports:
+          - 3306:3306
+        ################
+        # Options are docker container options to see the container health status
+        # Reference: https://docs.docker.com/engine/reference/run/#healthchecks
+        #
+        # --health-cmd         Command to run to check health
+        # --health-interval    Time between running the check
+        # --health-retries     Consecutive failures needed to report unhealthy
+        # --health-timeout     Maximum time to allow one check to run
+        ################
+        options: >-
+          --health-cmd="mysqladmin ping -h localhost"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.6', '3.9']
+    name: Python ${{ matrix.python-version }} test
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set up dependencies for python-ldap
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev
+
+    - name: Set up dependencies for mysqlclient
+      run: sudo apt-get install default-libmysqlclient-dev pkg-config
+
+    - name: Base requirements for SSM
+      run: pip install -r requirements.txt
+
+    - name: Additional requirements for the unit and coverage tests
+      run: pip install -r requirements-test.txt
+
+    - name: Pre-test setup
+      run: |
+        export TMPDIR=$PWD/tmp
+        mkdir $TMPDIR
+        export PYTHONPATH=$PYTHONPATH:`pwd -P`
+        cd test
+
+    - name: Run unit tests
+      run: coverage run --branch --source=apel,bin -m unittest discover -s test --buffer
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4

.pre-commit-config.yaml

@@ -15,15 +15,26 @@ repos:
   - id: check-added-large-files
   - id: check-merge-conflict
   - id: check-yaml
+  - id: debug-statements
   - id: end-of-file-fixer
   - id: mixed-line-ending
     name: Force line endings to LF
     args: ['--fix=lf']
   - id: trailing-whitespace
 
 - repo: https://github.com/pre-commit/pygrep-hooks
-  rev: v1.9.0
+  rev: v1.10.0
   hooks:
   - id: python-check-mock-methods
   - id: python-no-eval
   - id: python-no-log-warn
+  - id: python-use-type-annotations
+    # This file has a string that causes a false positive
+    exclude: 'test_summariser.py'
+
+# Pre-commit CI config, see https://pre-commit.ci/
+ci:
+  autofix_prs: false
+  autoupdate_schedule: quarterly
+  # Skipping as CI runs in Python 3
+  skip: [check-ast, check-builtin-literals, debug-statements]

CHANGELOG

@@ -1,5 +1,11 @@
 Changelog for apel
 ==================
+ * Thu May 25 2024 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.9.3-1
+ - [parsers] Reduced verbosity of parser to only report a single time that logs have been skipped.
+ - Fixed some unit tests so that they run properly.
+ - Refactored in support of move to Python 3.
+ - Updated CI config to update tooling.
+
  * Fri May 26 2023 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.9.2-1
  - [client] Enabled HEPscore23 as benchmark type for Job Records (using existing message format).
  - [client] Changed default to EGI BDII for fetching benchmarks as CERN BDII will be decommissioned.

apel.spec

@@ -4,7 +4,7 @@
 %endif
 
 Name:           apel
-Version:        1.9.2
+Version:        1.9.3
 %define releasenumber 1
 Release:        %{releasenumber}%{?dist}
 Summary:        APEL packages
@@ -199,6 +199,12 @@ exit 0
 # ==============================================================================
 
 %changelog
+ * Thu May 25 2024 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.9.3-1
+ - [parsers] Reduced verbosity of parser to only report a single time that logs have been skipped.
+ - Fixed some unit tests so that they run properly.
+ - Refactored in support of move to Python 3.
+ - Updated CI config to update tooling.
+
  * Fri May 26 2023 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 1.9.2-1
  - [client] Enabled HEPscore23 as benchmark type for Job Records (using existing message format).
  - [client] Changed default to EGI BDII for fetching benchmarks as CERN BDII will be decommissioned.

apel/__init__.py

@@ -15,4 +15,4 @@
 
    @author Konrad Jopek, Will Rogers
 '''
-__version__ = (1, 9, 2)
+__version__ = (1, 9, 3)

apel/db/backends/mysql.py

@@ -101,7 +101,7 @@ def __init__(self, host, port, username, pwd, db):
             self.db = MySQLdb.connect(host=self._db_host, port=self._db_port,
                                       user=self._db_username, passwd=self._db_pwd,
                                       db=self._db_name)
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error('Error connecting to database: %s', e)
             raise ApelDbException(e)
 
@@ -118,7 +118,7 @@ def test_connection(self):
             log.info('Connected to %s:%s', self._db_host, self._db_port)
             log.info('Database: %s; username: %s', self._db_name, self._db_username)
             db.close()
-        except MySQLdb.OperationalError, e:
+        except MySQLdb.OperationalError as e:
             raise ApelDbException("Failed to connect to database: " + str(e))
 
     def load_records(self, record_list, replace=True, source=None):
@@ -165,7 +165,7 @@ def load_records(self, record_list, replace=True, source=None):
                     proc = self.REPLACE_PROCEDURES[type(record)]
                 c.execute(proc, values)
             self.db.commit()
-        except (MySQLdb.Warning, MySQLdb.Error, KeyError), err:
+        except (MySQLdb.Warning, MySQLdb.Error, KeyError) as err:
             log.error("Error loading records: %s", err)
             log.error("Transaction will be rolled back.")
             self.db.rollback()
@@ -233,12 +233,12 @@ def _get_records(self, record_type, query_string, records_per_message=1000):
                 else:
                     break
 
-        except MySQLdb.Error, err:
+        except MySQLdb.Error as err:
             log.error('Error during getting records: %s', err)
             log.error('Transaction will be rolled back.')
             self.db.rollback()
             raise ApelDbException(err)
-        except MySQLdb.Warning, warning:
+        except MySQLdb.Warning as warning:
             log.warning('Warning from MySQL: %s', warning)
 
     def get_last_updated(self):
@@ -285,7 +285,7 @@ def check_duplicate_sites(self):
             if conflict:
                 raise ApelDbException("Records exist in both job and summary tables for the same site.")
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -313,7 +313,7 @@ def summarise_jobs(self):
             c.callproc(self._summarise_jobs_proc, ())
             log.info("Done.")
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -344,7 +344,7 @@ def normalise_summaries(self):
             c.callproc(self._normalise_summaries_proc, ())
             log.info("Done.")
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A MySQL error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -367,7 +367,7 @@ def copy_summaries(self):
             log.info("Done.")
 
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -396,7 +396,7 @@ def summarise_cloud(self):
             log.info("Done.")
 
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -422,7 +422,7 @@ def join_records(self):
             c.callproc(self._join_records_proc)
             log.info("Done.")
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -442,7 +442,7 @@ def create_local_jobs(self):
             c.callproc(self._local_jobs_proc)
             log.info("Done.")
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -489,7 +489,7 @@ def update_spec(self, site, ce, spec_level_type, spec_level):
 
             self.db.commit()
 
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
 
@@ -508,7 +508,7 @@ def clean_processed_files(self, hostname):
             c = self.db.cursor()
             c.execute(self._processed_clean, [hostname])
             self.db.commit()
-        except MySQLdb.Error, e:
+        except MySQLdb.Error as e:
             log.error("A mysql error occurred: %s", e)
             log.error("Any transaction will be rolled back.")
             if self.db is not None:
@@ -526,6 +526,6 @@ def _mysql_reconnect(self):
                 self.db = MySQLdb.connect(host=self._db_host, port=self._db_port,
                                           user=self._db_username, passwd=self._db_pwd,
                                           db=self._db_name)
-            except MySQLdb.Error, e:
+            except MySQLdb.Error as e:
                 log.error('Error connecting to database: %s', e)
                 raise ApelDbException(e)

apel/db/loader/loader.py

@@ -93,7 +93,7 @@ def startup(self):
             f.write(str(os.getpid()))
             f.write("\n")
             f.close()
-        except IOError, e:
+        except IOError as e:
             log.warning("Failed to create pidfile %s: %s", self._pidfile, e)
 
     def shutdown(self):
@@ -105,7 +105,7 @@ def shutdown(self):
         if hasattr(self, 'current_msg') and self.current_msg:
             try:
                 self._inq.unlock(self.current_msg)
-            except OSError, e:
+            except OSError as e:
                 log.error('Unable to remove lock: %s', e)
 
         pidfile = self._pidfile
@@ -114,7 +114,7 @@ def shutdown(self):
                 os.remove(pidfile)
             else:
                 log.warning("pidfile %s not found.", pidfile)
-        except IOError, e:
+        except IOError as e:
             log.warning("Failed to remove pidfile %s: %s", pidfile, e)
             log.warning("The loader may not start again until it is removed.")
 
@@ -156,7 +156,7 @@ def load_all_msgs(self):
 
             except (RecordFactoryException, LoaderException,
                     InvalidRecordException, apel.db.ApelDbException,
-                    XMLParserException, ExpatError), err:
+                    XMLParserException, ExpatError) as err:
                 errmsg = "Parsing unsuccessful: %s" % str(err)
                 log.warning('Message rejected. %s', errmsg)
                 name = self._rejectq.add({"body": msg_text,
@@ -178,7 +178,7 @@ def load_all_msgs(self):
                 # The accept queue is only created if _save_msgs is true.
                 if self._save_msgs:
                     self._acceptq.purge()
-            except OSError, e:
+            except OSError as e:
                 log.warning('OSError raised while purging message queues: %s', e)
 
         log.debug("Loader run finished.")