Merge pull request #21 from kiwix/worker-manger

Set up worker manager to run tasks in containers

.github/workflows/worker-manager-Publish.yaml

@@ -3,7 +3,7 @@ name: Publish worker-manager Docker image
 on:
   push:
     paths:
-      - 'worker/**'
+      - 'worker/manager/**'
     branches:
       - main
 
@@ -22,8 +22,7 @@ jobs:
           latest-on-tag: true
           tag-pattern: /^v([0-9.]+)$/
           restrict-to: kiwix/mirrors-qa
-          context: worker
-          dockerfile: manager.Dockerfile
+          context: worker/manager
           registries: ghcr.io
           credentials:
             GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}

.github/workflows/worker-manager-QA.yaml

@@ -0,0 +1,40 @@
+name: Worker Manager QA
+
+on:
+  pull_request:
+  push:
+    paths:
+      - 'worker/manager/**'
+    branches:
+      - main
+
+jobs:
+
+  check-qa:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version-file: worker/manager/pyproject.toml
+          architecture: x64
+
+      - name: Install dependencies (and project)
+        working-directory: worker/manager
+        run: |
+          pip install -U pip
+          pip install -e .[lint,scripts,test,check]
+
+      - name: Check black formatting
+        working-directory: worker/manager
+        run: inv lint-black
+
+      - name: Check ruff
+        working-directory: worker/manager
+        run: inv lint-ruff
+
+      - name: Check pyright
+        working-directory: worker/manager
+        run: inv check-pyright

.github/workflows/worker-task-Publish.yaml

@@ -0,0 +1,31 @@
+name: Publish worker-task Docker image
+
+on:
+  push:
+    paths:
+      - 'worker/task/**'
+    branches:
+      - main
+
+jobs:
+
+  publish-worker-task:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Retrieve source code
+        uses: actions/checkout@v4
+
+      - name: Build and publish Docker Image
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: kiwix/mirrors-qa-task-worker
+          latest-on-tag: true
+          tag-pattern: /^v([0-9.]+)$/
+          restrict-to: kiwix/mirrors-qa
+          context: worker/task
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+          repo_description: auto
+          repo_overview: auto

.github/workflows/worker-QA.yaml → .github/workflows/worker-task-QA.yaml

@@ -1,10 +1,10 @@
-name: Worker QA
+name: Worker Task QA
 
 on:
   pull_request:
   push:
     paths:
-      - 'worker/**'
+      - 'worker/task/**'
     branches:
       - main
 
@@ -18,23 +18,23 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: worker/pyproject.toml
+          python-version-file: worker/task/pyproject.toml
           architecture: x64
 
       - name: Install dependencies (and project)
-        working-directory: worker
+        working-directory: worker/task
         run: |
           pip install -U pip
           pip install -e .[lint,scripts,test,check]
 
       - name: Check black formatting
-        working-directory: worker
+        working-directory: worker/task
         run: inv lint-black
 
       - name: Check ruff
-        working-directory: worker
+        working-directory: worker/task
         run: inv lint-ruff
 
       - name: Check pyright
-        working-directory: worker
+        working-directory: worker/task
         run: inv check-pyright

.gitignore

@@ -162,3 +162,11 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# project files
+*.pem
+*.conf
+dev/data/**
+!dev/data/README.md
+!dev/.env
+id_rsa

README.md

@@ -1,2 +1,3 @@
 # mirrors-qa
+
 Q/A tools for Kiwix Download Mirrors

backend/Dockerfile

@@ -1,11 +1,12 @@
 FROM python:3.11-slim-bookworm
-LABEL org.opencontainers.image.source=https://github.com/kiwix/mirrors-qa
-# Copy code
+LABEL org.opencontainers.image.source=https://github.com/kiwix/mirrors-qa/backend
+
+RUN apt-get update && apt-get install -y curl
+
 COPY src /src/src
-# Copy pyproject.toml and its dependencies
+
 COPY pyproject.toml README.md /src/
 
-# Install + cleanup
 RUN pip install --no-cache-dir /src \
  && rm -rf /src
 

backend/pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "cryptography==42.0.8",
     "PyJWT==2.8.0",
     "paramiko==3.4.0",
+    "humanfriendly==10.0",
 ]
 license = {text = "GPL-3.0-or-later"}
 classifiers = [
@@ -37,8 +38,7 @@ dynamic = ["version"]
 Homepage = "https://github.com/kiwix/mirrors-qa"
 
 [project.scripts]
-update-mirrors = "mirrors_qa_backend.entrypoint:main"
-mirrors-qa-scheduler = "mirrors_qa_backend.scheduler:main"
+mirrors-qa-backend = "mirrors_qa_backend.entrypoint:main"
 
 [project.optional-dependencies]
 scripts = [

backend/src/mirrors_qa_backend/cli/mirrors.py

@@ -0,0 +1,21 @@
+import sys
+
+from mirrors_qa_backend import logger
+from mirrors_qa_backend.db import Session
+from mirrors_qa_backend.db.mirrors import create_or_update_mirror_status
+from mirrors_qa_backend.exceptions import MirrorsRequestError
+from mirrors_qa_backend.extract import get_current_mirrors
+
+
+def update_mirrors() -> None:
+    logger.info("Updating mirrors list.")
+    try:
+        with Session.begin() as session:
+            results = create_or_update_mirror_status(session, get_current_mirrors())
+    except MirrorsRequestError as exc:
+        logger.info(f"error while updating mirrors: {exc}")
+        sys.exit(1)
+    logger.info(
+        f"Updated mirrors list. Added {results.nb_mirrors_added} mirror(s), "
+        f"disabled {results.nb_mirrors_disabled} mirror(s)"
+    )

backend/src/mirrors_qa_backend/scheduler.py → backend/src/mirrors_qa_backend/cli/scheduler.py

@@ -9,13 +9,17 @@
 from mirrors_qa_backend.settings.scheduler import SchedulerSettings
 
 
-def main():
+def main(
+    sleep_seconds: float = SchedulerSettings.SLEEP_SECONDS,
+    expire_tests_since: float = SchedulerSettings.EXPIRE_TEST_SECONDS,
+    workers_since: float = SchedulerSettings.IDLE_WORKER_SECONDS,
+):
     while True:
         with Session.begin() as session:
             # expire tests whose results have not been reported
             expired_tests = expire_tests(
                 session,
-                interval=datetime.timedelta(hours=SchedulerSettings.EXPIRE_TEST_HOURS),
+                interval=datetime.timedelta(seconds=expire_tests_since),
             )
             for expired_test in expired_tests:
                 logger.info(
@@ -26,7 +30,9 @@ def main():
 
             idle_workers = get_idle_workers(
                 session,
-                interval=datetime.timedelta(hours=SchedulerSettings.IDLE_WORKER_HOURS),
+                interval=datetime.timedelta(
+                    seconds=workers_since,
+                ),
             )
             if not idle_workers:
                 logger.info("No idle workers found.")
@@ -69,9 +75,5 @@ def main():
                         f"{idle_worker.id} in country {country.name}"
                     )
 
-        sleep_interval = datetime.timedelta(
-            hours=SchedulerSettings.SCHEDULER_SLEEP_HOURS
-        ).total_seconds()
-
-        logger.info(f"Sleeping for {sleep_interval} seconds.")
-        time.sleep(sleep_interval)
+        logger.info(f"Sleeping for {sleep_seconds} seconds.")
+        time.sleep(sleep_seconds)

backend/src/mirrors_qa_backend/cli/worker.py

@@ -0,0 +1,42 @@
+import sys
+
+import pycountry
+from cryptography.hazmat.primitives import serialization
+
+from mirrors_qa_backend import logger
+from mirrors_qa_backend.db import Session
+from mirrors_qa_backend.db.worker import create_worker as create_db_worker
+
+
+def create_worker(worker_id: str, private_key_data: bytes, country_codes: list[str]):
+    # Ensure all the countries are valid country codes
+    for country_code in country_codes:
+        if len(country_code) != 2:  # noqa: PLR2004
+            logger.info(f"Country code '{country_code}' must be two characters long")
+            sys.exit(1)
+
+        if not pycountry.countries.get(alpha_2=country_code):
+            logger.info(f"'{country_code}' is not valid country code")
+            sys.exit(1)
+
+    try:
+        private_key = serialization.load_pem_private_key(
+            private_key_data, password=None
+        )  # pyright: ignore[reportReturnType]
+    except Exception as exc:
+        logger.info(f"Unable to load private key: {exc}")
+        sys.exit(1)
+
+    try:
+        with Session.begin() as session:
+            create_db_worker(
+                session,
+                worker_id,
+                country_codes,
+                private_key,  # pyright: ignore [reportGeneralTypeIssues, reportArgumentType]
+            )
+    except Exception as exc:
+        logger.info(f"error while creating worker: {exc}")
+        sys.exit(1)
+
+    logger.info(f"Created worker {worker_id} successfully")

backend/src/mirrors_qa_backend/cryptography.py

@@ -1,5 +1,4 @@
 # pyright: strict, reportGeneralTypeIssues=false
-from pathlib import Path
 
 import paramiko
 from cryptography.exceptions import InvalidSignature
@@ -43,13 +42,6 @@ def sign_message(private_key: RSAPrivateKey, message: bytes) -> bytes:
     )
 
 
-def load_private_key_from_path(private_key_fpath: Path) -> RSAPrivateKey:
-    with private_key_fpath.open("rb") as key_file:
-        return serialization.load_pem_private_key(
-            key_file.read(), password=None
-        )  # pyright: ignore[reportReturnType]
-
-
 def generate_public_key(private_key: RSAPrivateKey) -> RSAPublicKey:
     return private_key.public_key()
 

backend/src/mirrors_qa_backend/db/worker.py

@@ -1,19 +1,17 @@
 import datetime
-from pathlib import Path
 
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
 from sqlalchemy import select
 from sqlalchemy.orm import Session as OrmSession
 
 from mirrors_qa_backend.cryptography import (
     generate_public_key,
     get_public_key_fingerprint,
-    load_private_key_from_path,
     serialize_public_key,
 )
 from mirrors_qa_backend.db.country import get_countries
 from mirrors_qa_backend.db.exceptions import DuplicatePrimaryKeyError
 from mirrors_qa_backend.db.models import Worker
-from mirrors_qa_backend.exceptions import PEMPrivateKeyLoadError
 
 
 def get_worker(session: OrmSession, worker_id: str) -> Worker | None:
@@ -24,15 +22,11 @@ def create_worker(
     session: OrmSession,
     worker_id: str,
     country_codes: list[str],
-    private_key_fpath: Path,
+    private_key: RSAPrivateKey,
 ) -> Worker:
     """Creates a worker using RSA private key."""
     if get_worker(session, worker_id) is not None:
         raise DuplicatePrimaryKeyError(f"A worker with id {worker_id} already exists.")
-    try:
-        private_key = load_private_key_from_path(private_key_fpath)
-    except Exception as exc:
-        raise PEMPrivateKeyLoadError("unable to load private key from file") from exc
 
     public_key = generate_public_key(private_key)
     public_key_pkcs8 = serialize_public_key(public_key).decode(encoding="ascii")