fix(backend): env and scrapers.json file loading

EliotAmn · Jan 16, 2025 · 41c3cc9 · 41c3cc9
1 parent 5d3d855
commit 41c3cc9
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 16 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,4 @@
 /alembic.ini
 /.idea
 /scrapers-config.json
+/scrapers.json
diff --git a/Dockerfile b/Dockerfile
@@ -39,7 +39,8 @@ ENV PYTHONPATH=/app \
     PYTHONUNBUFFERED=1 \
     FLASK_ENV=production \
     FLASK_DEBUG=0 \
-    DASHBOARD_BUILD_PATH=/app/dashboard_build
+    DASHBOARD_BUILD_PATH=/app/dashboard_build \
+    SCRAPERS_CONFIG_FILE=/app/scrapers.json
 
 RUN useradd -m -s /bin/bash appuser && \
     chown -R appuser:appuser /app

diff --git a/README.md b/README.md
@@ -7,16 +7,16 @@ their daily life.
 
 ## Environment variables
 
-| Variable          | Description                                                                                                | Default value |
-|-------------------|------------------------------------------------------------------------------------------------------------|---------------|
-| PORT              | Port where the server will listen                                                                          | 3000          |
-| POSTGRES_USER     | Postgres user                                                                                              | postgres      |
-| POSTGRES_HOST     | Postgres host                                                                                              | postgres      |
-| POSTGRES_DB       | Postgres database                                                                                          | tekbetter     |
-| POSTGRES_PASSWORD | Postgres password                                                                                          | postgres      |
-| POSTGRES_PORT     | Postgres port                                                                                              | 5432          |
-| JWT_SECRET        | JWT secret                                                                                                 |               |
-| SHOW_BROWSER      | Disable the headless mode for the chromium browser (used for microsoft auth and intranet AntiDDoS system). | false         |
-| SHOW_SQL_LOGS     | Show SQL logs in the console.                                                                              | false         |
-| AES_KEY           | AES key used to encrypt sensible user informations                                                         |               |
-
+| Variable             | Description                                                                | Default value |
+|----------------------|----------------------------------------------------------------------------|---------------|
+| PORT                 | Port where the server will listen                                          | 3000          |
+| MONGO_HOST           | The mongodb database hostname.                                             | mongo         |
+| MONGO_PORT           | The mongodb database port.                                                 | 27017         |
+| MONGO_DB             | The mongodb database name.                                                 | tekbetter     |
+| REDIS_HOST           | The redis database hostname.                                               | redis         |
+| REDIS_PORT           | The redis database port.                                                   | 6379          |
+| REDIS_PASSWORD       | The redis database password.                                               |               |
+| REDIS_DB             | The redis database number.                                                 | 0             |
+| JWT_SECRET           | The secret key used to sign the JWT token.                                 |               |
+| AES_KEY              | The key used to encrypt sensible data in database. 64 bytes long required. |               |
+| SCRAPERS_CONFIG_FILE | The path to the scrapers configuration file. Optional                      | scrapers.json |
diff --git a/app/main.py b/app/main.py
@@ -19,6 +19,9 @@
 _is_initialized = False
 
 
+os.chdir(os.path.join(os.path.dirname(__file__), ".."))
+
+
 def init_services():
     """Initialize database connections and services"""
     global _is_initialized

diff --git a/app/services/publicscraper_service.py b/app/services/publicscraper_service.py
@@ -4,11 +4,14 @@
 from app.globals import Globals
 from app.models.PublicScraper import PublicScraper
 from app.services.student_service import StudentService
+from app.tools.teklogger import log_warning
 
 
 def get_config():
     if not os.path.exists(os.getenv("SCRAPERS_CONFIG_FILE")):
-        raise Exception("Invalid SCRAPER_CONFIG_FILE path")
+        log_warning("Scrapers config file does not exist, creating a new one at " + os.getenv("SCRAPERS_CONFIG_FILE"))
+        with open(os.getenv("SCRAPERS_CONFIG_FILE"), "w") as f:
+            f.write("[]")
     if not os.access(os.getenv("SCRAPERS_CONFIG_FILE"), os.R_OK):
         raise Exception(f"{os.getenv('SCRAPERS_CONFIG_FILE')} is not readable")
     if not os.access(os.getenv("SCRAPERS_CONFIG_FILE"), os.W_OK):

diff --git a/app/tools/envloader.py b/app/tools/envloader.py
@@ -15,7 +15,7 @@
     "REDIS_PASSWORD": None,
     "JWT_SECRET": None,
     "AES_KEY": None,
-    "SCRAPERS_CONFIG_FILE": "",
+    "SCRAPERS_CONFIG_FILE": "scrapers.json",
 }