Merge pull request #99 from hotosm/fix/encode-databases

Fix encode/databases usage, remove sqlalchemy db init

.github/workflows/build_and_deploy.yml

@@ -99,7 +99,7 @@ jobs:
 
           # Cleanup db upgrade containers
           docker compose --file contrib/pg-upgrade/docker-compose.yml down
- 
+
           docker compose --file docker-compose.vm.yml --env-file .env up \
             --detach --remove-orphans --force-recreate --pull=always
         env:

README.md

@@ -33,12 +33,11 @@
 <!-- markdownlint-restore -->
 <!-- prettier-ignore-end -->
 
-
 **Drone TM** is an integrated digital public good solution designed to harness
 the power of the crowd to generate high-resolution aerial maps of any location.
 
-This innovative platform provides drone pilots, particularly in developing 
-countries, with job opportunities while contributing to the creation of 
+This innovative platform provides drone pilots, particularly in developing
+countries, with job opportunities while contributing to the creation of
 high-resolution datasets crucial for disaster response and community resilience.
 
 ## Problem Statement

docs/INSTALL.md

@@ -1,2 +1 @@
 # Installation Guide
-

docs/about/about.md

@@ -1,2 +1 @@
 # 📖 History
-

docs/about/team.md

@@ -1 +1 @@
-# The Drone TM Team
+# The Drone TM Team

mkdocs.yml

@@ -72,4 +72,4 @@ nav:
       - Contribution Guidelines: CONTRIBUTING.md
       - Code of Conduct: https://docs.hotosm.org/code-of-conduct
       - FAQ: about/faq.md
-      - The Team: about/team.md
+      - The Team: about/team.md

src/backend/app/db/database.py

@@ -1,27 +1,17 @@
 """Config for the DTM database connection."""
+
 from databases import Database
 from app.config import settings
-from sqlalchemy import create_engine
-from sqlalchemy.orm import declarative_base, sessionmaker
-
-Base = declarative_base()
 
 
 class DatabaseConnection:
     """Manages database connection (sqlalchemy & encode databases)"""
 
     def __init__(self):
         self.database = Database(
-            settings.DTM_DB_URL.unicode_string(), min_size=5, max_size=20
-        )
-        # self.database = Database(settings.DTM_DB_URL.unicode_string())
-        self.engine = create_engine(
             settings.DTM_DB_URL.unicode_string(),
-            pool_size=20,
-            max_overflow=-1,
-        )
-        self.SessionLocal = sessionmaker(
-            autocommit=False, autoflush=False, bind=self.engine
+            min_size=5,
+            max_size=20,
         )
 
     async def connect(self):
@@ -32,27 +22,11 @@ async def disconnect(self):
         """Disconnect from the database."""
         await self.database.disconnect()
 
-    def create_db_session(self):
-        """Create a new SQLAlchemy DB session."""
-        db = self.SessionLocal()
-        try:
-            return db
-        finally:
-            db.close()
-
-
-db_connection = DatabaseConnection()  # Create a single instance
-
 
-def get_db():
-    """Yield a new database session."""
-    return db_connection.create_db_session()
+db_connection = DatabaseConnection()
 
 
-async def encode_db():
+async def get_db():
     """Get the encode database connection"""
-    try:
-        await db_connection.connect()
-        yield db_connection.database
-    finally:
-        await db_connection.disconnect()
+    await db_connection.connect()
+    yield db_connection.database

src/backend/app/db/db_models.py

@@ -14,9 +14,9 @@
     ARRAY,
     LargeBinary,
 )
+from sqlalchemy.orm import declarative_base
 from sqlalchemy.dialects.postgresql import UUID
 
-from app.db.database import Base
 from geoalchemy2 import Geometry, WKBElement
 from app.models.enums import (
     TaskStatus,
@@ -33,6 +33,9 @@
 from app.utils import timestamp
 
 
+Base = declarative_base()
+
+
 class DbUser(Base):
     __tablename__ = "users"
 

src/backend/app/email_templates/mapping_approved_or_rejected.html

@@ -115,9 +115,7 @@ <h1>{{ email_subject }}</h1>
       </div>
       <div class="content">
         <p>Dear {{ drone_operator_name }},</p>
-        <p>
-          {{ email_body }}
-        </p>
+        <p>{{ email_body }}</p>
         <p>Please find below the details of the {{ task_status }} task:</p>
         <div class="task">
           <h2>{{ task_status|capitalize }} Task Details</h2>
@@ -127,7 +125,9 @@ <h2>{{ task_status|capitalize }} Task Details</h2>
             <p><strong>Description:</strong> {{ description }}</p>
           </div>
           {% if task_status == 'approved' %}
-            <a href="https://dronetm-dev.naxa.com.np" class="task-button">Start Mapping</a>
+          <a href="https://dronetm-dev.naxa.com.np" class="task-button"
+            >Start Mapping</a
+          >
           {% endif %}
         </div>
       </div>

src/backend/app/email_templates/mapping_requests.html

@@ -125,11 +125,10 @@ <h1>Drone Tasking Manager Invite</h1>
         <div class="task">
           <h2>Mapping Task Details</h2>
           <div class="task-details">
-            <p><strong>Drone Operator:</strong> {{drone_operator_name}} </p>
-            <p><strong>Task ID:</strong>  {{task_id}} </p>
+            <p><strong>Drone Operator:</strong> {{drone_operator_name}}</p>
+            <p><strong>Task ID:</strong> {{task_id}}</p>
             <p><strong>Project:</strong>{{project_name}}</p>
-            <p>
-              <strong>Description:</strong> {{description}}</p>
+            <p><strong>Description:</strong> {{description}}</p>
           </div>
           <a href="https://dronetm-dev.naxa.com.np" class="task-button"
             >Start Mapping</a

src/backend/app/main.py

@@ -1,17 +1,20 @@
 import os
 import logging
 import sys
+from contextlib import asynccontextmanager
+from loguru import logger as log
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.templating import Jinja2Templates
+from fastapi.responses import RedirectResponse, JSONResponse
+
 from app.config import settings
 from app.projects import project_routes
 from app.waypoints import waypoint_routes
-from fastapi.responses import RedirectResponse, JSONResponse
 from app.users import oauth_routes
 from app.users import user_routes
 from app.tasks import task_routes
-from loguru import logger as log
-from fastapi.templating import Jinja2Templates
+from app.db.database import db_connection
 
 
 root = os.path.dirname(os.path.abspath(__file__))
@@ -101,6 +104,21 @@ def get_application() -> FastAPI:
     return _app
 
 
+@asynccontextmanager
+async def lifespan(
+    app: FastAPI,
+):
+    """FastAPI startup/shutdown event."""
+    log.debug("Starting up FastAPI server.")
+    await db_connection.connect()
+
+    yield
+
+    # Shutdown events
+    log.debug("Shutting down FastAPI server.")
+    await db_connection.disconnect()
+
+
 api = get_application()
 
 

src/backend/app/projects/project_routes.py

@@ -6,7 +6,6 @@
 import geojson
 from datetime import timedelta
 from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form
-from sqlalchemy.orm import Session
 from loguru import logger as log
 from app.projects import project_schemas, project_crud
 from app.db import database
@@ -15,7 +14,6 @@
 from app.s3 import s3_client
 from app.config import settings
 from databases import Database
-from app.db import db_models
 from shapely.geometry import shape, mapping
 from shapely.ops import unary_union
 
@@ -26,52 +24,49 @@
 
 
 @router.delete("/{project_id}", tags=["Projects"])
-def delete_project_by_id(
+async def delete_project_by_id(
     project_id: uuid.UUID,
-    db: Session = Depends(database.get_db),
+    db: Database = Depends(database.get_db),
     user: AuthUser = Depends(login_required),
 ):
     """
     Delete a project by its ID, along with all associated tasks.
 
     Args:
         project_id (int): The ID of the project to delete.
-        db (Session): The database session dependency.
+        db (Database): The database session dependency.
 
     Returns:
         dict: A confirmation message.
 
     Raises:
         HTTPException: If the project is not found.
     """
-    # Query for the project
-    project = (
-        db.query(db_models.DbProject)
-        .filter(db_models.DbProject.id == project_id)
-        .first()
-    )
-    if not project:
-        raise HTTPException(status_code=404, detail="Project not found.")
+    delete_query = """
+        WITH deleted_project AS (
+            DELETE FROM projects
+            WHERE id = :project_id
+            RETURNING id
+        ), deleted_tasks AS (
+            DELETE FROM tasks
+            WHERE project_id = :project_id
+            RETURNING project_id
+        )
+        SELECT id FROM deleted_project
+    """
 
-    # Query and delete associated tasks
-    tasks = (
-        db.query(db_models.DbTask)
-        .filter(db_models.DbTask.project_id == project_id)
-        .all()
-    )
-    for task in tasks:
-        db.delete(task)
+    result = await db.fetch_one(query=delete_query, values={"project_id": project_id})
+
+    if not result:
+        raise HTTPException(status_code=404)
 
-    # Delete the project
-    db.delete(project)
-    db.commit()
     return {"message": f"Project ID: {project_id} is deleted successfully."}
 
 
 @router.post("/create_project", tags=["Projects"])
 async def create_project(
     project_info: project_schemas.ProjectIn,
-    db: Database = Depends(database.encode_db),
+    db: Database = Depends(database.get_db),
     user_data: AuthUser = Depends(login_required),
 ):
     """Create a project in  database."""
@@ -90,7 +85,7 @@ async def create_project(
 async def upload_project_task_boundaries(
     project_id: uuid.UUID,
     task_geojson: UploadFile = File(...),
-    db: Database = Depends(database.encode_db),
+    db: Database = Depends(database.get_db),
     user: AuthUser = Depends(login_required),
 ):
     """Set project task boundaries using split GeoJSON from frontend.
@@ -209,7 +204,7 @@ async def generate_presigned_url(
 async def read_projects(
     skip: int = 0,
     limit: int = 100,
-    db: Database = Depends(database.encode_db),
+    db: Database = Depends(database.get_db),
     user_data: AuthUser = Depends(login_required),
 ):
     "Return all projects"
@@ -222,7 +217,7 @@ async def read_projects(
 )
 async def read_project(
     project_id: uuid.UUID,
-    db: Database = Depends(database.encode_db),
+    db: Database = Depends(database.get_db),
     user_data: AuthUser = Depends(login_required),
 ):
     """Get a specific project and all associated tasks by ID."""

src/backend/app/tasks/task_routes.py

@@ -20,9 +20,7 @@
 
 
 @router.get("/states/{project_id}")
-async def task_states(
-    project_id: uuid.UUID, db: Database = Depends(database.encode_db)
-):
+async def task_states(project_id: uuid.UUID, db: Database = Depends(database.get_db)):
     """Get all tasks states for a project."""
 
     return await task_crud.all_tasks_states(db, project_id)
@@ -35,7 +33,7 @@ async def new_event(
     task_id: uuid.UUID,
     detail: task_schemas.NewEvent,
     user_data: AuthUser = Depends(login_required),
-    db: Database = Depends(database.encode_db),
+    db: Database = Depends(database.get_db),
 ):
     user_id = user_data.id
 
@@ -205,7 +203,7 @@ async def new_event(
 async def get_pending_tasks(
     project_id: uuid.UUID,
     user_data: AuthUser = Depends(login_required),
-    db: Database = Depends(database.encode_db),
+    db: Database = Depends(database.get_db),
 ):
     """Get a list of pending tasks for a specific project and user."""
     user_id = user_data.id

src/backend/app/users/oauth_routes.py

@@ -62,7 +62,7 @@ async def update_token(user_data: AuthUser = Depends(login_required)):
 
 @router.get("/my-info/")
 async def my_data(
-    db: Database = Depends(database.encode_db),
+    db: Database = Depends(database.get_db),
     user_data: AuthUser = Depends(login_required),
 ):
     """Read access token and get user details from Google"""