Merge pull request #342 from uktrade/develop

Merge outstanding PRs for DBT platform

Procfile

@@ -1,3 +1,3 @@
-web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && waitress-serve --port=$PORT conf.wsgi:application
+web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-connections 1000
 celery_worker: celery -A conf worker -l info
-celery_beat: celery -A conf beat -l info -S django
+celery_beat: celery -A conf beat -l info -S django

conf/gunicorn.py

@@ -0,0 +1,10 @@
+from psycogreen.gevent import patch_psycopg
+
+
+def post_fork(server, worker):
+    patch_with_psycogreen_gevent()
+    worker.log.info("Enabled async Psycopg2")
+
+
+def patch_with_psycogreen_gevent():
+    patch_psycopg()

conf/tests/test_fork.py

@@ -0,0 +1,60 @@
+import pytest
+import os
+import logging
+from unittest.mock import patch
+from gevent.server import StreamServer
+from conf.gunicorn import post_fork
+from gunicorn.workers.ggevent import GeventWorker
+
+logger = logging.getLogger(__name__)
+
+
+class dotdict(dict):
+    """dot.notation access to dictionary attributes"""
+
+    __getattr__ = dict.get
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+
+@pytest.fixture
+def worker():
+    my_dict = {
+        "max_requests": 1000,
+        "max_requests_jitter": 10,
+        "umask": 22,
+        "worker_tmp_dir": "/tmp",
+        "uid": os.geteuid(),
+        "gid": os.getegid(),
+        "worker_connections": 1000,
+    }
+    cfg_dict = dotdict(my_dict)
+
+    worker = GeventWorker(
+        log=logger,
+        age=1,
+        ppid=9999,
+        sockets=[],
+        app="directory_forms_api",
+        timeout=30,
+        cfg=cfg_dict,
+    )
+    return worker
+
+
+@pytest.fixture
+def server():
+    def handler():
+        return None
+
+    server = StreamServer(
+        listener=("0.0.0.0:8020"),
+        handle=handler,
+    )
+    return server
+
+
+@patch("conf.gunicorn.patch_with_psycogreen_gevent")
+def test_post_fork(mock_patch_with_psycogreen_gevent, worker, server):
+    post_fork(server, worker)
+    mock_patch_with_psycogreen_gevent.assert_called_once()

constraints.txt

@@ -0,0 +1 @@
+setuptools<72

requirements.in

@@ -1,5 +1,5 @@
 dbt-copilot-python==0.2.0
-django==4.2.15
+django==4.2.16
 dj-database-url==2.1.*
 django-environ==0.4.5
 django_extensions==3.2.1
@@ -13,7 +13,7 @@ gunicorn==22.0.0
 sentry-sdk>=1.20.0.a,<3.0.0
 django-staff-sso-client==4.2.1
 requests[security]>=2.32.0
-sigauth==5.2.6
+sigauth==5.3.0
 directory-healthcheck==3.7
 django-cryptography==1.1
 zenpy==2.0.46
@@ -24,7 +24,6 @@ directory-components==40.2.3
 urllib3==2.2.2
 django-ratelimit==2.0.0
 django-redis==5.4.*
-waitress==2.1.2
 django-celery-beat==2.5.0
 pyjwt==2.4.0
 certifi==2024.07.04
@@ -34,3 +33,5 @@ elastic-apm==6.1.*
 django-log-formatter-asim==0.0.4
 pydantic==2.7.3
 pydantic-settings==2.3.1
+psycogreen==1.0.2
+gevent==23.9.1

requirements.txt

@@ -77,7 +77,7 @@ distlib==0.3.8
     # via virtualenv
 dj-database-url==2.1.0
     # via -r requirements.in
-django==4.2.15
+django==4.2.16
     # via
     #   -r requirements.in
     #   directory-components
@@ -137,10 +137,14 @@ elastic-apm==6.1.3
     # via -r requirements.in
 filelock==3.14.0
     # via virtualenv
+gevent==23.9.1
+    # via -r requirements.in
 googleapis-common-protos==1.63.1
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
+greenlet==3.1.1
+    # via gevent
 grpcio==1.64.1
     # via opentelemetry-exporter-otlp-proto-grpc
 gunicorn==22.0.0
@@ -227,6 +231,8 @@ protobuf==4.25.3
     # via
     #   googleapis-common-protos
     #   opentelemetry-proto
+psycogreen==1.0.2
+    # via -r requirements.in
 psycopg2==2.8.4
     # via -r requirements.in
 pycparser==2.21
@@ -278,7 +284,7 @@ requests-oauthlib==1.3.1
     # via django-staff-sso-client
 sentry-sdk==2.8.0
     # via -r requirements.in
-sigauth==5.2.6
+sigauth==5.3.0
     # via -r requirements.in
 six==1.16.0
     # via
@@ -320,8 +326,6 @@ vine==5.1.0
     #   kombu
 virtualenv==20.26.1
     # via pre-commit
-waitress==2.1.2
-    # via -r requirements.in
 wcwidth==0.2.6
     # via prompt-toolkit
 whitenoise==6.4.0
@@ -334,6 +338,10 @@ zenpy==2.0.46
     # via -r requirements.in
 zipp==3.19.1
     # via importlib-metadata
+zope-event==5.0
+    # via gevent
+zope-interface==7.1.1
+    # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements_test.txt

@@ -85,7 +85,7 @@ distlib==0.3.8
     # via virtualenv
 dj-database-url==2.1.0
     # via -r requirements.in
-django==4.2.15
+django==4.2.16
     # via
     #   -r requirements.in
     #   directory-components
@@ -155,10 +155,14 @@ flake8==3.7.9
     # via -r requirements_test.in
 freezegun==0.3.12
     # via -r requirements_test.in
+gevent==23.9.1
+    # via -r requirements.in
 googleapis-common-protos==1.63.1
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
+greenlet==3.1.1
+    # via gevent
 grpcio==1.64.1
     # via opentelemetry-exporter-otlp-proto-grpc
 gunicorn==22.0.0
@@ -254,6 +258,8 @@ protobuf==4.25.3
     # via
     #   googleapis-common-protos
     #   opentelemetry-proto
+psycogreen==1.0.2
+    # via -r requirements.in
 psycopg2==2.8.4
     # via -r requirements.in
 py==1.10.0
@@ -331,7 +337,7 @@ requests-oauthlib==1.3.0
     # via django-staff-sso-client
 sentry-sdk==2.8.0
     # via -r requirements.in
-sigauth==5.2.6
+sigauth==5.3.0
     # via -r requirements.in
 six==1.14.0
     # via
@@ -384,8 +390,6 @@ vine==5.1.0
     #   kombu
 virtualenv==20.26.1
     # via pre-commit
-waitress==2.1.2
-    # via -r requirements.in
 wcwidth==0.1.9
     # via prompt-toolkit
 whitenoise==6.4.0
@@ -398,6 +402,10 @@ zenpy==2.0.46
     # via -r requirements.in
 zipp==3.19.1
     # via importlib-metadata
+zope-event==5.0
+    # via gevent
+zope-interface==7.1.1
+    # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools