From cebb423e881bc7cd9071781543ccad0f514bc486 Mon Sep 17 00:00:00 2001 From: bobby-didcoding Date: Thu, 17 Oct 2024 16:40:45 +0100 Subject: [PATCH 01/18] Bumped Django & sigauth --- constraints.txt | 1 + requirements.in | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 constraints.txt diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 00000000..764a11f5 --- /dev/null +++ b/constraints.txt @@ -0,0 +1 @@ +setuptools<72 diff --git a/requirements.in b/requirements.in index 88667517..b86cd8f9 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ dbt-copilot-python==0.2.0 -django==4.2.15 +django==4.2.16 dj-database-url==2.1.* django-environ==0.4.5 django_extensions==3.2.1 @@ -13,7 +13,7 @@ gunicorn==22.0.0 sentry-sdk>=1.20.0.a,<3.0.0 django-staff-sso-client==4.2.1 requests[security]>=2.32.0 -sigauth==5.2.6 +sigauth==5.3.0 directory-healthcheck==3.7 django-cryptography==1.1 zenpy==2.0.46 diff --git a/requirements.txt b/requirements.txt index eed5e595..75efe5da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,7 +77,7 @@ distlib==0.3.8 # via virtualenv dj-database-url==2.1.0 # via -r requirements.in -django==4.2.15 +django==4.2.16 # via # -r requirements.in # directory-components @@ -278,7 +278,7 @@ requests-oauthlib==1.3.1 # via django-staff-sso-client sentry-sdk==2.8.0 # via -r requirements.in -sigauth==5.2.6 +sigauth==5.3.0 # via -r requirements.in six==1.16.0 # via From 5201187e5171264e82eaead9364c90566731a4d8 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Thu, 31 Oct 2024 14:26:31 +0000 Subject: [PATCH 02/18] Bump waitress to 3.0.1 --- requirements.in | 2 +- requirements.txt | 2 +- requirements_test.txt | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.in b/requirements.in index b86cd8f9..d4cb4a43 100644 --- a/requirements.in +++ b/requirements.in @@ -24,7 +24,7 @@ directory-components==40.2.3 urllib3==2.2.2 django-ratelimit==2.0.0 django-redis==5.4.* -waitress==2.1.2 +waitress==3.0.1 django-celery-beat==2.5.0 pyjwt==2.4.0 certifi==2024.07.04 diff --git a/requirements.txt b/requirements.txt index 75efe5da..22cb7001 100644 --- a/requirements.txt +++ b/requirements.txt @@ -320,7 +320,7 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==2.1.2 +waitress==3.0.1 # via -r requirements.in wcwidth==0.2.6 # via prompt-toolkit diff --git a/requirements_test.txt b/requirements_test.txt index d8dac720..58e8111b 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -85,7 +85,7 @@ distlib==0.3.8 # via virtualenv dj-database-url==2.1.0 # via -r requirements.in -django==4.2.15 +django==4.2.16 # via # -r requirements.in # directory-components @@ -331,7 +331,7 @@ requests-oauthlib==1.3.0 # via django-staff-sso-client sentry-sdk==2.8.0 # via -r requirements.in -sigauth==5.2.6 +sigauth==5.3.0 # via -r requirements.in six==1.14.0 # via @@ -384,7 +384,7 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==2.1.2 +waitress==3.0.1 # via -r requirements.in wcwidth==0.1.9 # via prompt-toolkit From fb731da2d159b00eddcac293eb91e04226b63dcf Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Fri, 1 Nov 2024 09:35:30 +0000 Subject: [PATCH 03/18] Rollback update of waitress --- requirements.in | 2 +- requirements.txt | 2 +- requirements_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index d4cb4a43..b86cd8f9 100644 --- a/requirements.in +++ b/requirements.in @@ -24,7 +24,7 @@ directory-components==40.2.3 urllib3==2.2.2 django-ratelimit==2.0.0 django-redis==5.4.* -waitress==3.0.1 +waitress==2.1.2 django-celery-beat==2.5.0 pyjwt==2.4.0 certifi==2024.07.04 diff --git a/requirements.txt b/requirements.txt index 22cb7001..75efe5da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -320,7 +320,7 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==3.0.1 +waitress==2.1.2 # via -r requirements.in wcwidth==0.2.6 # via prompt-toolkit diff --git a/requirements_test.txt b/requirements_test.txt index 58e8111b..ee3ba632 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -384,7 +384,7 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==3.0.1 +waitress==2.1.2 # via -r requirements.in wcwidth==0.1.9 # via prompt-toolkit From 46430c4e193a2218269b57d33bb1f7ad13098803 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Mon, 4 Nov 2024 12:24:57 +0000 Subject: [PATCH 04/18] Replace waitress with gunicorn --- Procfile | 2 +- conf/gunicorn.py | 6 ++++++ requirements.in | 2 +- requirements.txt | 4 ++-- requirements_test.txt | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 conf/gunicorn.py diff --git a/Procfile b/Procfile index 367c9e71..73667fe9 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && waitress-serve --port=$PORT conf.wsgi:application +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-class gevent --worker-connections 1000 --log-file - celery_worker: celery -A conf worker -l info celery_beat: celery -A conf beat -l info -S django \ No newline at end of file diff --git a/conf/gunicorn.py b/conf/gunicorn.py new file mode 100644 index 00000000..5ecb8df6 --- /dev/null +++ b/conf/gunicorn.py @@ -0,0 +1,6 @@ +from psycogreen.gevent import patch_psycopg + + +def post_fork(server, worker): + patch_psycopg() + worker.log.info("Enabled async Psycopg2") \ No newline at end of file diff --git a/requirements.in b/requirements.in index b86cd8f9..a2662ce5 100644 --- a/requirements.in +++ b/requirements.in @@ -24,7 +24,6 @@ directory-components==40.2.3 urllib3==2.2.2 django-ratelimit==2.0.0 django-redis==5.4.* -waitress==2.1.2 django-celery-beat==2.5.0 pyjwt==2.4.0 certifi==2024.07.04 @@ -33,4 +32,5 @@ drf-spectacular>=0.26.0 elastic-apm==6.1.* django-log-formatter-asim==0.0.4 pydantic==2.7.3 +psycogreen==1.0.2 pydantic-settings==2.3.1 diff --git a/requirements.txt b/requirements.txt index 75efe5da..0a165d56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -227,6 +227,8 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto +psycogreen==1.0.2 + # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in pycparser==2.21 @@ -320,8 +322,6 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==2.1.2 - # via -r requirements.in wcwidth==0.2.6 # via prompt-toolkit whitenoise==6.4.0 diff --git a/requirements_test.txt b/requirements_test.txt index ee3ba632..a18341fa 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -254,6 +254,8 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto +psycogreen==1.0.2 + # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in py==1.10.0 @@ -384,8 +386,6 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==2.1.2 - # via -r requirements.in wcwidth==0.1.9 # via prompt-toolkit whitenoise==6.4.0 From 7264a046288af76d67143cd2f9f630d9cd11682f Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Mon, 4 Nov 2024 12:28:53 +0000 Subject: [PATCH 05/18] revert code --- Procfile | 4 ++-- conf/gunicorn.py | 6 ------ requirements.in | 2 +- requirements.txt | 4 ++-- requirements_test.txt | 4 ++-- 5 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 conf/gunicorn.py diff --git a/Procfile b/Procfile index 73667fe9..ea468a53 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-class gevent --worker-connections 1000 --log-file - +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && waitress-serve --port=$PORT conf.wsgi:application celery_worker: celery -A conf worker -l info -celery_beat: celery -A conf beat -l info -S django \ No newline at end of file +celery_beat: celery -A conf beat -l info -S django diff --git a/conf/gunicorn.py b/conf/gunicorn.py deleted file mode 100644 index 5ecb8df6..00000000 --- a/conf/gunicorn.py +++ /dev/null @@ -1,6 +0,0 @@ -from psycogreen.gevent import patch_psycopg - - -def post_fork(server, worker): - patch_psycopg() - worker.log.info("Enabled async Psycopg2") \ No newline at end of file diff --git a/requirements.in b/requirements.in index a2662ce5..3a814457 100644 --- a/requirements.in +++ b/requirements.in @@ -32,5 +32,5 @@ drf-spectacular>=0.26.0 elastic-apm==6.1.* django-log-formatter-asim==0.0.4 pydantic==2.7.3 -psycogreen==1.0.2 +waitress==2.1.2 pydantic-settings==2.3.1 diff --git a/requirements.txt b/requirements.txt index 0a165d56..75efe5da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -227,8 +227,6 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto -psycogreen==1.0.2 - # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in pycparser==2.21 @@ -322,6 +320,8 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit +waitress==2.1.2 + # via -r requirements.in wcwidth==0.2.6 # via prompt-toolkit whitenoise==6.4.0 diff --git a/requirements_test.txt b/requirements_test.txt index a18341fa..ee3ba632 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -254,8 +254,6 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto -psycogreen==1.0.2 - # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in py==1.10.0 @@ -386,6 +384,8 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit +waitress==2.1.2 + # via -r requirements.in wcwidth==0.1.9 # via prompt-toolkit whitenoise==6.4.0 From ab821b84059d20b7cfcb4fb661a0c02e1f768db8 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Mon, 4 Nov 2024 12:40:49 +0000 Subject: [PATCH 06/18] Replace waitress with gunicorn --- Procfile | 2 +- gunicorn.py | 6 ++++++ requirements.in | 2 +- requirements.txt | 4 ++-- requirements_test.txt | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 gunicorn.py diff --git a/Procfile b/Procfile index ea468a53..c2351e80 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && waitress-serve --port=$PORT conf.wsgi:application +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-class gevent --worker-connections 1000 --log-file - celery_worker: celery -A conf worker -l info celery_beat: celery -A conf beat -l info -S django diff --git a/gunicorn.py b/gunicorn.py new file mode 100644 index 00000000..d3af0915 --- /dev/null +++ b/gunicorn.py @@ -0,0 +1,6 @@ +from psycogreen.gevent import patch_psycopg + + +def post_fork(server, worker): + patch_psycopg() + worker.log.info("Enabled async Psycopg2") diff --git a/requirements.in b/requirements.in index 3a814457..d9a9fd39 100644 --- a/requirements.in +++ b/requirements.in @@ -32,5 +32,5 @@ drf-spectacular>=0.26.0 elastic-apm==6.1.* django-log-formatter-asim==0.0.4 pydantic==2.7.3 -waitress==2.1.2 pydantic-settings==2.3.1 +psycogreen==1.0.2 diff --git a/requirements.txt b/requirements.txt index 75efe5da..0a165d56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -227,6 +227,8 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto +psycogreen==1.0.2 + # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in pycparser==2.21 @@ -320,8 +322,6 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==2.1.2 - # via -r requirements.in wcwidth==0.2.6 # via prompt-toolkit whitenoise==6.4.0 diff --git a/requirements_test.txt b/requirements_test.txt index ee3ba632..a18341fa 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -254,6 +254,8 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto +psycogreen==1.0.2 + # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in py==1.10.0 @@ -384,8 +386,6 @@ vine==5.1.0 # kombu virtualenv==20.26.1 # via pre-commit -waitress==2.1.2 - # via -r requirements.in wcwidth==0.1.9 # via prompt-toolkit whitenoise==6.4.0 From a045d9a1d9251a59e5a31f3881fa6b91bfbeed0a Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Mon, 4 Nov 2024 21:15:38 +0000 Subject: [PATCH 07/18] Replace waitress with gunicorn --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index c2351e80..b371a73c 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-class gevent --worker-connections 1000 --log-file - +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi --bind 0.0.0.0:$PORT celery_worker: celery -A conf worker -l info celery_beat: celery -A conf beat -l info -S django From 863fcd1372e976a4214c52a9950f66133a6638c4 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Mon, 4 Nov 2024 21:40:30 +0000 Subject: [PATCH 08/18] wip --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index b371a73c..3afaa153 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi --bind 0.0.0.0:$PORT +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --bind 0.0.0.0:$PORT celery_worker: celery -A conf worker -l info celery_beat: celery -A conf beat -l info -S django From 5a23734211ffdfafbb56dd5b97969fe7b5c8634f Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Tue, 5 Nov 2024 15:10:46 +0000 Subject: [PATCH 09/18] remove gunicorn config --- gunicorn.py | 6 ------ requirements.in | 1 - requirements.txt | 2 -- requirements_test.txt | 2 -- 4 files changed, 11 deletions(-) delete mode 100644 gunicorn.py diff --git a/gunicorn.py b/gunicorn.py deleted file mode 100644 index d3af0915..00000000 --- a/gunicorn.py +++ /dev/null @@ -1,6 +0,0 @@ -from psycogreen.gevent import patch_psycopg - - -def post_fork(server, worker): - patch_psycopg() - worker.log.info("Enabled async Psycopg2") diff --git a/requirements.in b/requirements.in index d9a9fd39..adaccddc 100644 --- a/requirements.in +++ b/requirements.in @@ -33,4 +33,3 @@ elastic-apm==6.1.* django-log-formatter-asim==0.0.4 pydantic==2.7.3 pydantic-settings==2.3.1 -psycogreen==1.0.2 diff --git a/requirements.txt b/requirements.txt index 0a165d56..75ad925c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -227,8 +227,6 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto -psycogreen==1.0.2 - # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in pycparser==2.21 diff --git a/requirements_test.txt b/requirements_test.txt index a18341fa..fefea36d 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -254,8 +254,6 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto -psycogreen==1.0.2 - # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in py==1.10.0 From a9c03e43b4b17749edf0fa8fe063bae1cad051be Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 09:50:40 +0000 Subject: [PATCH 10/18] use geevt worker class for concurrent transactions --- Procfile | 2 +- conf/gunicorn.py | 6 ++++++ requirements.in | 1 + requirements.txt | 2 ++ requirements_test.txt | 2 ++ 5 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 conf/gunicorn.py diff --git a/Procfile b/Procfile index 3afaa153..bb88a21e 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --bind 0.0.0.0:$PORT +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application ---config conf/gunicorn.py -bind 0.0.0.0:$PORT --worker-class gevent celery_worker: celery -A conf worker -l info celery_beat: celery -A conf beat -l info -S django diff --git a/conf/gunicorn.py b/conf/gunicorn.py new file mode 100644 index 00000000..d3af0915 --- /dev/null +++ b/conf/gunicorn.py @@ -0,0 +1,6 @@ +from psycogreen.gevent import patch_psycopg + + +def post_fork(server, worker): + patch_psycopg() + worker.log.info("Enabled async Psycopg2") diff --git a/requirements.in b/requirements.in index adaccddc..d9a9fd39 100644 --- a/requirements.in +++ b/requirements.in @@ -33,3 +33,4 @@ elastic-apm==6.1.* django-log-formatter-asim==0.0.4 pydantic==2.7.3 pydantic-settings==2.3.1 +psycogreen==1.0.2 diff --git a/requirements.txt b/requirements.txt index 75ad925c..0a165d56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -227,6 +227,8 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto +psycogreen==1.0.2 + # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in pycparser==2.21 diff --git a/requirements_test.txt b/requirements_test.txt index fefea36d..a18341fa 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -254,6 +254,8 @@ protobuf==4.25.3 # via # googleapis-common-protos # opentelemetry-proto +psycogreen==1.0.2 + # via -r requirements.in psycopg2==2.8.4 # via -r requirements.in py==1.10.0 From fbcc86d6671244d9cdac6a1ca26cd315896f6f41 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 10:09:02 +0000 Subject: [PATCH 11/18] typo --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index bb88a21e..d5563515 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application ---config conf/gunicorn.py -bind 0.0.0.0:$PORT --worker-class gevent +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-class gevent celery_worker: celery -A conf worker -l info celery_beat: celery -A conf beat -l info -S django From f85eafa18c8d44608e02efce5314b65551a6c628 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 10:26:01 +0000 Subject: [PATCH 12/18] add gevent==23.9.1 --- requirements.in | 1 + requirements.txt | 8 ++++++++ requirements_test.txt | 8 ++++++++ 3 files changed, 17 insertions(+) diff --git a/requirements.in b/requirements.in index d9a9fd39..f5f39ea4 100644 --- a/requirements.in +++ b/requirements.in @@ -34,3 +34,4 @@ django-log-formatter-asim==0.0.4 pydantic==2.7.3 pydantic-settings==2.3.1 psycogreen==1.0.2 +gevent==23.9.1 diff --git a/requirements.txt b/requirements.txt index 0a165d56..fc693c4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -137,10 +137,14 @@ elastic-apm==6.1.3 # via -r requirements.in filelock==3.14.0 # via virtualenv +gevent==23.9.1 + # via -r requirements.in googleapis-common-protos==1.63.1 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http +greenlet==3.1.1 + # via gevent grpcio==1.64.1 # via opentelemetry-exporter-otlp-proto-grpc gunicorn==22.0.0 @@ -334,6 +338,10 @@ zenpy==2.0.46 # via -r requirements.in zipp==3.19.1 # via importlib-metadata +zope-event==5.0 + # via gevent +zope-interface==7.1.1 + # via gevent # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements_test.txt b/requirements_test.txt index a18341fa..9135e45b 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -155,10 +155,14 @@ flake8==3.7.9 # via -r requirements_test.in freezegun==0.3.12 # via -r requirements_test.in +gevent==23.9.1 + # via -r requirements.in googleapis-common-protos==1.63.1 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http +greenlet==3.1.1 + # via gevent grpcio==1.64.1 # via opentelemetry-exporter-otlp-proto-grpc gunicorn==22.0.0 @@ -398,6 +402,10 @@ zenpy==2.0.46 # via -r requirements.in zipp==3.19.1 # via importlib-metadata +zope-event==5.0 + # via gevent +zope-interface==7.1.1 + # via gevent # The following packages are considered to be unsafe in a requirements file: # setuptools From f03aca6cf0eeb263e266053c97b87c9c4ca55ec7 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 14:00:09 +0000 Subject: [PATCH 13/18] added test_fork for gunicorn config --- Procfile | 2 +- conf/gunicorn.py | 6 +++- conf/tests/test_fork.py | 63 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 conf/tests/test_fork.py diff --git a/Procfile b/Procfile index d5563515..6b30f1d0 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-class gevent +web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn conf.wsgi:application --config conf/gunicorn.py --bind 0.0.0.0:$PORT --worker-connections 1000 celery_worker: celery -A conf worker -l info celery_beat: celery -A conf beat -l info -S django diff --git a/conf/gunicorn.py b/conf/gunicorn.py index d3af0915..c6c26355 100644 --- a/conf/gunicorn.py +++ b/conf/gunicorn.py @@ -2,5 +2,9 @@ def post_fork(server, worker): - patch_psycopg() + patch_with_psycogreen_gevent() worker.log.info("Enabled async Psycopg2") + +def patch_with_psycogreen_gevent(): + patch_psycopg() + diff --git a/conf/tests/test_fork.py b/conf/tests/test_fork.py new file mode 100644 index 00000000..bc849ef4 --- /dev/null +++ b/conf/tests/test_fork.py @@ -0,0 +1,63 @@ +import pytest +import logging +from urllib.parse import urlencode +from unittest import mock +from unittest.mock import patch +from gevent.server import StreamServer +from conf.gunicorn import post_fork +from gunicorn.workers.ggevent import GeventWorker +from gunicorn.workers.workertmp import WorkerTmp + +logger = logging.getLogger(__name__) + +class dotdict(dict): + """dot.notation access to dictionary attributes""" + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + + + +@pytest.fixture +def worker(): + my_dict = { + 'max_requests': 1000, + 'max_requests_jitter': 10, + 'umask': 22, + 'worker_tmp_dir': '/tmp', + 'uid': 503, + 'gid': 20, + + } + cfg_dict = dotdict(my_dict) + + worker = GeventWorker( + log=logger, + age=1, + ppid=9999, + sockets=[], + app='directory_forms_api', + timeout=30, + cfg=cfg_dict, + ) + return worker + + +@pytest.fixture +def server(): + def handler(): + return None + + server = StreamServer( + listener=( + '0.0.0.0:8020' + ), + handle=handler, + ) + return server + +@patch('conf.gunicorn.patch_with_psycogreen_gevent') +def test_post_fork(patch_with_psycogreen_gevent, worker, server): + post_fork(server, worker) + breakpoint() + patch_with_psycogreen_gevent.assert_called_once() From 955d208853fad8f5eb9a5387a6b24fe4987a064d Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 14:17:17 +0000 Subject: [PATCH 14/18] mock chown --- conf/tests/test_fork.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/conf/tests/test_fork.py b/conf/tests/test_fork.py index bc849ef4..d2b21efd 100644 --- a/conf/tests/test_fork.py +++ b/conf/tests/test_fork.py @@ -1,12 +1,10 @@ import pytest import logging -from urllib.parse import urlencode -from unittest import mock from unittest.mock import patch from gevent.server import StreamServer from conf.gunicorn import post_fork from gunicorn.workers.ggevent import GeventWorker -from gunicorn.workers.workertmp import WorkerTmp +import gunicorn.util logger = logging.getLogger(__name__) @@ -57,7 +55,7 @@ def handler(): return server @patch('conf.gunicorn.patch_with_psycogreen_gevent') -def test_post_fork(patch_with_psycogreen_gevent, worker, server): +@patch.object(gunicorn.util, 'chown') +def test_post_fork(mock_chown, mock_patch_with_psycogreen_gevent, worker, server): post_fork(server, worker) - breakpoint() - patch_with_psycogreen_gevent.assert_called_once() + mock_patch_with_psycogreen_gevent.assert_called_once() From 809ca76bfd18b93ea7164b2de0cac2d48a2230ab Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 14:19:23 +0000 Subject: [PATCH 15/18] flake8 --- conf/gunicorn.py | 2 +- conf/tests/test_fork.py | 33 ++++++++++++++++----------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/conf/gunicorn.py b/conf/gunicorn.py index c6c26355..af83c24b 100644 --- a/conf/gunicorn.py +++ b/conf/gunicorn.py @@ -5,6 +5,6 @@ def post_fork(server, worker): patch_with_psycogreen_gevent() worker.log.info("Enabled async Psycopg2") + def patch_with_psycogreen_gevent(): patch_psycopg() - diff --git a/conf/tests/test_fork.py b/conf/tests/test_fork.py index d2b21efd..8c351696 100644 --- a/conf/tests/test_fork.py +++ b/conf/tests/test_fork.py @@ -8,33 +8,33 @@ logger = logging.getLogger(__name__) + class dotdict(dict): """dot.notation access to dictionary attributes""" + __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ - @pytest.fixture def worker(): my_dict = { - 'max_requests': 1000, - 'max_requests_jitter': 10, - 'umask': 22, - 'worker_tmp_dir': '/tmp', - 'uid': 503, - 'gid': 20, - + "max_requests": 1000, + "max_requests_jitter": 10, + "umask": 22, + "worker_tmp_dir": "/tmp", + "uid": 503, + "gid": 20, } cfg_dict = dotdict(my_dict) - + worker = GeventWorker( log=logger, age=1, ppid=9999, sockets=[], - app='directory_forms_api', + app="directory_forms_api", timeout=30, cfg=cfg_dict, ) @@ -45,17 +45,16 @@ def worker(): def server(): def handler(): return None - + server = StreamServer( - listener=( - '0.0.0.0:8020' - ), + listener=("0.0.0.0:8020"), handle=handler, - ) + ) return server -@patch('conf.gunicorn.patch_with_psycogreen_gevent') -@patch.object(gunicorn.util, 'chown') + +@patch("conf.gunicorn.patch_with_psycogreen_gevent") +@patch.object(gunicorn.util, "chown") def test_post_fork(mock_chown, mock_patch_with_psycogreen_gevent, worker, server): post_fork(server, worker) mock_patch_with_psycogreen_gevent.assert_called_once() From ea48ef75641fe69ecc60f1aee0c7a1e374a13460 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 14:48:12 +0000 Subject: [PATCH 16/18] wip --- conf/tests/test_fork.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/conf/tests/test_fork.py b/conf/tests/test_fork.py index 8c351696..98ac6b8b 100644 --- a/conf/tests/test_fork.py +++ b/conf/tests/test_fork.py @@ -1,10 +1,11 @@ import pytest import logging +from unittest import mock from unittest.mock import patch from gevent.server import StreamServer from conf.gunicorn import post_fork from gunicorn.workers.ggevent import GeventWorker -import gunicorn.util +from gunicorn.workers.workertmp import WorkerTmp logger = logging.getLogger(__name__) @@ -24,8 +25,9 @@ def worker(): "max_requests_jitter": 10, "umask": 22, "worker_tmp_dir": "/tmp", - "uid": 503, - "gid": 20, + "uid": 999, + "gid": 999, + 'worker_connections': 1000, } cfg_dict = dotdict(my_dict) @@ -53,8 +55,8 @@ def handler(): return server -@patch("conf.gunicorn.patch_with_psycogreen_gevent") -@patch.object(gunicorn.util, "chown") -def test_post_fork(mock_chown, mock_patch_with_psycogreen_gevent, worker, server): - post_fork(server, worker) - mock_patch_with_psycogreen_gevent.assert_called_once() +@patch('conf.gunicorn.patch_with_psycogreen_gevent') +def test_post_fork(mock_patch_with_psycogreen_gevent, worker, server): + with patch.object(WorkerTmp, "__init__", lambda x: worker.cfg): + post_fork(server, worker) + mock_patch_with_psycogreen_gevent.assert_called_once() From 6a00858461ab791a19d0479a59fd318d9936a4c6 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 15:04:09 +0000 Subject: [PATCH 17/18] fix test --- conf/tests/test_fork.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/tests/test_fork.py b/conf/tests/test_fork.py index 98ac6b8b..73684449 100644 --- a/conf/tests/test_fork.py +++ b/conf/tests/test_fork.py @@ -1,4 +1,5 @@ import pytest +import os import logging from unittest import mock from unittest.mock import patch @@ -25,8 +26,8 @@ def worker(): "max_requests_jitter": 10, "umask": 22, "worker_tmp_dir": "/tmp", - "uid": 999, - "gid": 999, + "uid": os.geteuid(), + "gid": os.getegid(), 'worker_connections': 1000, } cfg_dict = dotdict(my_dict) @@ -57,6 +58,5 @@ def handler(): @patch('conf.gunicorn.patch_with_psycogreen_gevent') def test_post_fork(mock_patch_with_psycogreen_gevent, worker, server): - with patch.object(WorkerTmp, "__init__", lambda x: worker.cfg): - post_fork(server, worker) - mock_patch_with_psycogreen_gevent.assert_called_once() + post_fork(server, worker) + mock_patch_with_psycogreen_gevent.assert_called_once() From 53bb0d95feb6f24c34d53932bedd750d15fc0cb5 Mon Sep 17 00:00:00 2001 From: davidu1975 Date: Wed, 6 Nov 2024 15:06:56 +0000 Subject: [PATCH 18/18] fix test --- conf/tests/test_fork.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/conf/tests/test_fork.py b/conf/tests/test_fork.py index 73684449..032e4913 100644 --- a/conf/tests/test_fork.py +++ b/conf/tests/test_fork.py @@ -1,12 +1,10 @@ import pytest import os import logging -from unittest import mock from unittest.mock import patch from gevent.server import StreamServer from conf.gunicorn import post_fork from gunicorn.workers.ggevent import GeventWorker -from gunicorn.workers.workertmp import WorkerTmp logger = logging.getLogger(__name__) @@ -28,7 +26,7 @@ def worker(): "worker_tmp_dir": "/tmp", "uid": os.geteuid(), "gid": os.getegid(), - 'worker_connections': 1000, + "worker_connections": 1000, } cfg_dict = dotdict(my_dict) @@ -56,7 +54,7 @@ def handler(): return server -@patch('conf.gunicorn.patch_with_psycogreen_gevent') +@patch("conf.gunicorn.patch_with_psycogreen_gevent") def test_post_fork(mock_patch_with_psycogreen_gevent, worker, server): post_fork(server, worker) mock_patch_with_psycogreen_gevent.assert_called_once()