diff --git a/.github/workflows/lint-nginx.yaml b/.github/workflows/lint-nginx.yaml new file mode 100644 index 0000000..781682f --- /dev/null +++ b/.github/workflows/lint-nginx.yaml @@ -0,0 +1,18 @@ +# GitHub Actions workflow for validating NGINX configuration files +# https://github.com/jhinch/nginx-linter +name: Lint NGINX config files +on: + pull_request: + push: + branches: + - main +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install nginx-linter + run: npm install -g nginx-linter + - name: Run nginx linter + run: nginx-linter --include config/nginx/* --no-follow-includes diff --git a/.github/workflows/lint-shell-scripts.yaml b/.github/workflows/lint-shell-scripts.yaml new file mode 100644 index 0000000..94a52c8 --- /dev/null +++ b/.github/workflows/lint-shell-scripts.yaml @@ -0,0 +1,18 @@ +# Lint shell scripts +# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions +# https://github.com/marketplace/actions/shell-linter +name: Lint shell scripts +on: + pull_request: + push: + branches: + - main +jobs: + lint_shell: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: azohra/shell-linter@v0.6.0 + with: + severity: 'warning' + exclude-paths: 'LICENSE' diff --git a/.github/workflows/lint-systemd.yaml b/.github/workflows/lint-systemd.yaml new file mode 100644 index 0000000..1dec7b6 --- /dev/null +++ b/.github/workflows/lint-systemd.yaml @@ -0,0 +1,18 @@ +# GitHub Actions workflow for linting the systemd unit files +# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions +name: Lint systemd units +on: + pull_request: + push: + branches: + - main +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install systemdlint + run: pip install systemdlint==1.* + - name: Lint systemd units + run: systemdlint ./config/systemd/* diff --git a/README.md b/README.md index 571c781..5354ab5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Follow these steps to set up and run the app locally: Prerequisites -- Python 3.10 +- Python 3.12 - pip --- @@ -61,8 +61,9 @@ python manage.py createsuperuser 6. Create a `.env` file in the project root directory and add the following environment variables: -``` +```bash DJANGO_SECRET_KEY=your_secret_key +DJANGO_DEBUG=True ``` --- @@ -81,3 +82,7 @@ The app will be available at http://127.0.0.1:8000. ```bash python manage.py loaddata data\questionnaires.json data\questionnaires.json ``` + +# Deployment + +Please read [`docs/deployment.md`](docs/deployment.md). diff --git a/SORT/settings.py b/SORT/settings.py index 1b099cf..8535885 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -12,15 +12,32 @@ import os from pathlib import Path +from typing import Any from dotenv import load_dotenv -load_dotenv() # Load environment variables from .env file + +def cast_to_boolean(obj: Any) -> bool: + """ + Check if the string value is 1, yes, or true. + + Empty values are interpreted as False. + """ + # Cast to lower case string + obj = str(obj).casefold() + # False / off + if obj in {"", "off", "none"}: + return False + # True / on + return obj[0] in {"1", "y", "t", "o"} + + +# Load environment variables from .env file +load_dotenv(os.getenv("DJANGO_ENV_PATH")) # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - # Path when redirecting to login LOGIN_URL = "/login/" @@ -31,10 +48,9 @@ SECRET_KEY = os.getenv("DJANGO_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] +DEBUG = cast_to_boolean(os.getenv("DJANGO_DEBUG", "False")) +ALLOWED_HOSTS = os.getenv("DJANGO_ALLOWED_HOSTS", "sort-web-app.shef.ac.uk").split() # Application definition @@ -54,7 +70,9 @@ ] MIDDLEWARE = [ - "django.middleware.security.SecurityMiddleware", + # Implement security in the web server, not in Django. + # https://docs.djangoproject.com/en/5.1/ref/middleware/#module-django.middleware.security + # "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -82,21 +100,19 @@ }, ] -WSGI_APPLICATION = "SORT.wsgi.application" - +WSGI_APPLICATION = os.getenv("DJANGO_WSGI_APPLICATION", "SORT.wsgi.application") # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases - DATABASES = { + # Set the database settings using environment variables, or default to a local SQLite database file. "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": BASE_DIR / "db.sqlite3", - # "NAME": os.getenv("DB_NAME"), - # "USER": os.getenv("DB_USER"), - # "PASSWORD": os.getenv("DB_PASSWORD"), - # "HOST": os.getenv("DB_HOST"), - # "PORT": os.getenv("DB_PORT"), + "ENGINE": os.getenv("DJANGO_DATABASE_ENGINE", "django.db.backends.sqlite3"), + "NAME": os.getenv("DJANGO_DATABASE_NAME", BASE_DIR / "db.sqlite3"), + "USER": os.getenv("DJANGO_DATABASE_USER"), + "PASSWORD": os.getenv("DJANGO_DATABASE_PASSWORD"), + "HOST": os.getenv("DJANGO_DATABASE_HOST"), + "PORT": os.getenv("DJANGO_DATABASE_PORT"), } } @@ -118,7 +134,6 @@ }, ] - # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ @@ -131,7 +146,6 @@ USE_TZ = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.1/howto/static-files/ @@ -139,7 +153,6 @@ STATICFILES_DIRS = [BASE_DIR / "static"] - # Default primary key field type # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field @@ -148,7 +161,6 @@ LOGIN_REDIRECT_URL = "/" LOGOUT_REDIRECT_URL = "/" - # FA: End session when the browser is closed SESSION_EXPIRE_AT_BROWSER_CLOSE = True @@ -174,3 +186,11 @@ # FA: for production: # EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" + +STATIC_ROOT = os.getenv("DJANGO_STATIC_ROOT") + +# Security settings +SESSION_COOKIE_SECURE = cast_to_boolean( + os.getenv("DJANGO_SESSION_COOKIE_SECURE", not DEBUG) +) +CSRF_COOKIE_SECURE = cast_to_boolean(os.getenv("DJANGO_CSRF_COOKIE_SECURE", not DEBUG)) diff --git a/SORT/wsgi.py b/SORT/wsgi.py index 66c4f6c..6078f8e 100644 --- a/SORT/wsgi.py +++ b/SORT/wsgi.py @@ -13,4 +13,6 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "SORT.settings") +# Create WSGI application object application = get_wsgi_application() +"https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/#the-application-object" diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..0060b16 --- /dev/null +++ b/config/README.md @@ -0,0 +1,3 @@ +# Server configuration + +This directory contains configuration files for the production web server. diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf new file mode 100644 index 0000000..e95cd5c --- /dev/null +++ b/config/nginx/gunicorn.conf @@ -0,0 +1,61 @@ +# nginx configuration file + +# HTTP and SSL certificates +# https://nginx.org/en/docs/http/configuring_https_servers.html +# https://ssl-config.mozilla.org/#server=nginx&version=1.27.3&config=modern&openssl=3.4.0&ocsp=false&guideline=5.7 + +# Gunicorn proxy configuration +# https://docs.gunicorn.org/en/stable/deploy.html#nginx-configuration + +upstream app_server { + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response + # for UNIX domain socket setups + server unix:/run/gunicorn.sock fail_timeout=0; +} + +server { + # https://nginx.org/en/docs/http/ngx_http_core_module.html + listen 443 ssl; + listen [::]:443 ssl; + # Unavailable on nginx versions before 1.25.1 + # https://nginx.org/en/docs/http/ngx_http_v2_module.html + #http2 on; + # SSL options + ssl_certificate /etc/ssl/certs/sort.crt; + ssl_certificate_key /etc/ssl/private/sort.key; + ssl_protocols TLSv1.3; + ssl_ecdh_curve X25519:prime256v1:secp384r1; + ssl_prefer_server_ciphers off; + client_max_body_size 1m; + server_name sort-web-app.shef.ac.uk; + keepalive_timeout 5; + + # /server-status endpoint + # This is used by IT Services to monitor servers using collectd + # https://nginx.org/en/docs/http/ngx_http_stub_status_module.html + # https://www.collectd.org/documentation/manpages/collectd.conf.html + # It's based on Apache mod_status https://httpd.apache.org/docs/2.4/mod/mod_status.html + location = /server-status { + stub_status; + } + + # Serve static files without invoking Python WSGI + location /static/ { + # https://nginx.org/en/docs/http/ngx_http_core_module.html#root + root /var/www/sort; + } + + # Proxy forward to the WSGI Python app + location / { + # Set HTTP headers for the proxied service + # https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://app_server; + } +} diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service new file mode 100644 index 0000000..7845ea6 --- /dev/null +++ b/config/systemd/gunicorn.service @@ -0,0 +1,32 @@ +# This is a systemd unit that defines the Gunicorn service. +# https://docs.gunicorn.org/en/stable/deploy.html#systemd +# https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html + +[Unit] +Description=gunicorn service +Requires=gunicorn.socket +After=network.target + +[Service] +# gunicorn can let systemd know when it is ready +Type=notify +NotifyAccess=main +# the specific user that our service will run as +User=gunicorn +Group=gunicorn +# this user can be transiently created by systemd +DynamicUser=true +RuntimeDirectory=gunicorn +WorkingDirectory=/opt/sort +ExecStart=/opt/sort/venv/bin/gunicorn SORT.wsgi +ExecReload=/bin/kill -s HUP $MAINPID +KillMode=mixed +TimeoutStopSec=5 +PrivateTmp=true +# if your app does not need administrative capabilities, let systemd know +ProtectSystem=strict +Delegate=yes +LockPersonality=yes + +[Install] +WantedBy=multi-user.target diff --git a/config/systemd/gunicorn.socket b/config/systemd/gunicorn.socket new file mode 100644 index 0000000..3bc872e --- /dev/null +++ b/config/systemd/gunicorn.socket @@ -0,0 +1,17 @@ +# https://docs.gunicorn.org/en/stable/deploy.html#systemd +# https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html +[Unit] +Description=gunicorn socket + +[Socket] +ListenStream=/run/gunicorn.sock +# Our service won't need permissions for the socket, since it +# inherits the file descriptor by socket activation. +# Only the nginx daemon will need access to the socket: +SocketUser=www-data +SocketGroup=www-data +# Once the user/group is correct, restrict the permissions: +SocketMode=0660 + +[Install] +WantedBy=sockets.target diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..f38c5f6 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -e + +# SORT deployment script for Ubuntu 22.04 LTS +# See: How to deploy Django +# https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/#the-application-object + +# Usage: +# Clone the repository +# git clone git@github.com:RSE-Sheffield/SORT.git +# cd SORT +# sudo bash -x deploy.sh + +# Options +sort_dir="/opt/sort" +venv_dir="$sort_dir/venv" +pip="$venv_dir/bin/pip" +python_version="python3.12" +python="$venv_dir/bin/python" + +# Install British UTF-8 locale so we can use this with PostgreSQL. +# This is important to avoid the limitations of the LATIN1 character set. +sudo locale-gen en_GB +sudo locale-gen en_GB.UTF-8 +sudo update-locale + +# Create Python virtual environment +apt update -qq +apt install --upgrade --yes -qq "$python_version" "$python_version-venv" +python3 -m venv "$venv_dir" + +# Install the SORT Django app package +$pip install --quiet -r requirements.txt +cp --recursive * "$sort_dir/" + +# Install static files into DJANGO_STATIC_ROOT +# This runs in a subshell because it's changing directory +(cd "$sort_dir" && exec $python manage.py collectstatic --no-input) + +# Install Gunicorn service +cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service +cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket +systemctl daemon-reload +systemctl enable gunicorn.service +systemctl enable gunicorn.socket +systemctl start gunicorn.service +systemctl reload gunicorn.service + +# Install web reverse proxy server +# Install nginx +# https://nginx.org/en/docs/install.html +apt install --yes -qq nginx +nginx -version + +# Configure web server +rm -f /etc/nginx/sites-enabled/default +cp config/nginx/*.conf /etc/nginx/sites-available +# Enable the site by creating a symbolic link +ln --symbolic --force /etc/nginx/sites-available/gunicorn.conf /etc/nginx/sites-enabled/gunicorn.conf +systemctl reload nginx.service + +# Install PostgreSQL database +# https://ubuntu.com/server/docs/install-and-configure-postgresql +apt install --yes -qq postgresql +# Restart PostgreSQL to enable any new locales +systemctl restart postgresql + +# Run deployment checks +# https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ +(cd "$sort_dir" && exec $python manage.py check --deploy) diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..45b0bf6 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,290 @@ +# Deployment + +The production web server has the following architecture, which is a commonly-used and reasonably secure production setup for Django. + +```mermaid +--- +title: SORT architecture +--- +flowchart LR +Browser -- "HTTPS port 443" --> nginx +subgraph University of Sheffield network + subgraph sort-web-app machine + nginx -- "Unix socket" --> Gunicorn + nginx --> static["Static files"] + Gunicorn -- "WSGI" --> Django + Django --> PostgreSQL + end +end +``` + +When accessing the web site, the following process happens: + +1. A user uses their web browser to access the server using the HTTPS port 443; +2. The request is sent to the web server and is handled by Nginx, which acts as a ["reverse proxy" server](https://serverfault.com/a/331263); +3. Nginx uses the web server gateway interface (WSGI) to access Gunicorn, which serves the Django application; +4. Gunicorn spawns several workers to run the Python code that operates the website; +5. Django uses the PostgreSQL database to store data. + +You may also refer to the following guides: + +* Django documentation: [How to deploy Django](https://docs.djangoproject.com/en/5.1/howto/deployment/) +* [Deploying Gunicorn](https://docs.gunicorn.org/en/latest/deploy.html) using nginx + +The relevant files are: + +* The `config/` directory contains server configuration files. + +# Deployment process + +This app can be deployed to a web server using the script [`deploy.sh`](../deploy.sh) and configured as described in the section below. + +1. Configure the `.env` file as described below. +2. Run the deployment script: `sudo bash -x deploy.sh` +3. Configure the database + +We can run commands and Bash scripts as the superuser (`root`) using the [`sudo` command](https://manpages.ubuntu.com/manpages/noble/en/man8/sudo.8.html). + +# Configuration + +## Environment variables + +To configure the environment variables for the service, you can either edit the `.env` file and/or add them to the systemd service using `systemctl edit`. + +To edit the environment file: + +```bash +sudo mkdir --parents /opt/sort +sudo nano /opt/sort/.env +``` + +This file would typically look similar to the example below. The contents of these values should be stored in a password manager. In general, the name of each environment variable is the same as the [Django setting](https://docs.djangoproject.com/en/5.1/ref/settings) with the prefix `DJANGO_`. + +```ini +DJANGO_SECRET_KEY=******** +DJANGO_ALLOWED_HOSTS=sort-web-app.shef.ac.uk +DJANGO_STATIC_ROOT=/var/www/sort/static +WSGI_APPLICATION=SORT.wsgi.application +DJANGO_SETTINGS_MODULE=SORT.settings +# Database settings +DJANGO_DATABASE_ENGINE=django.db.backends.postgresql +DJANGO_DATABASE_NAME=sort +DJANGO_DATABASE_USER=sort +DJANGO_DATABASE_PASSWORD=******** +DJANGO_DATABASE_HOST=127.0.0.1 +DJANGO_DATABASE_PORT=5432 +``` + +You can generate a secret key using the Python [secrets library](https://docs.python.org/3/library/secrets.html): + +```bash +python -c "import secrets; print(secrets.token_urlsafe(37))" +``` + +## Service settings + +If needed, you can add environment variables to the `systemd` service like so: + +```bash +sudo systemctl edit gunicorn.service +``` + +And add environment variables, or other `systemd` settings, to that override configuration file: + +``` +[Service] +Environment="DJANGO_SECRET_KEY=********" +Environment="DEBUG=off" +``` + + + +# Database installation + +The database may be administered using command-line tools and SQL statements that are run as the `postgres` user. For more details, please refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/16/index.html) and [this guide](https://dev.to/matthewhegarty/postgresql-better-security-for-django-applications-3c7m). + +To run these commands, switch to the `postgres` user: + +```bash +sudo su - postgres +``` + +The [`su` command](https://manpages.ubuntu.com/manpages/noble/man1/su.1.html) creates a new shell on behalf of the `postgres` user. + +## Create a database + +[Create a database](https://www.postgresql.org/docs/16/tutorial-createdb.html) with the appropriate [encoding](https://www.postgresql.org/docs/current/multibyte.html). + +```bash +createdb --template=template0 --encoding=UTF8 --locale=en_GB.UTF-8 sort "SORT application" +``` + +We can list databases using `psql --list`. + +## Create a user + +The SORT app needs credentials to access the database. We'll create a database user that the application will use to read and write data. + +Create a user: + +```bash +createuser sort +``` + +Set the password for that user using the [psql tool](https://manpages.ubuntu.com/manpages/noble/man1/psql.1.html) which lets us run SQL queries on the PostgreSQL server. + +```bash +psql sort +``` + +```sql +ALTER USER sort WITH PASSWORD '********'; +``` + +We can list users (i.e. database "roles") using [the `\du` command](https://www.postgresql.org/docs/current/app-psql.html#APP-PSQL-META-COMMAND-DU) in PostgreSQL. + +```bash +psql sort --command "\du" +``` + + + +## Grant permissions + +We must allow this user the minimum necessary privileges to operate the web app. We authorise the user using the PostgreSQL [grant statement](https://www.postgresql.org/docs/current/sql-grant.html), which we execute in the `psql` tool. + +Create a schema, which is a "folder" in the database (a namespace) that will contain our tables. Remember to initialise the PostgreSQL command line with `psql sort`. + +```sql +CREATE SCHEMA sort AUTHORIZATION sort; +``` + +You can view the [list of schemas](https://www.postgresql.org/docs/current/app-psql.html#APP-PSQL-META-COMMAND-DN) using this PostgreSQL command: + +```sql +\dnS +``` + +Let's restrict the visibility of the schema so the app can only see the `sort` schema. + +```sql +ALTER ROLE sort SET SEARCH_PATH TO sort; +``` + +Let's allow the SORT app to read and write data to the database tables, including any new tables that are created. + +```sql +GRANT CONNECT ON DATABASE sort TO sort; +GRANT USAGE ON SCHEMA sort TO sort; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA sort TO sort; +ALTER DEFAULT PRIVILEGES FOR USER sort GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO sort; +``` + +On our PostgreSQL instance, this should create a database named `sort` with a user named `sort` that has all the necessary permissions on the `sort` schema to create, modify, and drop tables and read/write data to those tables. + +# Security + +## SSL Certificates + +See: ITS Wiki [SSL Certificates/Howto](https://itswiki.shef.ac.uk/wiki/SSL_Certificates/Howto) for the commands to generate a Certificate Signing Request (CSR) using [OpenSSL](https://docs.openssl.org/3.3/man1/openssl-req/#options) with an unencrypted private key. + +We can install the private key + +```bash +sudo mv "$(hostname -s)_shef_ac_uk.key" /etc/ssl/private/sort.key +sudo chmod 640 /etc/ssl/private/sort.key +sudo chown root:ssl-cert /etc/ssl/private/sort.key +``` + +The CSR may be used to get a signed SSL certificate via [Information Security](https://staff.sheffield.ac.uk/it-services/information-security) in IT Services. + +For *development purposes only* we can generate a self-signed certificate (which will cause web browsers to say "Not secure") + +```bash +openssl x509 -signkey /etc/ssl/private/sort.key -in "$(hostname -s)_shef_ac_uk.csr" -req -days 365 -out "$(hostname -s)_shef_ac_uk.crt" +``` + +Install the SSL certificate: + +```bash +sudo cp "$(hostname -s)_shef_ac_uk.crt" /etc/ssl/certs/sort.pem +``` + +# Management + +To use the [Django management tool](https://docs.djangoproject.com/en/5.1/ref/django-admin/), we need to load up the virtual environment of the SORT Django application and navigate to the directory containing the tool. + +```bash +sort_dir="/opt/sort" +venv_dir="$sort_dir/venv" +python="$venv_dir/bin/python" +cd "$sort_dir" +# Check the Django management tool works +$python "$sort_dir"/manage.py version +``` + +View available commands + +```bash +$python "$sort_dir"/manage.py help +``` + +Migrate the database + +```bash +sudo $python manage.py migrate +``` + +Create a super-user + +```bash +sudo $python manage.py createsuperuser +``` + +Load data + +```bash +sudo $python manage.py loaddata data/*.json +``` + +# Monitoring + +## View service status + +```bash +sudo systemctl status gunicorn +sudo systemctl status nginx +sudo systemctl status postgresql +``` + +# View logs + +View [nginx logs](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/) + +```bash +sudo tail --follow /var/log/nginx/access.log +sudo tail --follow /var/log/nginx/error.log +``` + +View [Gunicorn logs](https://docs.gunicorn.org/en/stable/settings.html#logging) + +```bash + sudo journalctl -u gunicorn.service --follow +``` + +# Control + +The services are controlled using [`systemd`](https://systemd.io/), which is the service management system on Ubuntu 24. To launch services: + +```bash +sudo systemctl start gunicorn +sudo systemctl start nginx +``` + +To stop services: + +```bash +sudo systemctl stop gunicorn +sudo systemctl stop nginx +``` + diff --git a/requirements.txt b/requirements.txt index 1390a3b..362cc6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,4 +25,5 @@ typing_extensions==4.12.2 tzdata==2024.1 wcwidth==0.2.13 django_debug_toolbar==4.4.6 +gunicorn==23.* strenum==0.4.15 \ No newline at end of file