From 96d1b10ee57de9af2a2e5b2597fe0203ba2ede72 Mon Sep 17 00:00:00 2001 From: Jeff Wong Date: Fri, 27 Oct 2023 12:16:52 -0700 Subject: [PATCH] Add tags to pups templates The purpose here is to allow greater flexibility in how and where docker images are built and run. It achieves this by breaking up build steps into distinct run steps which can be saved along the way. Customizable base images may then be prebuilt with as many batteries included as possible, with zero environment setup so those images can then be configured at a later stage. Add the ability to run partial pups configuration: `build`: build base image with no db - ember build. `precompile`: precompile stage that requires postgres and redis. `migrate`: run migration tasks. `db`: start bundled postgres/redis, if included. Adds a create_db script in postgres template for creating db on the fly. Called below in unicorn run: updates unicorn run command with 3 env flags: CREATE_DB_ON_BOOT: if 1, creates base db schema, allows for deferral of creation. MIGRATE_ON_BOOT: if 1, runs db:migrate - allows for deferral of db migration. PRECOMPILE_ON_BOOT: if 1, precompiles assets (without ember build). PRECOMPILE_ON_BOOT initially defaults to 1 in base builds (no tags). During the `precompile` build step, this updates the default to be 0. All other new flags default to 0 (off). With these three flags, we're now able to ship and start a container from a base image, and it'll be able to bootstrap a blank database. Updates hook to start redis before_db_migrate as before_code hook is not guaranteed to fire before migrate tasks if pups is filtered by tags. --- templates/postgres.template.yml | 78 ++++++++++++++++++++++----------- templates/redis.template.yml | 9 ++-- templates/web.template.yml | 30 +++++++++++-- 3 files changed, 84 insertions(+), 33 deletions(-) diff --git a/templates/postgres.template.yml b/templates/postgres.template.yml index bb2c0a232..fd920af03 100644 --- a/templates/postgres.template.yml +++ b/templates/postgres.template.yml @@ -17,17 +17,20 @@ hooks: to: sv start postgres || exit 1 run: - - exec: locale-gen $LANG && update-locale - - exec: mkdir -p /shared/postgres_run - - exec: chown postgres:postgres /shared/postgres_run - - exec: chmod 775 /shared/postgres_run - - exec: rm -fr /var/run/postgresql - - exec: ln -s /shared/postgres_run /var/run/postgresql - - exec: socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1 - - exec: rm -fr /shared/postgres_run/.s* - - exec: rm -fr /shared/postgres_run/*.pid - - exec: mkdir -p /shared/postgres_run/13-main.pg_stat_tmp - - exec: chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp + - exec: + cmd: + - locale-gen $LANG && update-locale + - mkdir -p /shared/postgres_run + - chown postgres:postgres /shared/postgres_run + - chmod 775 /shared/postgres_run + - rm -fr /var/run/postgresql + - ln -s /shared/postgres_run /var/run/postgresql + - socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1 + - rm -fr /shared/postgres_run/.s* + - rm -fr /shared/postgres_run/*.pid + - mkdir -p /shared/postgres_run/13-main.pg_stat_tmp + - chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp + tag: db - file: path: /etc/service/postgres/run chmod: "+x" @@ -117,14 +120,14 @@ run: fi - exec: + tag: db cmd: - chown -R root /var/lib/postgresql/13/main - "[ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data || exit 0" - chown -R postgres:postgres /shared/postgres_data - chown -R postgres:postgres /var/run/postgresql - - - exec: /root/upgrade_postgres - - exec: rm /root/upgrade_postgres + - /root/upgrade_postgres + - rm /root/upgrade_postgres - replace: filename: "/etc/postgresql/13/main/postgresql.conf" @@ -165,6 +168,7 @@ run: - exec: cmd: - install -d -m 0755 -o postgres -g postgres /shared/postgres_backup + tag: db - replace: filename: "/etc/postgresql/13/main/postgresql.conf" @@ -199,30 +203,51 @@ run: to: "host all all ::/0 md5" - exec: + tag: db background: true # use fast shutdown for pg stop_signal: INT cmd: HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/13/bin/postmaster -D /etc/postgresql/13/main # give db a few secs to start up - - exec: "sleep 5" - - - exec: su postgres -c 'createdb $db_name' || true - - exec: su postgres -c 'psql $db_name -c "create user $db_user;"' || true - - exec: su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true - - exec: su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"' - - exec: su postgres -c 'psql template1 -c "create extension if not exists hstore;"' - - exec: su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"' - - exec: su postgres -c 'psql template1 -c "create extension if not exists vector;"' - - exec: su postgres -c 'psql $db_name -c "create extension if not exists hstore;"' - - exec: su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"' - - exec: su postgres -c 'psql $db_name -c "create extension if not exists vector;"' - exec: + tag: db + cmd: + - "sleep 5" + - su postgres -c 'createdb $db_name' || true + - su postgres -c 'psql $db_name -c "create user $db_user;"' || true + - su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true + - su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"' + - su postgres -c 'psql template1 -c "create extension if not exists hstore;"' + - su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"' + - su postgres -c 'psql template1 -c "create extension if not exists vector;"' + - su postgres -c 'psql $db_name -c "create extension if not exists hstore;"' + - su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"' + - su postgres -c 'psql $db_name -c "create extension if not exists vector;"' + - exec: + tag: db stdin: | update pg_database set encoding = pg_char_to_encoding('UTF8') where datname = '$db_name' AND encoding = pg_char_to_encoding('SQL_ASCII'); cmd: sudo -u postgres psql $db_name raise_on_fail: false + - file: + path: /usr/local/bin/create_db + chmod: +x + contents: | + #!/bin/bash + su postgres -c 'createdb $db_name' || true + su postgres -c 'psql $db_name -c "create user $db_user;"' || true + su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true + su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"' + su postgres -c 'psql template1 -c "create extension if not exists hstore;"' + su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"' + su postgres -c 'psql template1 -c "create extension if not exists vector;"' + su postgres -c 'psql $db_name -c "create extension if not exists hstore;"' + su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"' + su postgres -c 'psql $db_name -c "create extension if not exists vector;"' + sudo -u postgres psql $db_name <<< "update pg_database set encoding = pg_char_to_encoding('UTF8') where datname = '$db_name' AND encoding = pg_char_to_encoding('SQL_ASCII');" || true + - file: path: /var/lib/postgresql/take-database-backup chown: postgres:postgres @@ -242,5 +267,6 @@ run: #0 */4 * * * /var/lib/postgresql/take-database-backup - exec: + tag: db hook: postgres cmd: "echo postgres installed!" diff --git a/templates/redis.template.yml b/templates/redis.template.yml index b5d2a2b10..ce3f019fd 100644 --- a/templates/redis.template.yml +++ b/templates/redis.template.yml @@ -68,16 +68,19 @@ run: - exec: background: true + tag: db cmd: exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf - - - exec: sleep 10 + - exec: + tag: db + cmd: sleep 10 # we can not migrate without redis, launch it if needed hooks: - before_code: + before_db_migrate: - exec: background: true cmd: exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf + - exec: sleep 10 after_code: - replace: filename: /etc/service/unicorn/run diff --git a/templates/web.template.yml b/templates/web.template.yml index 12ba93565..8990f80b2 100644 --- a/templates/web.template.yml +++ b/templates/web.template.yml @@ -22,9 +22,12 @@ params: run: - exec: thpoff echo "thpoff is installed!" - - exec: /usr/local/bin/ruby -e 'if ENV["DISCOURSE_SMTP_ADDRESS"] == "smtp.example.com"; puts "Aborting! Mail is not configured!"; exit 1; end' - - exec: /usr/local/bin/ruby -e 'if ENV["DISCOURSE_HOSTNAME"] == "discourse.example.com"; puts "Aborting! Domain is not configured!"; exit 1; end' - - exec: /usr/local/bin/ruby -e 'if (ENV["DISCOURSE_CDN_URL"] || "")[0..1] == "//"; puts "Aborting! CDN must have a protocol specified. Once fixed you should rebake your posts now to correct all posts."; exit 1; end' + - exec: + tag: precompile + cmd: + - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_SMTP_ADDRESS"] == "smtp.example.com"; puts "Aborting! Mail is not configured!"; exit 1; end' + - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_HOSTNAME"] == "discourse.example.com"; puts "Aborting! Domain is not configured!"; exit 1; end' + - /usr/local/bin/ruby -e 'if (ENV["DISCOURSE_CDN_URL"] || "")[0..1] == "//"; puts "Aborting! CDN must have a protocol specified. Once fixed you should rebake your posts now to correct all posts."; exit 1; end' # TODO: move to base image (anacron can not be fired up using rc.d) - exec: rm -f /etc/cron.d/anacron - file: @@ -55,6 +58,12 @@ run: # postgres cd $home chown -R discourse:www-data /shared/log/rails + if [[ -z "$PRECOMPILE_ON_BOOT" ]]; then + PRECOMPILE_ON_BOOT=1 + fi + if [ -f /usr/local/bin/create_db ] && [ "$CREATE_DB_ON_BOOT" = "1" ]; then /usr/local/bin/create_db; fi; + if [ "$MIGRATE_ON_BOOT" = "1" ]; then su discourse -c 'bundle exec rake db:migrate'; fi + if [ "$PRECOMPILE_ON_BOOT" = "1" ]; then SKIP_EMBER_CLI_COMPILE=1 su discourse -c 'bundle exec rake assets:precompile'; fi LD_PRELOAD=$RUBY_ALLOCATOR HOME=/home/discourse USER=discourse exec thpoff chpst -u discourse:www-data -U discourse:www-data bundle exec config/unicorn_launcher -E production -c config/unicorn.conf.rb - file: @@ -193,14 +202,27 @@ run: - exec: cd: $home + tag: migrate hook: db_migrate cmd: - su discourse -c 'bundle exec rake db:migrate' - exec: cd: $home + tag: build + hook: assets_precompile_build + cmd: + - su discourse -c 'bundle exec rake assets:precompile:build' + - exec: + cd: $home + tag: precompile hook: assets_precompile cmd: - - su discourse -c 'bundle exec rake themes:update assets:precompile' + - su discourse -c 'SKIP_EMBER_CLI_COMPILE=1 bundle exec rake themes:update assets:precompile' + - replace: + tag: precompile + filename: /etc/service/unicorn/run + from: PRECOMPILE_ON_BOOT=1 + to: "PRECOMPILE_ON_BOOT=0" - file: path: /usr/local/bin/discourse