Add tags to pups templates

The purpose here is to allow greater flexibility in how and where
docker images are built and run. It achieves this by breaking up
build steps into distinct run steps which can be saved along the way.
Customizable base images may then be prebuilt with as many batteries
included as possible, with zero environment setup so those images
can then be configured at a later stage.

Add the ability to run partial pups configuration:
`build`: build base image with no db - ember build.
`precompile`: precompile stage that requires postgres and redis.
`migrate`: run migration tasks.
`db`: start bundled postgres/redis, if included.

Adds a create_db script in postgres template for creating db on the fly.
Called below in unicorn run:

updates unicorn run command with 3 env flags:
CREATE_DB_ON_BOOT: if 1, creates base db schema, allows for deferral of creation.
MIGRATE_ON_BOOT: if 1, runs db:migrate - allows for deferral of db migration.
PRECOMPILE_ON_BOOT: if 1, precompiles assets (without ember build).

PRECOMPILE_ON_BOOT initially defaults to 1 in base builds (no tags).
During the `precompile` build step, this updates the default to be 0.

All other new flags default to 0 (off). With these three flags, we're now able
to ship and start a container from a base image, and it'll be able to bootstrap
a blank database.

Updates hook to start redis before_db_migrate as before_code hook
is not guaranteed to fire before migrate tasks if pups is filtered by tags.

templates/postgres.template.yml

@@ -17,17 +17,20 @@ hooks:
        to: sv start postgres || exit 1
 
 run:
-  - exec: locale-gen $LANG && update-locale
-  - exec: mkdir -p /shared/postgres_run
-  - exec: chown postgres:postgres /shared/postgres_run
-  - exec: chmod 775 /shared/postgres_run
-  - exec: rm -fr /var/run/postgresql
-  - exec: ln -s /shared/postgres_run /var/run/postgresql
-  - exec: socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1
-  - exec: rm -fr /shared/postgres_run/.s*
-  - exec: rm -fr /shared/postgres_run/*.pid
-  - exec: mkdir -p /shared/postgres_run/13-main.pg_stat_tmp
-  - exec: chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp
+  - exec:
+      cmd:
+        - locale-gen $LANG && update-locale
+        - mkdir -p /shared/postgres_run
+        - chown postgres:postgres /shared/postgres_run
+        - chmod 775 /shared/postgres_run
+        - rm -fr /var/run/postgresql
+        - ln -s /shared/postgres_run /var/run/postgresql
+        - socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1
+        - rm -fr /shared/postgres_run/.s*
+        - rm -fr /shared/postgres_run/*.pid
+        - mkdir -p /shared/postgres_run/13-main.pg_stat_tmp
+        - chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp
+      tag: db
   - file:
      path: /etc/service/postgres/run
      chmod: "+x"
@@ -117,14 +120,14 @@ run:
        fi
 
   - exec:
+      tag: db
       cmd:
         - chown -R root /var/lib/postgresql/13/main
         - "[ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data || exit 0"
         - chown -R postgres:postgres /shared/postgres_data
         - chown -R postgres:postgres /var/run/postgresql
-
-  - exec: /root/upgrade_postgres
-  - exec: rm /root/upgrade_postgres
+        - /root/upgrade_postgres
+        - rm /root/upgrade_postgres
 
   - replace:
       filename: "/etc/postgresql/13/main/postgresql.conf"
@@ -165,6 +168,7 @@ run:
   - exec:
       cmd:
         - install -d -m 0755 -o postgres -g postgres /shared/postgres_backup
+      tag: db
 
   - replace:
       filename: "/etc/postgresql/13/main/postgresql.conf"
@@ -199,30 +203,51 @@ run:
       to: "host all all ::/0 md5"
 
   - exec:
+      tag: db
       background: true
       # use fast shutdown for pg
       stop_signal: INT
       cmd: HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/13/bin/postmaster -D /etc/postgresql/13/main
 
   # give db a few secs to start up
-  - exec: "sleep 5"
-
-  - exec: su postgres -c 'createdb $db_name' || true
-  - exec: su postgres -c 'psql $db_name -c "create user $db_user;"' || true
-  - exec: su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true
-  - exec: su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"'
-  - exec: su postgres -c 'psql template1 -c "create extension if not exists hstore;"'
-  - exec: su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"'
-  - exec: su postgres -c 'psql template1 -c "create extension if not exists vector;"'
-  - exec: su postgres -c 'psql $db_name -c "create extension if not exists hstore;"'
-  - exec: su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"'
-  - exec: su postgres -c 'psql $db_name -c "create extension if not exists vector;"'
   - exec:
+      tag: db
+      cmd:
+        - "sleep 5"
+        - su postgres -c 'createdb $db_name' || true
+        - su postgres -c 'psql $db_name -c "create user $db_user;"' || true
+        - su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true
+        - su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"'
+        - su postgres -c 'psql template1 -c "create extension if not exists hstore;"'
+        - su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"'
+        - su postgres -c 'psql template1 -c "create extension if not exists vector;"'
+        - su postgres -c 'psql $db_name -c "create extension if not exists hstore;"'
+        - su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"'
+        - su postgres -c 'psql $db_name -c "create extension if not exists vector;"'
+  - exec:
+      tag: db
       stdin: |
         update pg_database set encoding = pg_char_to_encoding('UTF8') where datname = '$db_name' AND encoding = pg_char_to_encoding('SQL_ASCII');
       cmd: sudo -u postgres psql $db_name
       raise_on_fail: false
 
+  - file:
+      path: /usr/local/bin/create_db
+      chmod: +x
+      contents: |
+        #!/bin/bash
+        su postgres -c 'createdb $db_name' || true
+        su postgres -c 'psql $db_name -c "create user $db_user;"' || true
+        su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true
+        su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"'
+        su postgres -c 'psql template1 -c "create extension if not exists hstore;"'
+        su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"'
+        su postgres -c 'psql template1 -c "create extension if not exists vector;"'
+        su postgres -c 'psql $db_name -c "create extension if not exists hstore;"'
+        su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"'
+        su postgres -c 'psql $db_name -c "create extension if not exists vector;"'
+        sudo -u postgres psql $db_name <<< "update pg_database set encoding = pg_char_to_encoding('UTF8') where datname = '$db_name' AND encoding = pg_char_to_encoding('SQL_ASCII');" || true
+
   - file:
      path: /var/lib/postgresql/take-database-backup
      chown: postgres:postgres
@@ -242,5 +267,6 @@ run:
         #0 */4 * * * /var/lib/postgresql/take-database-backup
 
   - exec:
+      tag: db
       hook: postgres
       cmd: "echo postgres installed!"

templates/redis.template.yml

@@ -68,16 +68,19 @@ run:
 
   - exec:
       background: true
+      tag: db
       cmd: exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf
-
-  - exec: sleep 10
+  - exec:
+      tag: db
+      cmd: sleep 10
 
 # we can not migrate without redis, launch it if needed
 hooks:
-  before_code:
+  before_db_migrate:
     - exec:
         background: true
         cmd: exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf
+    - exec: sleep 10
   after_code:
     - replace:
        filename: /etc/service/unicorn/run

templates/web.template.yml

@@ -22,9 +22,12 @@ params:
 
 run:
   - exec: thpoff echo "thpoff is installed!"
-  - exec: /usr/local/bin/ruby -e 'if ENV["DISCOURSE_SMTP_ADDRESS"] == "smtp.example.com"; puts "Aborting! Mail is not configured!"; exit 1; end'
-  - exec: /usr/local/bin/ruby -e 'if ENV["DISCOURSE_HOSTNAME"] == "discourse.example.com"; puts "Aborting! Domain is not configured!"; exit 1; end'
-  - exec: /usr/local/bin/ruby -e 'if (ENV["DISCOURSE_CDN_URL"] || "")[0..1] == "//"; puts "Aborting! CDN must have a protocol specified. Once fixed you should rebake your posts now to correct all posts."; exit 1; end'
+  - exec:
+      tag: precompile
+      cmd:
+        - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_SMTP_ADDRESS"] == "smtp.example.com"; puts "Aborting! Mail is not configured!"; exit 1; end'
+        - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_HOSTNAME"] == "discourse.example.com"; puts "Aborting! Domain is not configured!"; exit 1; end'
+        - /usr/local/bin/ruby -e 'if (ENV["DISCOURSE_CDN_URL"] || "")[0..1] == "//"; puts "Aborting! CDN must have a protocol specified. Once fixed you should rebake your posts now to correct all posts."; exit 1; end'
   # TODO: move to base image (anacron can not be fired up using rc.d)
   - exec: rm -f /etc/cron.d/anacron
   - file:
@@ -55,6 +58,12 @@ run:
         # postgres
         cd $home
         chown -R discourse:www-data /shared/log/rails
+        if [[ -z "$PRECOMPILE_ON_BOOT" ]]; then
+          PRECOMPILE_ON_BOOT=1
+        fi
+        if [ -f /usr/local/bin/create_db ] && [ "$CREATE_DB_ON_BOOT" = "1" ]; then /usr/local/bin/create_db; fi;
+        if [ "$MIGRATE_ON_BOOT" = "1" ]; then su discourse -c 'bundle exec rake db:migrate'; fi
+        if [ "$PRECOMPILE_ON_BOOT" = "1" ]; then SKIP_EMBER_CLI_COMPILE=1 su discourse -c 'bundle exec rake assets:precompile'; fi
         LD_PRELOAD=$RUBY_ALLOCATOR HOME=/home/discourse USER=discourse exec thpoff chpst -u discourse:www-data -U discourse:www-data bundle exec config/unicorn_launcher -E production -c config/unicorn.conf.rb
 
   - file:
@@ -193,14 +202,27 @@ run:
 
   - exec:
       cd: $home
+      tag: migrate
       hook: db_migrate
       cmd:
         - su discourse -c 'bundle exec rake db:migrate'
   - exec:
       cd: $home
+      tag: build
+      hook: assets_precompile_build
+      cmd:
+        - su discourse -c 'bundle exec rake assets:precompile:build'
+  - exec:
+      cd: $home
+      tag: precompile
       hook: assets_precompile
       cmd:
-        - su discourse -c 'bundle exec rake themes:update assets:precompile'
+        - su discourse -c 'SKIP_EMBER_CLI_COMPILE=1 bundle exec rake themes:update assets:precompile'
+  - replace:
+      tag: precompile
+      filename: /etc/service/unicorn/run
+      from: PRECOMPILE_ON_BOOT=1
+      to: "PRECOMPILE_ON_BOOT=0"
 
   - file:
      path: /usr/local/bin/discourse