#147: Allow to run Ansible pre/post playbooks during CI build

.gitmodules

@@ -16,3 +16,9 @@
 [submodule "matrix/roles/api/files/cikit-rest-api"]
 	path = matrix/roles/api/files/cikit-rest-api
 	url = https://github.com/BR0kEN-/cikit-rest-api.git
+[submodule "scripts/roles/cikit-ci/files/bash-process-controller"]
+	path = scripts/roles/cikit-ci/files/bash-process-controller
+	url = https://github.com/BR0kEN-/bash-process-controller.git
+[submodule "tests/travis/tests-runner"]
+	path = tests/travis/tests-runner
+	url = https://github.com/BR0kEN-/tests-runner.git

.travis.yml

@@ -25,15 +25,15 @@ env:
 
 install:
   - sudo pip install ansible
-  - sudo bash ./install.sh --no-requirements-check
-  - sudo wget -nv https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_x86_64.deb
-  - sudo dpkg -i vagrant_${VAGRANT_VERSION}_x86_64.deb
+  - sudo bash ./install.sh --no-requirements-check "${TRAVIS_BRANCH}"
+  - sudo wget -nv "https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_x86_64.deb"
+  - sudo dpkg -i "vagrant_${VAGRANT_VERSION}_x86_64.deb"
   - cd /usr/local/share/cikit
-  - sudo git fetch origin ${TRAVIS_BRANCH}
-  - sudo git checkout ${TRAVIS_BRANCH}
+  - sudo git fetch origin "${TRAVIS_BRANCH}"
+  - sudo git checkout "${TRAVIS_BRANCH}"
 
 script:
-  - sudo ./tests/travis/runner.sh
+  - sudo ./tests/travis/tests-runner/runner.sh ./tests/travis
   - env
 
 notifications:

cmf/all/.cikit/ci/post-deploy.yml

@@ -0,0 +1,24 @@
+# This is a playbook that runs after a successful build of the application.
+# To do something before the build create the "pre-deploy.yml" that'll have
+# the same set of input variables but will run early, before any other tasks.
+#
+# Available variables:
+# - rc: an exit code of the build process. If it's not "0" then a build failed;
+# - dist: an absolute path to the project's destination;
+# - env: a name of the environment being built;
+# - site-url: a URL of a website being built;
+# - build-id: an ID of CI build;
+# - workspace: an absolute path to the sandbox directory on CI server.
+---
+- hosts: localhost
+  gather_facts: no
+  connection: local
+  become: yes
+
+  vars_files:
+    - ../config.yml
+    - ../../scripts/vars/main.yml
+
+  tasks:
+    - name: Move reports to the website directory
+      shell: "cp -r {{ workspace }}/docroot/reports/ {{ dist }}/docroot"

cmf/all/.cikit/ci/server-cleaner.yml

@@ -0,0 +1,36 @@
+# This is a playbook that runs if CI server has less than 90% of a drive space
+# available.
+#
+# Available variables:
+# - rc: an exit code of the build process. If it's not "0" then a build failed;
+# - dist: an absolute path to the project's destination;
+# - env: a name of the environment being built;
+# - site-url: a URL of a website being built;
+# - build-id: an ID of CI build;
+# - workspace: an absolute path to the sandbox directory on CI server.
+---
+- hosts: localhost
+  gather_facts: no
+  connection: local
+  become: yes
+
+  vars_files:
+    - ../config.yml
+    - ../../scripts/vars/main.yml
+
+  tasks:
+    - name: Compute disk usage
+      shell: df -H | head -2 | tail -1 | awk '{printf "%d", $5}'
+      register: disk_usage
+
+    - name: Remove builds
+      shell: |-
+        COMMAND="mysql -u{{ mysql.user }} -p{{ mysql.pass }} -se"
+
+        ${COMMAND} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" ${COMMAND} "DROP DATABASE \`@@\`"
+        rm -rf {{ webroot }}/*{{ build_slug }}*
+      args:
+        warn: no
+        executable: bash
+      # The drive space is occupied by more than 90%.
+      when: disk_usage.stdout > 90

cmf/all/.cikit/config.yml

@@ -4,6 +4,12 @@ tmproot: /tmp
 webroot: /var/www
 # The directory to "cd" into after "vagrant ssh". Defaults to the value of "webroot".
 ssh_home: ~
+# The part of a directory/database name for CI builds.
+build_slug: build
+# Do not try changing these values.
+mysql:
+  user: root
+  pass: root
 APPLICATION_CONFIG: ~
 vm:
   ip: 192.168.56.132

cmf/all/scripts/vars/main.yml

@@ -10,13 +10,9 @@ reinstall_mode: full
 env_vars:
   APP_ENV: "{{ env }}"
 
-mysql:
-  user: root
-  pass: root
-
 databases:
   default:
-    name: "{{ cmf }}_{{ project | replace('-', '_') }}_{{ build_id | default(env) }}"
+    name: "{{ [cmf, build_id | default(project + '_' + env)] | join('_') | replace('-', '_') }}"
 #    source:
 #      # Database name on remote host.
 #      db: ""

cmf/drupal/all/scripts/vars/environments/demo.yml

@@ -7,7 +7,6 @@ commands:
 #    - cset: ["system.file", "path.temporary", "{{ tmproot }}"]
 #    - cset: ["system.logging", "error_level", "none"]
     - en: ["dblog"]
-    - dis: ["update"]
 
 drupal:
   # Data for super-admin (UID 1).

docs/_posts/2018-04-24-travis-tests-automation.md

@@ -158,3 +158,7 @@ When skipping a test the extension of a file **MUST NOT** be specified. If file
 ## Check in action
 
 The [.travis.yml](https://github.com/BR0kEN-/cikit/blob/c37173b93d1eaee9b090fe4655cf6e5081122942/.travis.yml#L37) of CIKit uses exactly same [runner](https://github.com/BR0kEN-/cikit/blob/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/runner.sh#L1), which is responsible for launching [Bash](https://github.com/BR0kEN-/cikit/tree/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/bash) and [Python](https://github.com/BR0kEN-/cikit/tree/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/python) tests.
+
+## Project page
+
+Project codebase is available at [https://github.com/BR0kEN-/tests-runner](https://github.com/BR0kEN-/tests-runner).

install.sh

@@ -1,27 +1,34 @@
 #!/usr/bin/env bash
 
 INSTALL_PATH="/usr/local/share/cikit"
+NOCHECK="--no-requirements-check"
+ARGS="$*"
 
-if [ "--no-requirements-check" != "${1}" ]; then
+if [[ ! "$ARGS" =~ .*$NOCHECK.* ]]; then
   MISSING=""
 
   for COMMAND in vagrant VBoxManage ansible-playbook; do
-    if ! \command -v "${COMMAND}" > /dev/null; then
-      MISSING+="\n- ${COMMAND}"
+    if ! \command -v "$COMMAND" > /dev/null; then
+      MISSING+="\n- $COMMAND"
     fi
   done
 
-  if [ -n "${MISSING}" ]; then
-    \echo -e "The following software were not found on your machine, so continuation is not possible:${MISSING}"
+  if [ -n "$MISSING" ]; then
+    \echo -e "The following software were not found on your machine, so continuation is not possible:$MISSING"
     \exit 1
   fi
 fi
 
-if [ ! -d "${INSTALL_PATH}" ]; then
-  sudo \mkdir -p "${INSTALL_PATH}"
+ARGS="${ARGS//$NOCHECK/}"
+VERSION="$(\tr -d '[:blank:]' <<< "$ARGS")"
+
+: "${VERSION:="master"}"
+
+if [ ! -d "$INSTALL_PATH" ]; then
+  sudo \mkdir -p "$INSTALL_PATH"
 fi
 
-if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive "${INSTALL_PATH}"; then
-  sudo \ln -s "${INSTALL_PATH}/lib/cikit" /usr/local/bin/cikit
-  sudo \chown -R "$(\whoami)" "${INSTALL_PATH}"
+if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive --branch="$VERSION" "$INSTALL_PATH"; then
+  sudo \ln -s "$INSTALL_PATH/lib/cikit" /usr/local/bin/cikit
+  sudo \chown -R "$(\whoami)" "$INSTALL_PATH"
 fi

lib/.version

@@ -1 +1 @@
-1.0.3
+1.0.4

lib/platformsh/setup.yml

@@ -1,3 +1,4 @@
+# This playbook runs by Vagrantfile only.
 ---
 - hosts: localhost
   connection: local

scripts/roles/cikit-ci/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+cikit_ci:
+  process_controller:
+    repo: https://github.com/BR0kEN-/bash-process-controller.git
+    dest: /var/ci/process-controller
+    # Either branch name or tag is also allowed.
+    version: HEAD

scripts/roles/cikit-ci/files/tasks.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+
+# Allows continuing the process regardless of an exit code at one of the stages.
+set +e
+
+# ------------------------------------------------------------------------------
+# The required configuration of a process.
+# ------------------------------------------------------------------------------
+
+declare -rA VARIABLES=(
+  [BUILD_NUMBER]="either \"stable\" or any custom value, like \"54\" or \"my-pr\""
+  [BUILD_MODE]="either \"full\", \"pull\" or whatever you have defined in \"scripts/tasks/reinstall/modes/*.yml\""
+  [BUILD_ENV]="the name of an environment to build or \"default\""
+  [CIKIT_PROJECT_DIR]="the path to directory where repository clones to"
+  [CIKIT_PROJECT_HOSTNAME]="the hostname where the project can be accessed"
+  [RUN_SNIFFERS]="either \"yes\" or whatever"
+  [RUN_TESTS]="either \"yes\" or whatever"
+)
+
+for VARIABLE in "${!VARIABLES[@]}"; do
+  if [ -z ${!VARIABLE+x} ]; then
+    echo "The \"$VARIABLE\" variable is missing! Its value must be ${VARIABLES[$VARIABLE]}."
+    exit 101
+  fi
+
+  # Ensure the variable is available in the subshells.
+  # shellcheck disable=SC2163
+  # https://github.com/koalaman/shellcheck/wiki/SC2163
+  export "${VARIABLE}"
+done
+
+# ------------------------------------------------------------------------------
+# Read CIKit configuration.
+# ------------------------------------------------------------------------------
+
+declare -A CIKIT_PROJECT_CONFIG=()
+
+for VARIABLE in webroot project build_slug; do
+  VALUE="$(awk '/^'"$VARIABLE"':/ {print $2}' < "$CIKIT_PROJECT_DIR/.cikit/config.yml")"
+
+  if [ -z "$VALUE" ]; then
+    echo "The value of \"$VARIABLE\" variable cannot be empty!"
+    exit 102
+  fi
+
+  CIKIT_PROJECT_CONFIG["$VARIABLE"]="$VALUE"
+done
+
+# ------------------------------------------------------------------------------
+# Compute build parameters.
+# ------------------------------------------------------------------------------
+
+if [ "$BUILD_NUMBER" == "stable" ]; then
+  export IS_COMMIT=false
+  BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-$BUILD_ENV"
+else
+  export IS_COMMIT=true
+  BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-${CIKIT_PROJECT_CONFIG['build_slug']}-$BUILD_NUMBER"
+fi
+
+# https://docs.python.org/2/using/cmdline.html#cmdoption-u
+export PYTHONUNBUFFERED=1
+# Replace underscores by dashes in the ID of a build.
+export BUILD_ID="${BUILD_ID//_/-}"
+# Form an absolute path to directory where the project is accessible from web.
+export DESTINATION="${CIKIT_PROJECT_CONFIG['webroot']}/$BUILD_ID"
+
+# Print the environment.
+env
+
+# ------------------------------------------------------------------------------
+# Define the runtime.
+# ------------------------------------------------------------------------------
+
+cikit_run() {
+  cikit "$1" \
+    "${@:2}" \
+    --env="$BUILD_ENV" \
+    --site-url="https://$BUILD_ID.$CIKIT_PROJECT_HOSTNAME" \
+    --build-id="$BUILD_ID" \
+    --workspace="$CIKIT_PROJECT_DIR"
+}
+
+cikit_hook() {
+  local HOOK_PLAYBOOK="$CIKIT_PROJECT_DIR/.cikit/ci/$1.yml"
+
+  if [ -f "$HOOK_PLAYBOOK" ]; then
+    cikit_run "$HOOK_PLAYBOOK" --dist="$DESTINATION" --rc="$PROCESS_EXIT_CODE"
+  else
+    echo "==> There's no \"$HOOK_PLAYBOOK\" hook to run."
+  fi
+}
+
+export -f cikit_run cikit_hook
+
+# ------------------------------------------------------------------------------
+# Define the process.
+# ------------------------------------------------------------------------------
+
+PROCESS_pre() {
+  cikit_hook pre-deploy
+}
+
+PROCESS_main() {
+  # Install a project.
+  if ${IS_COMMIT}; then
+    cikit_run reinstall --actions="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")"
+  else
+    cikit_run reinstall --reinstall-mode="$BUILD_MODE"
+  fi
+
+  # Copy codebase to directory accessible from the web.
+  sudo rsync -ra --delete --chown=www-data:www-data ./ "$DESTINATION/"
+
+  if [ "$RUN_SNIFFERS" == "yes" ]; then
+    cikit_run sniffers
+  fi
+
+  if [ "$RUN_TESTS" == "yes" ]; then
+    cikit_run tests --run --headless
+  fi
+}
+
+PROCESS_post() {
+  cikit_hook post-deploy
+}
+
+PROCESS_clean() {
+  cikit_hook server-cleaner
+}
+
+PROCESS_finish() {
+  echo "Restore permissions for \"$USER\"."
+  # The "$USER" must be either "jenkins" or "gitlab-runner".
+  sudo chown -R "$USER":"$USER" "$CIKIT_PROJECT_DIR"
+  # The "$HOME" will be for the "$USER".
+  sudo chown -R "$USER":"$USER" "$HOME"
+}
+
+export -f PROCESS_pre PROCESS_main PROCESS_post PROCESS_clean PROCESS_finish

scripts/roles/cikit-ci/tasks/main.yml

@@ -1,5 +1,38 @@
 ---
-- include_role:
+- name: Determine CI service
+  set_fact:
     # Install Gitlab CI if either "--gitlab-ci-token" or "--gitlab-ci-url"
     # is passed. Use Jenkins otherwise.
-    name: "cikit-{{ 'gitlab-ci' if gitlab_ci_token | default or gitlab_ci_url | default else 'jenkins' }}"
+    cikit_ci_gitlab: "{{ gitlab_ci_token | default or gitlab_ci_url | default }}"
+
+- include_role:
+    name: "cikit-{{ 'gitlab-ci' if cikit_ci_gitlab else 'jenkins' }}"
+
+- name: Determine the group and owner of CI service
+  set_fact:
+    cikit_ci_owner: "{{ 'root' if cikit_ci_gitlab else jenkins_data.user }}"
+    cikit_ci_group: "{{ 'root' if cikit_ci_gitlab else jenkins_data.group }}"
+
+- name: Ensure the directory for CI scripts exist
+  file:
+    path: "{{ cikit_ci.process_controller.dest | dirname }}"
+    mode: 0755
+    state: directory
+    owner: "{{ cikit_ci_owner }}"
+    group: "{{ cikit_ci_group }}"
+  register: cikit_ci_scripts_path
+
+- name: Install/update CI process controller
+  git:
+    repo: "{{ cikit_ci.process_controller.repo }}"
+    dest: "{{ cikit_ci.process_controller.dest }}"
+    force: yes
+    update: yes
+    version: "{{ cikit_ci.process_controller.version }}"
+
+- name: Deploy CI process handler
+  copy:
+    src: tasks.sh
+    dest: "{{ cikit_ci_scripts_path.path }}"
+    owner: "{{ cikit_ci_owner }}"
+    group: "{{ cikit_ci_group }}"