From 2b9cf12c0ac62be794bcab425c281b9d9fe15dfa Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 24 Jun 2018 01:11:39 +0400 Subject: [PATCH 01/15] #147: Allow to run Ansible pre/post playbooks during CI build --- cmf/all/.cikit/ci/post-deploy.yml | 19 +++++++ .../templates/.gitlab-ci.yml.j2 | 5 +- .../templates/jobs/builder.xml.j2 | 56 ++++++++++--------- .../templates/jobs/pr_builder.xml.j2 | 43 ++++++++------ 4 files changed, 78 insertions(+), 45 deletions(-) create mode 100644 cmf/all/.cikit/ci/post-deploy.yml diff --git a/cmf/all/.cikit/ci/post-deploy.yml b/cmf/all/.cikit/ci/post-deploy.yml new file mode 100644 index 00000000..f3bc663e --- /dev/null +++ b/cmf/all/.cikit/ci/post-deploy.yml @@ -0,0 +1,19 @@ +# This is a playbook that runs after a successful build of the application. +# To do something before the build create the "pre-deploy.yml" that'll have +# the same set of input variables but will run early, before any other tasks. +# +# Available variables: +# - dist: an absolute path to the project's destination; +# - env: a name of the environment being built; +# - site-url: a URL of a website being built; +# - build-id: an ID of CI build; +# - workspace: an absolute path to the sandbox directory on CI server. +--- +- hosts: localhost + gather_facts: no + connection: local + become: yes + + tasks: + - name: Move reports to the website directory + shell: "cp -r {{ workspace }}/docroot/reports/ {{ dist }}/docroot" diff --git a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 index 3fc5e113..63041481 100644 --- a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 +++ b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 @@ -44,14 +44,15 @@ build/run: - export BUILD_ACTIONS="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" # Dump all available variables for debugging purposes. - env + # @todo Unify hooks triggers as it's done for Jenkins. + - [ -f "${CI_PROJECT_DIR}/.cikit/ci/pre-deploy.yml" ] && cikit "${CI_PROJECT_DIR}/.cikit/ci/pre-deploy.yml" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" --dist="${DESTINATION}" # Reinstall a project. - CIKIT_PROJECT_DIR="${CI_PROJECT_DIR}" cikit reinstall --actions="${BUILD_ACTIONS}" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" # Copy codebase to the "web" directory. - sudo rsync -ra --delete --chown=www-data:www-data ./ "${DESTINATION}" # Run sniffers. - CIKIT_PROJECT_DIR="${DESTINATION}" cikit sniffers --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${DESTINATION}" - # Copy artifacts. - - cp -r "${DESTINATION}/docroot/reports/" "${CI_PROJECT_DIR}/artifacts/" + - [ -f "${CI_PROJECT_DIR}/.cikit/ci/post-deploy.yml" ] && cikit "${CI_PROJECT_DIR}/.cikit/ci/post-deploy.yml" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" --dist="${DESTINATION}" artifacts: paths: - artifacts/ diff --git a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 index 71abca39..d081cad0 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 @@ -81,47 +81,49 @@ Pull - do not reinstall an application and just grab latest changes from reposit false - #!/usr/bin/env bash -e + #!/usr/bin/env bash -export PYTHONUNBUFFERED=1 -export PROJECT="$(echo ${JOB_NAME//_BUILDER/} | tr '[:upper:]' '[:lower:]')" +set -e + +export PROJECT="$(echo "${JOB_NAME//_BUILDER/}" | tr '[:upper:]' '[:lower:]')" export BUILD_NAME="${PROJECT}_${BUILD_ENV}" -export SITE_URL="https://${BUILD_NAME}.$(php -r "echo parse_url('${JOB_URL}')['host'];")" -export DESTINATION="/var/www/${BUILD_NAME}" -export CIKIT_PROJECT_DIR="${WORKSPACE}" +export SITE_URL="https://$BUILD_NAME.$(php -r "echo parse_url('${JOB_URL}')['host'];")" +export DESTINATION="/var/www/$BUILD_NAME" +export CIKIT_PROJECT_DIR="$WORKSPACE" +export PYTHONUNBUFFERED=1 export ANSIBLE_VERBOSITY=2 -cikit reinstall \ - --env="${BUILD_ENV}" \ - --site-url="${SITE_URL}" \ - --build-id="${BUILD_NAME}" \ - --workspace="${WORKSPACE}" \ - --reinstall-mode="${BUILD_MODE}" +ARGS=( + "--env=$BUILD_ENV" + "--site-url=$SITE_URL" + "--build-id=$BUILD_NAME" + "--workspace=$WORKSPACE" +) + +deploy_hook() { + local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml" + + if [ -f "$HOOK_PLAYBOOK" ]; then + cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" + fi +} + +deploy_hook pre-deploy +cikit reinstall "${ARGS[@]}" --reinstall-mode="$BUILD_MODE" # Copy codebase to directory, accessible from web. -sudo rsync --delete -ra ./ "${DESTINATION}/" +sudo rsync --delete -ra ./ "$DESTINATION/" sudo chown -R www-data:jenkins $_ if ${RUN_SNIFFERS}; then - cikit sniffers \ - --env="${BUILD_ENV}" \ - --site-url="${SITE_URL}" \ - --build-id="${BUILD_NAME}" \ - --workspace="${WORKSPACE}" + cikit sniffers "${ARGS[@]}" fi if ${RUN_TESTS}; then - cikit tests \ - --run \ - --headless \ - --env=${BUILD_ENV} \ - --site-url=${SITE_URL} \ - --build-id=${BUILD_NAME} \ - --workspace=${WORKSPACE} + cikit tests "${ARGS[@]}" --run --headless fi -# Move reports to the website directory. -sudo cp -r ./docroot/reports/ "${DESTINATION}/docroot/" +deploy_hook post-deploy sudo chown -R jenkins:jenkins ./ diff --git a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 index f74e4f73..a854f6e2 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 @@ -93,35 +93,46 @@ false - #!/usr/bin/env bash -e + #!/usr/bin/env bash -export PROJECT=$(echo ${JOB_NAME//_PR_BUILDER/} | tr '[:upper:]' '[:lower:]') +set -e + +export PROJECT="$(echo "${JOB_NAME//_BUILDER/}" | tr '[:upper:]' '[:lower:]')" export BUILD_NAME="${PROJECT}_build_${BUILD_NUMBER}" -export SITE_URL="https://${BUILD_NAME}.$(php -r "echo parse_url('${JOB_URL}')['host'];")" -export DESTINATION="/var/www/${BUILD_NAME}" -export CIKIT_PROJECT_DIR="${WORKSPACE}" +export SITE_URL="https://$BUILD_NAME.$(php -r "echo parse_url('${JOB_URL}')['host'];")" +export DESTINATION="/var/www/$BUILD_NAME" +export CIKIT_PROJECT_DIR="$WORKSPACE" +export PYTHONUNBUFFERED=1 export ANSIBLE_VERBOSITY=2 export BUILD_ACTIONS=$(php -r "echo json_encode(array_map('trim', array_filter( explode(PHP_EOL, '$(git log --format=%B -n1 ${ghprbActualCommit} | awk -vRS="]" -vFS="[" '{print $2}')') )));") -cikit reinstall \ - --actions="${BUILD_ACTIONS}" \ - --site-url="${SITE_URL}" \ - --build-id="${BUILD_NAME}" \ - --workspace="${WORKSPACE}" +ARGS=( + "--site-url=$SITE_URL" + "--build-id=$BUILD_NAME" + "--workspace=$WORKSPACE" +) + +deploy_hook() { + local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml" + + if [ -f "$HOOK_PLAYBOOK" ]; then + cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" + fi +} + +deploy_hook pre-deploy +cikit reinstall "${ARGS[@]}" --actions="$BUILD_ACTIONS" # Copy codebase to directory, accessible from web. -sudo rsync --delete -ra ./ "${DESTINATION}/" +sudo rsync --delete -ra ./ "$DESTINATION/" sudo chown -R www-data:jenkins $_ -cikit sniffers \ - --site-url=${SITE_URL} \ - --build-id=${BUILD_NAME} \ - --workspace=${WORKSPACE} +cikit sniffers "${ARGS[@]}" # Move reports to the website directory. -sudo cp -r ./docroot/reports/ "${DESTINATION}/docroot/" +deploy_hook post-deploy sudo chown -R jenkins:jenkins ./ From 7934b58422a2658e9a5a2b9509a102ac45205c5d Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Mon, 25 Jun 2018 12:06:21 +0400 Subject: [PATCH 02/15] #147: Run "post-deploy.yml" even if a build failed --- cmf/all/.cikit/ci/post-deploy.yml | 1 + .../cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 | 1 + .../cikit-jenkins/templates/jobs/builder.xml.j2 | 11 +++++++---- .../cikit-jenkins/templates/jobs/pr_builder.xml.j2 | 12 +++++++----- scripts/roles/cikit-sniffers/defaults/main.yml | 4 +++- scripts/roles/cikit-sniffers/tasks/main.yml | 7 +++++++ 6 files changed, 26 insertions(+), 10 deletions(-) diff --git a/cmf/all/.cikit/ci/post-deploy.yml b/cmf/all/.cikit/ci/post-deploy.yml index f3bc663e..1536e5e0 100644 --- a/cmf/all/.cikit/ci/post-deploy.yml +++ b/cmf/all/.cikit/ci/post-deploy.yml @@ -3,6 +3,7 @@ # the same set of input variables but will run early, before any other tasks. # # Available variables: +# - rc: an exit code of the build process. If it's not "0" then a build failed; # - dist: an absolute path to the project's destination; # - env: a name of the environment being built; # - site-url: a URL of a website being built; diff --git a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 index 63041481..45bff27a 100644 --- a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 +++ b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 @@ -44,6 +44,7 @@ build/run: - export BUILD_ACTIONS="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" # Dump all available variables for debugging purposes. - env + # @todo Add "trap" to run "post-deploy" anyway. # @todo Unify hooks triggers as it's done for Jenkins. - [ -f "${CI_PROJECT_DIR}/.cikit/ci/pre-deploy.yml" ] && cikit "${CI_PROJECT_DIR}/.cikit/ci/pre-deploy.yml" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" --dist="${DESTINATION}" # Reinstall a project. diff --git a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 index d081cad0..eb532b9e 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 @@ -104,10 +104,16 @@ deploy_hook() { local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml" if [ -f "$HOOK_PLAYBOOK" ]; then - cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" + cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" --rc="$2" fi } +handle_shutdown() { + deploy_hook post-deploy $? + sudo chown -R jenkins:jenkins "$WORKSPACE" +} + +trap handle_shutdown EXIT deploy_hook pre-deploy cikit reinstall "${ARGS[@]}" --reinstall-mode="$BUILD_MODE" @@ -122,9 +128,6 @@ fi if ${RUN_TESTS}; then cikit tests "${ARGS[@]}" --run --headless fi - -deploy_hook post-deploy -sudo chown -R jenkins:jenkins ./ diff --git a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 index a854f6e2..8a318c56 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 @@ -118,10 +118,16 @@ deploy_hook() { local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml" if [ -f "$HOOK_PLAYBOOK" ]; then - cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" + cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" --rc="$2" fi } +handle_shutdown() { + deploy_hook post-deploy $? + sudo chown -R jenkins:jenkins "$WORKSPACE" +} + +trap handle_shutdown EXIT deploy_hook pre-deploy cikit reinstall "${ARGS[@]}" --actions="$BUILD_ACTIONS" @@ -130,10 +136,6 @@ sudo rsync --delete -ra ./ "$DESTINATION/" sudo chown -R www-data:jenkins $_ cikit sniffers "${ARGS[@]}" - -# Move reports to the website directory. -deploy_hook post-deploy -sudo chown -R jenkins:jenkins ./ diff --git a/scripts/roles/cikit-sniffers/defaults/main.yml b/scripts/roles/cikit-sniffers/defaults/main.yml index 2571dfb6..43e6f5c7 100644 --- a/scripts/roles/cikit-sniffers/defaults/main.yml +++ b/scripts/roles/cikit-sniffers/defaults/main.yml @@ -12,6 +12,7 @@ cikit_sniffers: Drupal: version: 8.x-2.x repo: git://git.drupal.org/project/coder.git + build: composer install DrupalSecure: version: master repo: http://git.drupal.org/sandbox/coltrane/1921926.git @@ -25,5 +26,6 @@ cikit_sniffers: version: master repo: https://github.com/WordPress-Coding-Standards/WordPress-Coding-Standards.git HTML: - version: 2.1.1 + version: 2.2.0 repo: https://github.com/squizlabs/HTML_CodeSniffer.git + build: npm install && ./node_modules/.bin/grunt build diff --git a/scripts/roles/cikit-sniffers/tasks/main.yml b/scripts/roles/cikit-sniffers/tasks/main.yml index 40dfe590..fe62d411 100644 --- a/scripts/roles/cikit-sniffers/tasks/main.yml +++ b/scripts/roles/cikit-sniffers/tasks/main.yml @@ -6,6 +6,13 @@ version: "{{ item.value.version }}" with_dict: "{{ cikit_sniffers.coding_standards.list }}" +- name: "Build '{{ item.key }}'" + shell: "{{ item.value.build }}" + args: + chdir: "{{ cikit_sniffers.coding_standards.dest }}/{{ item.key }}" + when: item.value.build is defined + with_dict: "{{ cikit_sniffers.coding_standards.list }}" + - name: Create HTMLCS binary template: src: htmlcs.j2 From b16a855532db1c40416d3dde072fa3934c9aef40 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Thu, 5 Jul 2018 14:05:46 +0100 Subject: [PATCH 03/15] #147: Fix usage of "version_compare" filter --- scripts/self-update.yml | 2 +- scripts/tasks/migrations/run.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/self-update.yml b/scripts/self-update.yml index dbbc3a4c..4d5dc504 100644 --- a/scripts/self-update.yml +++ b/scripts/self-update.yml @@ -53,4 +53,4 @@ # Run the migrations only when the version of CIKit has grown. - include_tasks: tasks/migrations/run.yml # A new version of CIKit is greater than current. - when: cikit_version | version_compare(cikit_current_version, 'gt') + when: cikit_version is version_compare(cikit_current_version, 'gt') diff --git a/scripts/tasks/migrations/run.yml b/scripts/tasks/migrations/run.yml index 2f947df1..57695256 100644 --- a/scripts/tasks/migrations/run.yml +++ b/scripts/tasks/migrations/run.yml @@ -13,4 +13,4 @@ with_items: "{{ cikit_migrations }}" # This file runs only if a new version of the package is greater than current. This # means we have to run only those tasks, which were added only after the current version. - when: "item | version_compare(cikit_current_version, 'gt')" + when: "item is version_compare(cikit_current_version, 'gt')" From a8b832b165c167f1bb569b73f2b1d1f7f40f4a63 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Thu, 5 Jul 2018 19:22:10 +0100 Subject: [PATCH 04/15] #147: Add unified CI builder for Jenkins and Gitlab CI --- cmf/all/.cikit/ci/server-cleaner.yml | 31 +++++ cmf/all/.cikit/config.yml | 6 + cmf/all/scripts/vars/main.yml | 4 - lib/platformsh/setup.yml | 1 + scripts/roles/cikit-ci/tasks/main.yml | 14 ++- .../roles/cikit-ci/templates/ci-builder.sh.j2 | 118 ++++++++++++++++++ scripts/roles/cikit-env/defaults/main.yml | 8 +- scripts/roles/cikit-jenkins/defaults/main.yml | 3 - .../templates/jobs/builder.xml.j2 | 45 +------ .../templates/jobs/pr_builder.xml.j2 | 67 ++-------- .../templates/jobs/server_cleaner.xml.j2 | 50 -------- 11 files changed, 183 insertions(+), 164 deletions(-) create mode 100644 cmf/all/.cikit/ci/server-cleaner.yml create mode 100755 scripts/roles/cikit-ci/templates/ci-builder.sh.j2 delete mode 100644 scripts/roles/cikit-jenkins/templates/jobs/server_cleaner.xml.j2 diff --git a/cmf/all/.cikit/ci/server-cleaner.yml b/cmf/all/.cikit/ci/server-cleaner.yml new file mode 100644 index 00000000..92abcc6c --- /dev/null +++ b/cmf/all/.cikit/ci/server-cleaner.yml @@ -0,0 +1,31 @@ +# This is a playbook that runs if CI server has less than 90% of a drive space +# available. +# +# Available variables: +# - rc: an exit code of the build process. If it's not "0" then a build failed; +# - dist: an absolute path to the project's destination; +# - env: a name of the environment being built; +# - site-url: a URL of a website being built; +# - build-id: an ID of CI build; +# - workspace: an absolute path to the sandbox directory on CI server. +--- +- hosts: localhost + gather_facts: no + connection: local + become: yes + + vars_files: + - ../config.yml + + vars: + mysql_query: "mysql -u{{ mysql.user }} -p{{ mysql.pass }} -se" + + tasks: + - name: Remove all builds + shell: "{{ item }}" + args: + warn: no + executable: bash + with_items: + - '{{ mysql_query }} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" {{ mysql_query }} "DROP DATABASE @@"' + - 'rm -rf {{ webroot }}/*{{ build_slug }}*' diff --git a/cmf/all/.cikit/config.yml b/cmf/all/.cikit/config.yml index 3ca88ac3..ffdb3efc 100644 --- a/cmf/all/.cikit/config.yml +++ b/cmf/all/.cikit/config.yml @@ -4,6 +4,12 @@ tmproot: /tmp webroot: /var/www # The directory to "cd" into after "vagrant ssh". Defaults to the value of "webroot". ssh_home: ~ +# The part of a directory/database name for CI builds. +build_slug: build +# Do not try changing these values. +mysql: + user: root + pass: root APPLICATION_CONFIG: ~ vm: ip: 192.168.56.132 diff --git a/cmf/all/scripts/vars/main.yml b/cmf/all/scripts/vars/main.yml index ded2e2ef..028aa04a 100644 --- a/cmf/all/scripts/vars/main.yml +++ b/cmf/all/scripts/vars/main.yml @@ -10,10 +10,6 @@ reinstall_mode: full env_vars: APP_ENV: "{{ env }}" -mysql: - user: root - pass: root - databases: default: name: "{{ cmf }}_{{ project | replace('-', '_') }}_{{ build_id | default(env) }}" diff --git a/lib/platformsh/setup.yml b/lib/platformsh/setup.yml index dfabcfb7..2c78f4dd 100644 --- a/lib/platformsh/setup.yml +++ b/lib/platformsh/setup.yml @@ -1,3 +1,4 @@ +# This playbook runs by Vagrantfile only. --- - hosts: localhost connection: local diff --git a/scripts/roles/cikit-ci/tasks/main.yml b/scripts/roles/cikit-ci/tasks/main.yml index d87f80ae..91a1afc8 100644 --- a/scripts/roles/cikit-ci/tasks/main.yml +++ b/scripts/roles/cikit-ci/tasks/main.yml @@ -1,5 +1,17 @@ --- +- name: Determine CI service + set_fact: + cikit_ci: "{{ 'gitlab-ci' if gitlab_ci_token | default or gitlab_ci_url | default else 'jenkins' }}" + - include_role: # Install Gitlab CI if either "--gitlab-ci-token" or "--gitlab-ci-url" # is passed. Use Jenkins otherwise. - name: "cikit-{{ 'gitlab-ci' if gitlab_ci_token | default or gitlab_ci_url | default else 'jenkins' }}" + name: "cikit-{{ cikit_ci }}" + +- name: Deploy CI builder + template: + src: ci-builder.sh.j2 + dest: /var/ci-builder.sh + mode: a+x + owner: "{{ jenkins_data.user if cikit_ci == 'jenkins' else 'root' }}" + group: "{{ jenkins_data.group if cikit_ci == 'jenkins' else 'root' }}" diff --git a/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 b/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 new file mode 100755 index 00000000..f96840d9 --- /dev/null +++ b/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +set -e +export PYTHONUNBUFFERED=1 + +# ------------------------------------------------------------------------------ +# The required configuration of a process. +# ------------------------------------------------------------------------------ + +declare -rA VARIABLES=( + [BUILD_NUMBER]="either \"stable\" or any custom value" + [BUILD_MODE]="either \"full\" or \"pull\"" + [BUILD_ENV]="the name of an environment to build or \"default\"" + [CIKIT_PROJECT_DIR]="the path to directory where repository clones to" + [CIKIT_PROJECT_HOSTNAME]="the hostname where the project can be accessed" + [RUN_SNIFFERS]="either \"yes\" or whatever" + [RUN_TESTS]="either \"yes\" or whatever" +) + +for VARIABLE in "${!VARIABLES[@]}"; do + if [ -z ${!VARIABLE+x} ]; then + echo "The \"$VARIABLE\" variable is missing! It's value must be ${VARIABLES[$VARIABLE]}." + exit 1 + fi +done + +# ------------------------------------------------------------------------------ +# Read CIKit configuration. +# ------------------------------------------------------------------------------ + +declare -A CIKIT_PROJECT_CONFIG=() + +for VARIABLE in webroot project build_slug; do + VALUE="$(awk '/'"$VARIABLE"':/ {print $2}' < "$CIKIT_PROJECT_DIR/.cikit/config.yml")" + + if [ -z "$VALUE" ]; then + echo "The value of \"$VARIABLE\" variable cannot be empty!" + exit 2 + fi + + CIKIT_PROJECT_CONFIG["$VARIABLE"]="$VALUE" +done + +# ------------------------------------------------------------------------------ +# Compute build parameters. +# ------------------------------------------------------------------------------ + +BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-" + +if [ "$BUILD_NUMBER" == "stable" ]; then + IS_COMMIT=false + BUILD_ID+="$BUILD_ENV" +else + IS_COMMIT=true + BUILD_ID+="${CIKIT_PROJECT_CONFIG['build_slug']}-$BUILD_NUMBER" +fi + +# Replace underscores by dashes in the ID of a build. +BUILD_ID="${BUILD_ID//_/-}" +# Form an absolute path to directory where the project is accessible from web. +DESTINATION="${CIKIT_PROJECT_CONFIG['webroot']}/$BUILD_ID" +PLAYBOOK_ARGS=( + "--site-url=https://$BUILD_ID.$CIKIT_PROJECT_HOSTNAME" + "--build-id=$BUILD_ID" + "--workspace=$CIKIT_PROJECT_DIR" +) + +# Pass the environment name to Ansible playbooks if it's not a default one. +if [ "$BUILD_ENV" != "default" ]; then + PLAYBOOK_ARGS+=("--env=$BUILD_ENV") +fi + +# ------------------------------------------------------------------------------ +# Define functions. +# ------------------------------------------------------------------------------ + +ci_hook() { + local HOOK_PLAYBOOK="$CIKIT_PROJECT_DIR/.cikit/ci/$1.yml" + + if [ -f "$HOOK_PLAYBOOK" ]; then + cikit "$HOOK_PLAYBOOK" "${PLAYBOOK_ARGS[@]}" --dist="$DESTINATION" --rc="$2" + fi +} + +handle_exit() { + ci_hook post-deploy $? + + # The drive space is occupied by more than 90%. + if [ $(df -H | head -2 | tail -1 | awk '{printf "%d", $5}') -gt 90 ]; then + ci_hook server-cleaner 0 + fi + + # The "$USER" must be either "jenkins" or "gitlab-runner". + sudo chown -R "$USER":"$USER" "$CIKIT_PROJECT_DIR" + sudo chown -R "$USER":"$USER" "$HOME" +} + +trap handle_exit EXIT +env +ci_hook pre-deploy 0 + +# Install a project. +if ${IS_COMMIT}; then + cikit reinstall "${PLAYBOOK_ARGS[@]}" --actions="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" +else + cikit reinstall "${PLAYBOOK_ARGS[@]}" --reinstall-mode="$BUILD_MODE" +fi + +# Copy codebase to directory accessible from the web. +sudo rsync -ra --delete --chown=www-data:www-data ./ "$DESTINATION/" + +if [ "$RUN_SNIFFERS" == "yes" ]; then + cikit sniffers "${PLAYBOOK_ARGS[@]}" +fi + +if [ "$RUN_TESTS" == "yes" ]; then + cikit tests "${PLAYBOOK_ARGS[@]}" --run --headless +fi diff --git a/scripts/roles/cikit-env/defaults/main.yml b/scripts/roles/cikit-env/defaults/main.yml index f5caaaf7..5e78d6ad 100644 --- a/scripts/roles/cikit-env/defaults/main.yml +++ b/scripts/roles/cikit-env/defaults/main.yml @@ -1,9 +1,7 @@ --- cikit_env: # The list of per-file variables to unconditionally add everywhere. - global: - /etc/environment: - CIKIT_PROJECT: "{{ project }}" + global: {} # The list of per-file variables to add to development environment only (VM). local: /etc/environment: @@ -12,6 +10,4 @@ cikit_env: CIKIT_PROJECT_URI: "{{ site_url }}" SSH_HOME: "{{ ssh_home if ssh_home | default else webroot }}" # The list of per-file variables for CI environment (remote CI server). - ci: - /etc/environment: - CIKIT_CI: true + ci: {} diff --git a/scripts/roles/cikit-jenkins/defaults/main.yml b/scripts/roles/cikit-jenkins/defaults/main.yml index 511566a2..d679fbf6 100644 --- a/scripts/roles/cikit-jenkins/defaults/main.yml +++ b/scripts/roles/cikit-jenkins/defaults/main.yml @@ -8,9 +8,6 @@ jenkins_data: user: jenkins group: jenkins jobs: - - name: SERVER_CLEANER - template: server_cleaner.xml.j2 - - name: "{{ jenkins_job }}_BUILDER" template: builder.xml.j2 diff --git a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 index eb532b9e..a2bad635 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 @@ -85,49 +85,12 @@ Pull - do not reinstall an application and just grab latest changes from reposit set -e -export PROJECT="$(echo "${JOB_NAME//_BUILDER/}" | tr '[:upper:]' '[:lower:]')" -export BUILD_NAME="${PROJECT}_${BUILD_ENV}" -export SITE_URL="https://$BUILD_NAME.$(php -r "echo parse_url('${JOB_URL}')['host'];")" -export DESTINATION="/var/www/$BUILD_NAME" -export CIKIT_PROJECT_DIR="$WORKSPACE" -export PYTHONUNBUFFERED=1 export ANSIBLE_VERBOSITY=2 +export BUILD_NUMBER="stable" +export CIKIT_PROJECT_DIR="$WORKSPACE" +export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")" -ARGS=( - "--env=$BUILD_ENV" - "--site-url=$SITE_URL" - "--build-id=$BUILD_NAME" - "--workspace=$WORKSPACE" -) - -deploy_hook() { - local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml" - - if [ -f "$HOOK_PLAYBOOK" ]; then - cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" --rc="$2" - fi -} - -handle_shutdown() { - deploy_hook post-deploy $? - sudo chown -R jenkins:jenkins "$WORKSPACE" -} - -trap handle_shutdown EXIT -deploy_hook pre-deploy -cikit reinstall "${ARGS[@]}" --reinstall-mode="$BUILD_MODE" - -# Copy codebase to directory, accessible from web. -sudo rsync --delete -ra ./ "$DESTINATION/" -sudo chown -R www-data:jenkins $_ - -if ${RUN_SNIFFERS}; then - cikit sniffers "${ARGS[@]}" -fi - -if ${RUN_TESTS}; then - cikit tests "${ARGS[@]}" --run --headless -fi +/var/ci-builder.sh diff --git a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 index 8a318c56..6693c16a 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 @@ -97,70 +97,19 @@ set -e -export PROJECT="$(echo "${JOB_NAME//_BUILDER/}" | tr '[:upper:]' '[:lower:]')" -export BUILD_NAME="${PROJECT}_build_${BUILD_NUMBER}" -export SITE_URL="https://$BUILD_NAME.$(php -r "echo parse_url('${JOB_URL}')['host'];")" -export DESTINATION="/var/www/$BUILD_NAME" -export CIKIT_PROJECT_DIR="$WORKSPACE" -export PYTHONUNBUFFERED=1 export ANSIBLE_VERBOSITY=2 -export BUILD_ACTIONS=$(php -r "echo json_encode(array_map('trim', array_filter( - explode(PHP_EOL, '$(git log --format=%B -n1 ${ghprbActualCommit} | awk -vRS="]" -vFS="[" '{print $2}')') -)));") - -ARGS=( - "--site-url=$SITE_URL" - "--build-id=$BUILD_NAME" - "--workspace=$WORKSPACE" -) - -deploy_hook() { - local HOOK_PLAYBOOK="$WORKSPACE/.cikit/ci/$1.yml" - - if [ -f "$HOOK_PLAYBOOK" ]; then - cikit "$HOOK_PLAYBOOK" "${ARGS[@]}" --dist="$DESTINATION" --rc="$2" - fi -} - -handle_shutdown() { - deploy_hook post-deploy $? - sudo chown -R jenkins:jenkins "$WORKSPACE" -} - -trap handle_shutdown EXIT -deploy_hook pre-deploy -cikit reinstall "${ARGS[@]}" --actions="$BUILD_ACTIONS" - -# Copy codebase to directory, accessible from web. -sudo rsync --delete -ra ./ "$DESTINATION/" -sudo chown -R www-data:jenkins $_ +export BUILD_MODE="full" +export BUILD_ENV="default" +export CIKIT_PROJECT_DIR="$WORKSPACE" +export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")" +export RUN_SNIFFERS="yes" +export RUN_TESTS="no" -cikit sniffers "${ARGS[@]}" +/var/ci-builder.sh - - - - - - - - - FORCE - false - - - - - SERVER_CLEANER - ALWAYS - false - false - - - - + xterm diff --git a/scripts/roles/cikit-jenkins/templates/jobs/server_cleaner.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/server_cleaner.xml.j2 deleted file mode 100644 index 6e40eae2..00000000 --- a/scripts/roles/cikit-jenkins/templates/jobs/server_cleaner.xml.j2 +++ /dev/null @@ -1,50 +0,0 @@ - - - - Remove all files and databases of currently existing builds. Not applicable for "demo", "stage" etc. - false - - - - - FORCE - Forcibly remove all builds files and databases. Otherwise data will be deleted only in case when device has less than 90% of free space. - true - - - - - - true - false - false - false - - - H 23 * * * - - - false - - - COMMAND="mysql -uroot -proot -se" - -# This build could be triggered by one of PR builders. Ansible might change the -# owner of some directories making further builds impossible. -sudo chown -R "${USER}":"${USER}" "${HOME}" - -if ! ${FORCE}; then - # Return "force" state if empty space on device is coming to the end. - FORCE="[ $(df -H | head -2 | tail -1 | awk '{printf "%d", $5}') -gt 90 ]" -fi - -if ${FORCE}; then - ${COMMAND} "SHOW DATABASES" | grep "build" | xargs -I "@@" ${COMMAND} "DROP DATABASE @@" - sudo rm -rf /var/www/*build* -fi - - - - - - From 9489147d20f9e1e2c36ab021cbd511a5823bf8f4 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Thu, 5 Jul 2018 21:19:55 +0100 Subject: [PATCH 05/15] #147: Modify Gitlab CI template --- .../all/scripts/vars/environments/demo.yml | 1 - .../roles/cikit-ci/templates/ci-builder.sh.j2 | 1 + scripts/roles/cikit-gitlab-ci/tasks/main.yml | 5 ++ .../templates/.gitlab-ci.yml.j2 | 70 ++++++++----------- 4 files changed, 35 insertions(+), 42 deletions(-) diff --git a/cmf/drupal/all/scripts/vars/environments/demo.yml b/cmf/drupal/all/scripts/vars/environments/demo.yml index b1461d70..b5621e3e 100644 --- a/cmf/drupal/all/scripts/vars/environments/demo.yml +++ b/cmf/drupal/all/scripts/vars/environments/demo.yml @@ -7,7 +7,6 @@ commands: # - cset: ["system.file", "path.temporary", "{{ tmproot }}"] # - cset: ["system.logging", "error_level", "none"] - en: ["dblog"] - - dis: ["update"] drupal: # Data for super-admin (UID 1). diff --git a/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 b/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 index f96840d9..4abc3155 100755 --- a/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 +++ b/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 @@ -92,6 +92,7 @@ handle_exit() { # The "$USER" must be either "jenkins" or "gitlab-runner". sudo chown -R "$USER":"$USER" "$CIKIT_PROJECT_DIR" + # The "$HOME" will be for the "$USER". sudo chown -R "$USER":"$USER" "$HOME" } diff --git a/scripts/roles/cikit-gitlab-ci/tasks/main.yml b/scripts/roles/cikit-gitlab-ci/tasks/main.yml index 57fcdbb6..4a93d0c5 100644 --- a/scripts/roles/cikit-gitlab-ci/tasks/main.yml +++ b/scripts/roles/cikit-gitlab-ci/tasks/main.yml @@ -26,6 +26,11 @@ --tls-cert-file='{{ ssl_folder }}/ssl.crt' \ --registration-token='{{ gitlab_ci_token }}' +- name: Add a user to nopasswd sudoers + lineinfile: + dest: /etc/sudoers + line: "gitlab-runner ALL=(ALL) NOPASSWD:ALL" + - name: Create the configuration in a project template: src: .gitlab-ci.yml.j2 diff --git a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 index 45bff27a..a22254c3 100644 --- a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 +++ b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 @@ -1,13 +1,9 @@ variables: - # The name of a project from the ".cikit/config.yml" (underscores - # are replaced by dashes). + # The value of a "project" variable from the ".cikit/config.yml" (underscores + # should be replaced by dashes). PROJECT: "{{ project | replace('_', '-') }}" - # A value of the "webroot" variable from the ".cikit/config.yml". - WEBROOT: "{{ webroot }}" - -after_script: - # Restore possibly broken permissions to prevent failures of next builds. - - sudo chown -R gitlab-runner:gitlab-runner ${CI_PROJECT_DIR} + # The value of a "build_slug" variable from the ".cikit/config.yml". + BUILD_SLUG: "{{ build_slug }}" # Consider changing "CI_BUILD_REF_SLUG" by the "CI_BUILD_ID" if you are # interested in creating a build per commit. The default behavior is to @@ -18,11 +14,10 @@ build/run: # IMPORTANT! Bear in mind that substitution of nested variables is # not working. There's a bug in Gitlab CI. # https://gitlab.com/gitlab-org/gitlab-ce/issues/27098 - url: "https://${PROJECT}-${CI_BUILD_REF_SLUG}.${CI_RUNNER_DESCRIPTION}" + url: "https://$PROJECT-$BUILD_SLUG-$CI_BUILD_REF_SLUG.$CI_RUNNER_DESCRIPTION" # IMPORTANT! Do not use slashes here! They'll be treated as directory # separators. - name: "${PROJECT}-${CI_BUILD_REF_SLUG}" - on_stop: build/stop + name: "$PROJECT-$BUILD_SLUG-$CI_BUILD_REF_SLUG" # Valid branch names: # - "feature/*" # - "bug/*" @@ -31,51 +26,44 @@ build/run: # - "dev" only: - /^(feature|bug|NOJIRA)\/.+?$/ - - master - dev variables: # Display useful verbosity. Max - 4, min - 1. ANSIBLE_VERBOSITY: 2 + BUILD_NUMBER: "$CI_BUILD_REF_SLUG" + BUILD_MODE: "full" + BUILD_ENV: "default" + CIKIT_PROJECT_DIR: "$CI_PROJECT_DIR" + CIKIT_PROJECT_HOSTNAME: "$CI_RUNNER_DESCRIPTION" + RUN_SNIFFERS: "yes" + RUN_TESTS: "no" script: # IMPORTANT: Multiline YAML declaration ("- |", or "- >") not working here. # https://gitlab.com/gitlab-org/gitlab-runner/issues/166#note_3805463 - - export DESTINATION="${WEBROOT}/${CI_ENVIRONMENT_NAME}" - # Seek for actions definitions in the commit message. - - export BUILD_ACTIONS="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" - # Dump all available variables for debugging purposes. - - env - # @todo Add "trap" to run "post-deploy" anyway. - # @todo Unify hooks triggers as it's done for Jenkins. - - [ -f "${CI_PROJECT_DIR}/.cikit/ci/pre-deploy.yml" ] && cikit "${CI_PROJECT_DIR}/.cikit/ci/pre-deploy.yml" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" --dist="${DESTINATION}" - # Reinstall a project. - - CIKIT_PROJECT_DIR="${CI_PROJECT_DIR}" cikit reinstall --actions="${BUILD_ACTIONS}" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" - # Copy codebase to the "web" directory. - - sudo rsync -ra --delete --chown=www-data:www-data ./ "${DESTINATION}" - # Run sniffers. - - CIKIT_PROJECT_DIR="${DESTINATION}" cikit sniffers --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${DESTINATION}" - - [ -f "${CI_PROJECT_DIR}/.cikit/ci/post-deploy.yml" ] && cikit "${CI_PROJECT_DIR}/.cikit/ci/post-deploy.yml" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" --dist="${DESTINATION}" + - /var/ci-builder.sh artifacts: paths: - artifacts/ expire_in: 1 day -build/stop: +build/demo: stage: deploy environment: - name: "${PROJECT}-${CI_BUILD_REF_SLUG}" - action: stop + url: "https://$PROJECT-$BUILD_ENV.$CI_RUNNER_DESCRIPTION" + name: "$PROJECT-$BUILD_ENV" only: - - /^(feature|bug|NOJIRA)\/.+?$/ - master - - dev variables: - # https://docs.gitlab.com/ee/ci/environments.html#stopping-an-environment - GIT_STRATEGY: none + ANSIBLE_VERBOSITY: 2 + BUILD_NUMBER: "stable" + BUILD_MODE: "full" + BUILD_ENV: "demo" + CIKIT_PROJECT_DIR: "$CI_PROJECT_DIR" + CIKIT_PROJECT_HOSTNAME: "$CI_RUNNER_DESCRIPTION" + RUN_SNIFFERS: "no" + RUN_TESTS: "no" script: - - export DESTINATION="${WEBROOT}/${CI_ENVIRONMENT_NAME}" - - env - - if [ -d "${DESTINATION}" ]; then - - drush sql-drop -r "${DESTINATION}/docroot" -y - - sudo rm -rf "${DESTINATION}" - - fi - when: manual + - /var/ci-builder.sh + artifacts: + paths: + - artifacts/ From 8dac814fc0e868c7b5583c7c30deec67a4b43508 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Fri, 6 Jul 2018 16:56:38 +0100 Subject: [PATCH 06/15] #147: Use Bash process controller for CI process --- .gitmodules | 3 + cmf/all/.cikit/ci/post-deploy.yml | 4 + cmf/all/.cikit/ci/server-cleaner.yml | 21 ++- cmf/all/scripts/vars/main.yml | 2 +- scripts/roles/cikit-ci/defaults/main.yml | 7 + .../cikit-ci/files/bash-process-controller | 1 + scripts/roles/cikit-ci/files/tasks.sh | 138 ++++++++++++++++++ scripts/roles/cikit-ci/tasks/main.yml | 43 ++++-- .../roles/cikit-ci/templates/ci-builder.sh.j2 | 119 --------------- .../templates/.gitlab-ci.yml.j2 | 10 +- .../templates/jobs/builder.xml.j2 | 2 +- .../templates/jobs/pr_builder.xml.j2 | 4 +- 12 files changed, 209 insertions(+), 145 deletions(-) create mode 100644 scripts/roles/cikit-ci/defaults/main.yml create mode 160000 scripts/roles/cikit-ci/files/bash-process-controller create mode 100644 scripts/roles/cikit-ci/files/tasks.sh delete mode 100755 scripts/roles/cikit-ci/templates/ci-builder.sh.j2 diff --git a/.gitmodules b/.gitmodules index edfa6350..07c43e70 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "matrix/roles/api/files/cikit-rest-api"] path = matrix/roles/api/files/cikit-rest-api url = https://github.com/BR0kEN-/cikit-rest-api.git +[submodule "scripts/roles/cikit-ci/files/bash-process-controller"] + path = scripts/roles/cikit-ci/files/bash-process-controller + url = https://github.com/BR0kEN-/bash-process-controller.git diff --git a/cmf/all/.cikit/ci/post-deploy.yml b/cmf/all/.cikit/ci/post-deploy.yml index 1536e5e0..ae448795 100644 --- a/cmf/all/.cikit/ci/post-deploy.yml +++ b/cmf/all/.cikit/ci/post-deploy.yml @@ -15,6 +15,10 @@ connection: local become: yes + vars_files: + - ../config.yml + - ../../scripts/vars/main.yml + tasks: - name: Move reports to the website directory shell: "cp -r {{ workspace }}/docroot/reports/ {{ dist }}/docroot" diff --git a/cmf/all/.cikit/ci/server-cleaner.yml b/cmf/all/.cikit/ci/server-cleaner.yml index 92abcc6c..c6315c10 100644 --- a/cmf/all/.cikit/ci/server-cleaner.yml +++ b/cmf/all/.cikit/ci/server-cleaner.yml @@ -16,16 +16,21 @@ vars_files: - ../config.yml - - vars: - mysql_query: "mysql -u{{ mysql.user }} -p{{ mysql.pass }} -se" + - ../../scripts/vars/main.yml tasks: - - name: Remove all builds - shell: "{{ item }}" + - name: Compute disk usage + shell: df -H | head -2 | tail -1 | awk '{printf "%d", $5}' + register: disk_usage + + - name: Remove builds + shell: |- + COMMAND="mysql -u{{ mysql.user }} -p{{ mysql.pass }} -se" + + ${COMMAND} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" ${COMMAND} "DROP DATABASE @@" + rm -rf {{ webroot }}/*{{ build_slug }}* args: warn: no executable: bash - with_items: - - '{{ mysql_query }} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" {{ mysql_query }} "DROP DATABASE @@"' - - 'rm -rf {{ webroot }}/*{{ build_slug }}*' + # The drive space is occupied by more than 90%. + when: disk_usage.stdout > 90 diff --git a/cmf/all/scripts/vars/main.yml b/cmf/all/scripts/vars/main.yml index 028aa04a..a52abdec 100644 --- a/cmf/all/scripts/vars/main.yml +++ b/cmf/all/scripts/vars/main.yml @@ -12,7 +12,7 @@ env_vars: databases: default: - name: "{{ cmf }}_{{ project | replace('-', '_') }}_{{ build_id | default(env) }}" + name: "{{ [cmf, build_id | default(project + '_' + env)] | join('_') | replace('-', '_') }}" # source: # # Database name on remote host. # db: "" diff --git a/scripts/roles/cikit-ci/defaults/main.yml b/scripts/roles/cikit-ci/defaults/main.yml new file mode 100644 index 00000000..ebd3f47f --- /dev/null +++ b/scripts/roles/cikit-ci/defaults/main.yml @@ -0,0 +1,7 @@ +--- +cikit_ci: + process_controller: + repo: https://github.com/BR0kEN-/bash-process-controller.git + dest: /var/ci/process-controller + # Either branch name or tag is also allowed. + version: HEAD diff --git a/scripts/roles/cikit-ci/files/bash-process-controller b/scripts/roles/cikit-ci/files/bash-process-controller new file mode 160000 index 00000000..4742774f --- /dev/null +++ b/scripts/roles/cikit-ci/files/bash-process-controller @@ -0,0 +1 @@ +Subproject commit 4742774f16db718c0296cdde64b7a0897fc5d31c diff --git a/scripts/roles/cikit-ci/files/tasks.sh b/scripts/roles/cikit-ci/files/tasks.sh new file mode 100644 index 00000000..231c8d30 --- /dev/null +++ b/scripts/roles/cikit-ci/files/tasks.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# Allows continuing the process regardless of an exit code at one of the stages. +set +e + +# ------------------------------------------------------------------------------ +# The required configuration of a process. +# ------------------------------------------------------------------------------ + +declare -rA VARIABLES=( + [BUILD_NUMBER]="either \"stable\" or any custom value, like \"54\" or \"my-pr\"" + [BUILD_MODE]="either \"full\", \"pull\" or whatever you have defined in \"scripts/tasks/reinstall/modes/*.yml\"" + [BUILD_ENV]="the name of an environment to build or \"default\"" + [CIKIT_PROJECT_DIR]="the path to directory where repository clones to" + [CIKIT_PROJECT_HOSTNAME]="the hostname where the project can be accessed" + [RUN_SNIFFERS]="either \"yes\" or whatever" + [RUN_TESTS]="either \"yes\" or whatever" +) + +for VARIABLE in "${!VARIABLES[@]}"; do + if [ -z ${!VARIABLE+x} ]; then + echo "The \"$VARIABLE\" variable is missing! It's value must be ${VARIABLES[$VARIABLE]}." + exit 101 + fi + + # Ensure the variable is available in the subshells. + # shellcheck disable=SC2163 + # https://github.com/koalaman/shellcheck/wiki/SC2163 + export "${VARIABLE}" +done + +# ------------------------------------------------------------------------------ +# Read CIKit configuration. +# ------------------------------------------------------------------------------ + +declare -A CIKIT_PROJECT_CONFIG=() + +for VARIABLE in webroot project build_slug; do + VALUE="$(awk '/^'"$VARIABLE"':/ {print $2}' < "$CIKIT_PROJECT_DIR/.cikit/config.yml")" + + if [ -z "$VALUE" ]; then + echo "The value of \"$VARIABLE\" variable cannot be empty!" + exit 102 + fi + + CIKIT_PROJECT_CONFIG["$VARIABLE"]="$VALUE" +done + +# ------------------------------------------------------------------------------ +# Compute build parameters. +# ------------------------------------------------------------------------------ + +if [ "$BUILD_NUMBER" == "stable" ]; then + export IS_COMMIT=false + BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-$BUILD_ENV" +else + export IS_COMMIT=true + BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-${CIKIT_PROJECT_CONFIG['build_slug']}-$BUILD_NUMBER" +fi + +# https://docs.python.org/2/using/cmdline.html#cmdoption-u +export PYTHONUNBUFFERED=1 +# Replace underscores by dashes in the ID of a build. +export BUILD_ID="${BUILD_ID//_/-}" +# Form an absolute path to directory where the project is accessible from web. +export DESTINATION="${CIKIT_PROJECT_CONFIG['webroot']}/$BUILD_ID" + +# Print the environment. +env + +# ------------------------------------------------------------------------------ +# Define the runtime. +# ------------------------------------------------------------------------------ + +cikit_run() { + cikit "$1" \ + "${@:2}" \ + --env="$BUILD_ENV" \ + --site-url="https://$BUILD_ID.$CIKIT_PROJECT_HOSTNAME" \ + --build-id="$BUILD_ID" \ + --workspace="$CIKIT_PROJECT_DIR" +} + +cikit_hook() { + local HOOK_PLAYBOOK="$CIKIT_PROJECT_DIR/.cikit/ci/$1.yml" + + if [ -f "$HOOK_PLAYBOOK" ]; then + cikit_run "$HOOK_PLAYBOOK" --dist="$DESTINATION" --rc="$PROCESS_EXIT_CODE" + fi +} + +export -f cikit_run cikit_hook + +# ------------------------------------------------------------------------------ +# Define the process. +# ------------------------------------------------------------------------------ + +PROCESS_pre() { + cikit_hook pre-deploy +} + +PROCESS_main() { + # Install a project. + if ${IS_COMMIT}; then + cikit_run reinstall --actions="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" + else + cikit_run reinstall --reinstall-mode="$BUILD_MODE" + fi + + # Copy codebase to directory accessible from the web. + sudo rsync -ra --delete --chown=www-data:www-data ./ "$DESTINATION/" + + if [ "$RUN_SNIFFERS" == "yes" ]; then + cikit_run sniffers + fi + + if [ "$RUN_TESTS" == "yes" ]; then + cikit_run tests --run --headless + fi +} + +PROCESS_post() { + cikit_hook post-deploy +} + +PROCESS_clean() { + cikit_hook server-cleaner +} + +PROCESS_finish() { + echo "Restore permissions for \"$USER\"." + # The "$USER" must be either "jenkins" or "gitlab-runner". + sudo chown -R "$USER":"$USER" "$CIKIT_PROJECT_DIR" + # The "$HOME" will be for the "$USER". + sudo chown -R "$USER":"$USER" "$HOME" +} + +export -f PROCESS_pre PROCESS_main PROCESS_post PROCESS_clean PROCESS_finish diff --git a/scripts/roles/cikit-ci/tasks/main.yml b/scripts/roles/cikit-ci/tasks/main.yml index 91a1afc8..30e78d1e 100644 --- a/scripts/roles/cikit-ci/tasks/main.yml +++ b/scripts/roles/cikit-ci/tasks/main.yml @@ -1,17 +1,38 @@ --- - name: Determine CI service set_fact: - cikit_ci: "{{ 'gitlab-ci' if gitlab_ci_token | default or gitlab_ci_url | default else 'jenkins' }}" - -- include_role: # Install Gitlab CI if either "--gitlab-ci-token" or "--gitlab-ci-url" # is passed. Use Jenkins otherwise. - name: "cikit-{{ cikit_ci }}" + cikit_ci_gitlab: "{{ gitlab_ci_token | default or gitlab_ci_url | default }}" + +- include_role: + name: "cikit-{{ 'gitlab-ci' if cikit_ci_gitlab else 'jenkins' }}" + +- name: Determine the group and owner of CI service + set_fact: + cikit_ci_owner: "{{ 'root' if cikit_ci_gitlab else jenkins_data.user }}" + cikit_ci_group: "{{ 'root' if cikit_ci_gitlab else jenkins_data.group }}" + +- name: Ensure the directory for CI scripts exist + file: + path: "{{ cikit_ci.process_controller.dest | dirname }}" + mode: 0755 + state: directory + owner: "{{ cikit_ci_owner }}" + group: "{{ cikit_ci_group }}" + register: cikit_ci_scripts_path + +- name: Install/update CI process controller + git: + repo: "{{ cikit_ci.process_controller.repo }}" + dest: "{{ cikit_ci.process_controller.dest }}" + force: yes + update: yes + version: "{{ cikit_ci.process_controller.version }}" -- name: Deploy CI builder - template: - src: ci-builder.sh.j2 - dest: /var/ci-builder.sh - mode: a+x - owner: "{{ jenkins_data.user if cikit_ci == 'jenkins' else 'root' }}" - group: "{{ jenkins_data.group if cikit_ci == 'jenkins' else 'root' }}" +- name: Deploy CI process handler + copy: + src: tasks.sh + dest: "{{ cikit_ci_scripts_path.path }}" + owner: "{{ cikit_ci_owner }}" + group: "{{ cikit_ci_group }}" diff --git a/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 b/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 deleted file mode 100755 index 4abc3155..00000000 --- a/scripts/roles/cikit-ci/templates/ci-builder.sh.j2 +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -set -e -export PYTHONUNBUFFERED=1 - -# ------------------------------------------------------------------------------ -# The required configuration of a process. -# ------------------------------------------------------------------------------ - -declare -rA VARIABLES=( - [BUILD_NUMBER]="either \"stable\" or any custom value" - [BUILD_MODE]="either \"full\" or \"pull\"" - [BUILD_ENV]="the name of an environment to build or \"default\"" - [CIKIT_PROJECT_DIR]="the path to directory where repository clones to" - [CIKIT_PROJECT_HOSTNAME]="the hostname where the project can be accessed" - [RUN_SNIFFERS]="either \"yes\" or whatever" - [RUN_TESTS]="either \"yes\" or whatever" -) - -for VARIABLE in "${!VARIABLES[@]}"; do - if [ -z ${!VARIABLE+x} ]; then - echo "The \"$VARIABLE\" variable is missing! It's value must be ${VARIABLES[$VARIABLE]}." - exit 1 - fi -done - -# ------------------------------------------------------------------------------ -# Read CIKit configuration. -# ------------------------------------------------------------------------------ - -declare -A CIKIT_PROJECT_CONFIG=() - -for VARIABLE in webroot project build_slug; do - VALUE="$(awk '/'"$VARIABLE"':/ {print $2}' < "$CIKIT_PROJECT_DIR/.cikit/config.yml")" - - if [ -z "$VALUE" ]; then - echo "The value of \"$VARIABLE\" variable cannot be empty!" - exit 2 - fi - - CIKIT_PROJECT_CONFIG["$VARIABLE"]="$VALUE" -done - -# ------------------------------------------------------------------------------ -# Compute build parameters. -# ------------------------------------------------------------------------------ - -BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-" - -if [ "$BUILD_NUMBER" == "stable" ]; then - IS_COMMIT=false - BUILD_ID+="$BUILD_ENV" -else - IS_COMMIT=true - BUILD_ID+="${CIKIT_PROJECT_CONFIG['build_slug']}-$BUILD_NUMBER" -fi - -# Replace underscores by dashes in the ID of a build. -BUILD_ID="${BUILD_ID//_/-}" -# Form an absolute path to directory where the project is accessible from web. -DESTINATION="${CIKIT_PROJECT_CONFIG['webroot']}/$BUILD_ID" -PLAYBOOK_ARGS=( - "--site-url=https://$BUILD_ID.$CIKIT_PROJECT_HOSTNAME" - "--build-id=$BUILD_ID" - "--workspace=$CIKIT_PROJECT_DIR" -) - -# Pass the environment name to Ansible playbooks if it's not a default one. -if [ "$BUILD_ENV" != "default" ]; then - PLAYBOOK_ARGS+=("--env=$BUILD_ENV") -fi - -# ------------------------------------------------------------------------------ -# Define functions. -# ------------------------------------------------------------------------------ - -ci_hook() { - local HOOK_PLAYBOOK="$CIKIT_PROJECT_DIR/.cikit/ci/$1.yml" - - if [ -f "$HOOK_PLAYBOOK" ]; then - cikit "$HOOK_PLAYBOOK" "${PLAYBOOK_ARGS[@]}" --dist="$DESTINATION" --rc="$2" - fi -} - -handle_exit() { - ci_hook post-deploy $? - - # The drive space is occupied by more than 90%. - if [ $(df -H | head -2 | tail -1 | awk '{printf "%d", $5}') -gt 90 ]; then - ci_hook server-cleaner 0 - fi - - # The "$USER" must be either "jenkins" or "gitlab-runner". - sudo chown -R "$USER":"$USER" "$CIKIT_PROJECT_DIR" - # The "$HOME" will be for the "$USER". - sudo chown -R "$USER":"$USER" "$HOME" -} - -trap handle_exit EXIT -env -ci_hook pre-deploy 0 - -# Install a project. -if ${IS_COMMIT}; then - cikit reinstall "${PLAYBOOK_ARGS[@]}" --actions="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" -else - cikit reinstall "${PLAYBOOK_ARGS[@]}" --reinstall-mode="$BUILD_MODE" -fi - -# Copy codebase to directory accessible from the web. -sudo rsync -ra --delete --chown=www-data:www-data ./ "$DESTINATION/" - -if [ "$RUN_SNIFFERS" == "yes" ]; then - cikit sniffers "${PLAYBOOK_ARGS[@]}" -fi - -if [ "$RUN_TESTS" == "yes" ]; then - cikit tests "${PLAYBOOK_ARGS[@]}" --run --headless -fi diff --git a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 index a22254c3..e04ad244 100644 --- a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 +++ b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 @@ -37,10 +37,12 @@ build/run: CIKIT_PROJECT_HOSTNAME: "$CI_RUNNER_DESCRIPTION" RUN_SNIFFERS: "yes" RUN_TESTS: "no" + # IMPORTANT: Multiline YAML declaration ("- |", or "- >") not working here. + # https://gitlab.com/gitlab-org/gitlab-runner/issues/166#note_3805463 script: - # IMPORTANT: Multiline YAML declaration ("- |", or "- >") not working here. - # https://gitlab.com/gitlab-org/gitlab-runner/issues/166#note_3805463 - - /var/ci-builder.sh + # The "CI_JOB_ID" is provided by Gitlab CI. + # See https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables + - /var/ci/process-controller/main.sh "/tmp/$CI_JOB_ID" "/var/ci/tasks.sh" artifacts: paths: - artifacts/ @@ -63,7 +65,7 @@ build/demo: RUN_SNIFFERS: "no" RUN_TESTS: "no" script: - - /var/ci-builder.sh + - /var/ci/process-controller/main.sh "/tmp/$CI_JOB_ID" "/var/ci/tasks.sh" artifacts: paths: - artifacts/ diff --git a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 index a2bad635..8613b16c 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 @@ -90,7 +90,7 @@ export BUILD_NUMBER="stable" export CIKIT_PROJECT_DIR="$WORKSPACE" export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")" -/var/ci-builder.sh +/var/ci/process-controller/main.sh "/tmp/$BUILD_TAG" "/var/ci/tasks.sh" diff --git a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 index 6693c16a..1db44575 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 @@ -105,7 +105,9 @@ export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")" export RUN_SNIFFERS="yes" export RUN_TESTS="no" -/var/ci-builder.sh +# The "BUILD_TAG" is provided by Jenkins. +# See https://wiki.jenkins.io/display/JENKINS/Building+a+software+project +/var/ci/process-controller/main.sh "/tmp/$BUILD_TAG" "/var/ci/tasks.sh" From aebeb7bdc4f458ab324a2228a86e6009138146c3 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Fri, 6 Jul 2018 17:21:29 +0100 Subject: [PATCH 07/15] #147: Allow server cleaner to remove databases with hyphens in name --- cmf/all/.cikit/ci/server-cleaner.yml | 2 +- scripts/roles/cikit-ci/files/tasks.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmf/all/.cikit/ci/server-cleaner.yml b/cmf/all/.cikit/ci/server-cleaner.yml index c6315c10..81818afa 100644 --- a/cmf/all/.cikit/ci/server-cleaner.yml +++ b/cmf/all/.cikit/ci/server-cleaner.yml @@ -27,7 +27,7 @@ shell: |- COMMAND="mysql -u{{ mysql.user }} -p{{ mysql.pass }} -se" - ${COMMAND} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" ${COMMAND} "DROP DATABASE @@" + ${COMMAND} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" ${COMMAND} "DROP DATABASE \`@@\`" rm -rf {{ webroot }}/*{{ build_slug }}* args: warn: no diff --git a/scripts/roles/cikit-ci/files/tasks.sh b/scripts/roles/cikit-ci/files/tasks.sh index 231c8d30..eaef04a9 100644 --- a/scripts/roles/cikit-ci/files/tasks.sh +++ b/scripts/roles/cikit-ci/files/tasks.sh @@ -86,6 +86,8 @@ cikit_hook() { if [ -f "$HOOK_PLAYBOOK" ]; then cikit_run "$HOOK_PLAYBOOK" --dist="$DESTINATION" --rc="$PROCESS_EXIT_CODE" + else + echo "==> There's no \"$HOOK_PLAYBOOK\" hook to run." fi } From 996fa1d4f07b2c6866c187d9224ff5598566c23c Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 8 Jul 2018 06:38:29 +0300 Subject: [PATCH 08/15] #147: Move tests runner to a separate project --- .gitmodules | 3 ++ .travis.yml | 2 +- .../2018-04-24-travis-tests-automation.md | 4 ++ tests/travis/runner.sh | 54 ------------------- tests/travis/tests-runner | 1 + 5 files changed, 9 insertions(+), 55 deletions(-) delete mode 100755 tests/travis/runner.sh create mode 160000 tests/travis/tests-runner diff --git a/.gitmodules b/.gitmodules index 07c43e70..b9d82d5b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "scripts/roles/cikit-ci/files/bash-process-controller"] path = scripts/roles/cikit-ci/files/bash-process-controller url = https://github.com/BR0kEN-/bash-process-controller.git +[submodule "tests/travis/tests-runner"] + path = tests/travis/tests-runner + url = https://github.com/BR0kEN-/tests-runner.git diff --git a/.travis.yml b/.travis.yml index 3b5a3a80..a27a7e22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ install: - sudo git checkout ${TRAVIS_BRANCH} script: - - sudo ./tests/travis/runner.sh + - sudo ./tests/travis/tests-runner/runner.sh - env notifications: diff --git a/docs/_posts/2018-04-24-travis-tests-automation.md b/docs/_posts/2018-04-24-travis-tests-automation.md index fe4db050..12354016 100644 --- a/docs/_posts/2018-04-24-travis-tests-automation.md +++ b/docs/_posts/2018-04-24-travis-tests-automation.md @@ -158,3 +158,7 @@ When skipping a test the extension of a file **MUST NOT** be specified. If file ## Check in action The [.travis.yml](https://github.com/BR0kEN-/cikit/blob/c37173b93d1eaee9b090fe4655cf6e5081122942/.travis.yml#L37) of CIKit uses exactly same [runner](https://github.com/BR0kEN-/cikit/blob/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/runner.sh#L1), which is responsible for launching [Bash](https://github.com/BR0kEN-/cikit/tree/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/bash) and [Python](https://github.com/BR0kEN-/cikit/tree/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/python) tests. + +## Project page + +Project codebase is available at [https://github.com/BR0kEN-/tests-runner](https://github.com/BR0kEN-/tests-runner). diff --git a/tests/travis/runner.sh b/tests/travis/runner.sh deleted file mode 100755 index 2ea755c5..00000000 --- a/tests/travis/runner.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Restrictions: -# - Will not work with BSD "awk" (i.e. on macOS) due to "awk: invalid -v option". -# -# Usage: -# - Run all tests. -# bash runner.sh -# -# - Run non-bash tests. -# TRAVIS_COMMIT_MESSAGE="[skip bash]" bash runner.sh -# -# - List available tests. -# bash runner.sh --list -# -# - List non-bash tests. -# TRAVIS_COMMIT_MESSAGE="[skip bash]" bash runner.sh --list - -cd ./tests/travis -declare -A TESTS=() -declare -r OPTION="$1" - -# Iterate all over subdirectories. -for INTERPRETER in [a-z]*/; do - EXTENSION="$INTERPRETER/.extension" - - # Assume tests are in a directory that has the ".extension" file. - if [ -f "$EXTENSION" ]; then - TESTS["${INTERPRETER%%/}"]="$(head -n1 "$EXTENSION")" - fi -done - -# Parse the commit message that looks like "#120: [skip bash/init][ skip python] Commit name". -# The resulting string will be: "|skipbash/init|skippython|" -if [ -v TRAVIS_COMMIT_MESSAGE ]; then - PARAMS="|$(awk -vRS="]" -vFS="[" '{print $2}' <<< "$TRAVIS_COMMIT_MESSAGE" | head -n -1 | tr '\n' '|' | tr -d '[:space:]')" -fi - -for INTERPRETER in "${!TESTS[@]}"; do - if [[ ! "$PARAMS" =~ \|skip$INTERPRETER\| ]]; then - for TEST in "$INTERPRETER"/[a-z]*."${TESTS[$INTERPRETER]}"; do - if [[ ! "$PARAMS" =~ \|skip$TEST\| ]]; then - if [ "--list" == "$OPTION" ]; then - echo "- $TEST" - else - echo "[$(date --iso-8601=seconds)] -- $TEST" - ${INTERPRETER} "$TEST" - fi - fi - done - fi -done diff --git a/tests/travis/tests-runner b/tests/travis/tests-runner new file mode 160000 index 00000000..00401281 --- /dev/null +++ b/tests/travis/tests-runner @@ -0,0 +1 @@ +Subproject commit 004012816c3daef7d684cecaa9c4a0aa3fea094a From 814eb71d4baab0018a46e632bd58af0115c373fb Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 8 Jul 2018 06:44:42 +0300 Subject: [PATCH 09/15] #147: Update tests runner --- .travis.yml | 2 +- tests/travis/tests-runner | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a27a7e22..433b53bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ install: - sudo git checkout ${TRAVIS_BRANCH} script: - - sudo ./tests/travis/tests-runner/runner.sh + - sudo ./tests/travis/tests-runner/runner.sh ./tests/travis - env notifications: diff --git a/tests/travis/tests-runner b/tests/travis/tests-runner index 00401281..e2f33d9b 160000 --- a/tests/travis/tests-runner +++ b/tests/travis/tests-runner @@ -1 +1 @@ -Subproject commit 004012816c3daef7d684cecaa9c4a0aa3fea094a +Subproject commit e2f33d9bf76dd20cce50e196cae6463a59712a9a From 5418b85a2e4bf47e4c6659e89ef8976ac4aa79fa Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 8 Jul 2018 08:15:10 +0300 Subject: [PATCH 10/15] #147: Update tests runner and CI process controller --- scripts/roles/cikit-ci/files/bash-process-controller | 2 +- tests/travis/tests-runner | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/roles/cikit-ci/files/bash-process-controller b/scripts/roles/cikit-ci/files/bash-process-controller index 4742774f..8023c1f6 160000 --- a/scripts/roles/cikit-ci/files/bash-process-controller +++ b/scripts/roles/cikit-ci/files/bash-process-controller @@ -1 +1 @@ -Subproject commit 4742774f16db718c0296cdde64b7a0897fc5d31c +Subproject commit 8023c1f60e4f118b7164aa96316821f558f96b55 diff --git a/tests/travis/tests-runner b/tests/travis/tests-runner index e2f33d9b..0646b8b6 160000 --- a/tests/travis/tests-runner +++ b/tests/travis/tests-runner @@ -1 +1 @@ -Subproject commit e2f33d9bf76dd20cce50e196cae6463a59712a9a +Subproject commit 0646b8b62a81f37e2f8351f5c518d344a93c9375 From 997330ccd15c593e6b192219022df7f4d5325ad3 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 8 Jul 2018 08:53:12 +0300 Subject: [PATCH 11/15] #147: Allow installing specific version of CIKit --- .travis.yml | 10 +++++----- install.sh | 27 +++++++++++++++++---------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 433b53bb..0353a3e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,12 +25,12 @@ env: install: - sudo pip install ansible - - sudo bash ./install.sh --no-requirements-check - - sudo wget -nv https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_x86_64.deb - - sudo dpkg -i vagrant_${VAGRANT_VERSION}_x86_64.deb + - sudo bash ./install.sh --no-requirements-check "${TRAVIS_BRANCH}" + - sudo wget -nv "https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_x86_64.deb" + - sudo dpkg -i "vagrant_${VAGRANT_VERSION}_x86_64.deb" - cd /usr/local/share/cikit - - sudo git fetch origin ${TRAVIS_BRANCH} - - sudo git checkout ${TRAVIS_BRANCH} + - sudo git fetch origin "${TRAVIS_BRANCH}" + - sudo git checkout "${TRAVIS_BRANCH}" script: - sudo ./tests/travis/tests-runner/runner.sh ./tests/travis diff --git a/install.sh b/install.sh index a8dbd012..e2e32876 100755 --- a/install.sh +++ b/install.sh @@ -1,27 +1,34 @@ #!/usr/bin/env bash INSTALL_PATH="/usr/local/share/cikit" +NOCHECK="--no-requirements-check" +ARGS="$*" -if [ "--no-requirements-check" != "${1}" ]; then +if [[ "$ARGS" =~ .*$NOCHECK.* ]]; then MISSING="" for COMMAND in vagrant VBoxManage ansible-playbook; do - if ! \command -v "${COMMAND}" > /dev/null; then - MISSING+="\n- ${COMMAND}" + if ! \command -v "$COMMAND" > /dev/null; then + MISSING+="\n- $COMMAND" fi done - if [ -n "${MISSING}" ]; then - \echo -e "The following software were not found on your machine, so continuation is not possible:${MISSING}" + if [ -n "$MISSING" ]; then + \echo -e "The following software were not found on your machine, so continuation is not possible:$MISSING" \exit 1 fi fi -if [ ! -d "${INSTALL_PATH}" ]; then - sudo \mkdir -p "${INSTALL_PATH}" +ARGS="${ARGS//$NOCHECK/}" +VERSION="$(\tr -d '[:blank:]' <<< "$ARGS")" + +: "${VERSION:="master"}" + +if [ ! -d "$INSTALL_PATH" ]; then + sudo \mkdir -p "$INSTALL_PATH" fi -if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive "${INSTALL_PATH}"; then - sudo \ln -s "${INSTALL_PATH}/lib/cikit" /usr/local/bin/cikit - sudo \chown -R "$(\whoami)" "${INSTALL_PATH}" +if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive --version="$VERSION" "$INSTALL_PATH"; then + sudo \ln -s "$INSTALL_PATH/lib/cikit" /usr/local/bin/cikit + sudo \chown -R "$(\whoami)" "$INSTALL_PATH" fi From b944a66fdb388cf72579338a218f349cd518acda Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 8 Jul 2018 11:41:37 +0300 Subject: [PATCH 12/15] #147: Fix installation without requirements --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index e2e32876..4b3e1a6e 100755 --- a/install.sh +++ b/install.sh @@ -4,7 +4,7 @@ INSTALL_PATH="/usr/local/share/cikit" NOCHECK="--no-requirements-check" ARGS="$*" -if [[ "$ARGS" =~ .*$NOCHECK.* ]]; then +if [[ ! "$ARGS" =~ .*$NOCHECK.* ]]; then MISSING="" for COMMAND in vagrant VBoxManage ansible-playbook; do From dff467af45655b4eaf5156fa8ef693e83f572696 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 8 Jul 2018 11:44:01 +0300 Subject: [PATCH 13/15] #147: Fix installation without requirements --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 4b3e1a6e..aac32ebf 100755 --- a/install.sh +++ b/install.sh @@ -28,7 +28,7 @@ if [ ! -d "$INSTALL_PATH" ]; then sudo \mkdir -p "$INSTALL_PATH" fi -if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive --version="$VERSION" "$INSTALL_PATH"; then +if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive --branch="$VERSION" "$INSTALL_PATH"; then sudo \ln -s "$INSTALL_PATH/lib/cikit" /usr/local/bin/cikit sudo \chown -R "$(\whoami)" "$INSTALL_PATH" fi From 671621206ade57389dc5e18040cc1195937ff07e Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sun, 8 Jul 2018 20:06:31 +0300 Subject: [PATCH 14/15] #147: Bump a new version --- lib/.version | 2 +- scripts/roles/cikit-ci/files/tasks.sh | 2 +- scripts/tasks/migrations/1.0.3.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/.version b/lib/.version index 21e8796a..ee90284c 100644 --- a/lib/.version +++ b/lib/.version @@ -1 +1 @@ -1.0.3 +1.0.4 diff --git a/scripts/roles/cikit-ci/files/tasks.sh b/scripts/roles/cikit-ci/files/tasks.sh index eaef04a9..422c137e 100644 --- a/scripts/roles/cikit-ci/files/tasks.sh +++ b/scripts/roles/cikit-ci/files/tasks.sh @@ -19,7 +19,7 @@ declare -rA VARIABLES=( for VARIABLE in "${!VARIABLES[@]}"; do if [ -z ${!VARIABLE+x} ]; then - echo "The \"$VARIABLE\" variable is missing! It's value must be ${VARIABLES[$VARIABLE]}." + echo "The \"$VARIABLE\" variable is missing! Its value must be ${VARIABLES[$VARIABLE]}." exit 101 fi diff --git a/scripts/tasks/migrations/1.0.3.yml b/scripts/tasks/migrations/1.0.3.yml index cda4f91f..e59f813b 100644 --- a/scripts/tasks/migrations/1.0.3.yml +++ b/scripts/tasks/migrations/1.0.3.yml @@ -2,8 +2,8 @@ - name: Seeking CIKit projects shell: |- vagrant global-status | awk '{print $5}' | while read -r PROJECT_PATH; do - if [ -d "${PROJECT_PATH}/.cikit/" ]; then - echo ${PROJECT_PATH} + if [ -d "$PROJECT_PATH/.cikit" ]; then + echo "$PROJECT_PATH" fi done register: cikit_projects_list From 305640e3ebabd25dbc7e5fd522c2778e4e0bb137 Mon Sep 17 00:00:00 2001 From: Sergii Bondarenko Date: Sat, 14 Jul 2018 18:00:48 +0300 Subject: [PATCH 15/15] #147: Add the "PHPCompatibility" coding standards --- scripts/roles/cikit-sniffers/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/roles/cikit-sniffers/defaults/main.yml b/scripts/roles/cikit-sniffers/defaults/main.yml index 43e6f5c7..bef79241 100644 --- a/scripts/roles/cikit-sniffers/defaults/main.yml +++ b/scripts/roles/cikit-sniffers/defaults/main.yml @@ -19,6 +19,9 @@ cikit_sniffers: Security: version: master repo: https://github.com/FloeDesignTechnologies/phpcs-security-audit.git + PHPCompatibility: + version: 8.1.0 + repo: https://github.com/wimg/PHPCompatibility.git Symfony2: version: master repo: https://github.com/podarok/Symfony2-coding-standard.git