diff --git a/.gitmodules b/.gitmodules index edfa6350..b9d82d5b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,9 @@ [submodule "matrix/roles/api/files/cikit-rest-api"] path = matrix/roles/api/files/cikit-rest-api url = https://github.com/BR0kEN-/cikit-rest-api.git +[submodule "scripts/roles/cikit-ci/files/bash-process-controller"] + path = scripts/roles/cikit-ci/files/bash-process-controller + url = https://github.com/BR0kEN-/bash-process-controller.git +[submodule "tests/travis/tests-runner"] + path = tests/travis/tests-runner + url = https://github.com/BR0kEN-/tests-runner.git diff --git a/.travis.yml b/.travis.yml index 3b5a3a80..0353a3e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,15 +25,15 @@ env: install: - sudo pip install ansible - - sudo bash ./install.sh --no-requirements-check - - sudo wget -nv https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_x86_64.deb - - sudo dpkg -i vagrant_${VAGRANT_VERSION}_x86_64.deb + - sudo bash ./install.sh --no-requirements-check "${TRAVIS_BRANCH}" + - sudo wget -nv "https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_x86_64.deb" + - sudo dpkg -i "vagrant_${VAGRANT_VERSION}_x86_64.deb" - cd /usr/local/share/cikit - - sudo git fetch origin ${TRAVIS_BRANCH} - - sudo git checkout ${TRAVIS_BRANCH} + - sudo git fetch origin "${TRAVIS_BRANCH}" + - sudo git checkout "${TRAVIS_BRANCH}" script: - - sudo ./tests/travis/runner.sh + - sudo ./tests/travis/tests-runner/runner.sh ./tests/travis - env notifications: diff --git a/cmf/all/.cikit/ci/post-deploy.yml b/cmf/all/.cikit/ci/post-deploy.yml new file mode 100644 index 00000000..ae448795 --- /dev/null +++ b/cmf/all/.cikit/ci/post-deploy.yml @@ -0,0 +1,24 @@ +# This is a playbook that runs after a successful build of the application. +# To do something before the build create the "pre-deploy.yml" that'll have +# the same set of input variables but will run early, before any other tasks. +# +# Available variables: +# - rc: an exit code of the build process. If it's not "0" then a build failed; +# - dist: an absolute path to the project's destination; +# - env: a name of the environment being built; +# - site-url: a URL of a website being built; +# - build-id: an ID of CI build; +# - workspace: an absolute path to the sandbox directory on CI server. +--- +- hosts: localhost + gather_facts: no + connection: local + become: yes + + vars_files: + - ../config.yml + - ../../scripts/vars/main.yml + + tasks: + - name: Move reports to the website directory + shell: "cp -r {{ workspace }}/docroot/reports/ {{ dist }}/docroot" diff --git a/cmf/all/.cikit/ci/server-cleaner.yml b/cmf/all/.cikit/ci/server-cleaner.yml new file mode 100644 index 00000000..81818afa --- /dev/null +++ b/cmf/all/.cikit/ci/server-cleaner.yml @@ -0,0 +1,36 @@ +# This is a playbook that runs if CI server has less than 90% of a drive space +# available. +# +# Available variables: +# - rc: an exit code of the build process. If it's not "0" then a build failed; +# - dist: an absolute path to the project's destination; +# - env: a name of the environment being built; +# - site-url: a URL of a website being built; +# - build-id: an ID of CI build; +# - workspace: an absolute path to the sandbox directory on CI server. +--- +- hosts: localhost + gather_facts: no + connection: local + become: yes + + vars_files: + - ../config.yml + - ../../scripts/vars/main.yml + + tasks: + - name: Compute disk usage + shell: df -H | head -2 | tail -1 | awk '{printf "%d", $5}' + register: disk_usage + + - name: Remove builds + shell: |- + COMMAND="mysql -u{{ mysql.user }} -p{{ mysql.pass }} -se" + + ${COMMAND} "SHOW DATABASES" | grep "{{ build_slug }}" | xargs -I "@@" ${COMMAND} "DROP DATABASE \`@@\`" + rm -rf {{ webroot }}/*{{ build_slug }}* + args: + warn: no + executable: bash + # The drive space is occupied by more than 90%. + when: disk_usage.stdout > 90 diff --git a/cmf/all/.cikit/config.yml b/cmf/all/.cikit/config.yml index 3ca88ac3..ffdb3efc 100644 --- a/cmf/all/.cikit/config.yml +++ b/cmf/all/.cikit/config.yml @@ -4,6 +4,12 @@ tmproot: /tmp webroot: /var/www # The directory to "cd" into after "vagrant ssh". Defaults to the value of "webroot". ssh_home: ~ +# The part of a directory/database name for CI builds. +build_slug: build +# Do not try changing these values. +mysql: + user: root + pass: root APPLICATION_CONFIG: ~ vm: ip: 192.168.56.132 diff --git a/cmf/all/scripts/vars/main.yml b/cmf/all/scripts/vars/main.yml index ded2e2ef..a52abdec 100644 --- a/cmf/all/scripts/vars/main.yml +++ b/cmf/all/scripts/vars/main.yml @@ -10,13 +10,9 @@ reinstall_mode: full env_vars: APP_ENV: "{{ env }}" -mysql: - user: root - pass: root - databases: default: - name: "{{ cmf }}_{{ project | replace('-', '_') }}_{{ build_id | default(env) }}" + name: "{{ [cmf, build_id | default(project + '_' + env)] | join('_') | replace('-', '_') }}" # source: # # Database name on remote host. # db: "" diff --git a/cmf/drupal/all/scripts/vars/environments/demo.yml b/cmf/drupal/all/scripts/vars/environments/demo.yml index b1461d70..b5621e3e 100644 --- a/cmf/drupal/all/scripts/vars/environments/demo.yml +++ b/cmf/drupal/all/scripts/vars/environments/demo.yml @@ -7,7 +7,6 @@ commands: # - cset: ["system.file", "path.temporary", "{{ tmproot }}"] # - cset: ["system.logging", "error_level", "none"] - en: ["dblog"] - - dis: ["update"] drupal: # Data for super-admin (UID 1). diff --git a/docs/_posts/2018-04-24-travis-tests-automation.md b/docs/_posts/2018-04-24-travis-tests-automation.md index fe4db050..12354016 100644 --- a/docs/_posts/2018-04-24-travis-tests-automation.md +++ b/docs/_posts/2018-04-24-travis-tests-automation.md @@ -158,3 +158,7 @@ When skipping a test the extension of a file **MUST NOT** be specified. If file ## Check in action The [.travis.yml](https://github.com/BR0kEN-/cikit/blob/c37173b93d1eaee9b090fe4655cf6e5081122942/.travis.yml#L37) of CIKit uses exactly same [runner](https://github.com/BR0kEN-/cikit/blob/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/runner.sh#L1), which is responsible for launching [Bash](https://github.com/BR0kEN-/cikit/tree/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/bash) and [Python](https://github.com/BR0kEN-/cikit/tree/c37173b93d1eaee9b090fe4655cf6e5081122942/tests/travis/python) tests. + +## Project page + +Project codebase is available at [https://github.com/BR0kEN-/tests-runner](https://github.com/BR0kEN-/tests-runner). diff --git a/install.sh b/install.sh index a8dbd012..aac32ebf 100755 --- a/install.sh +++ b/install.sh @@ -1,27 +1,34 @@ #!/usr/bin/env bash INSTALL_PATH="/usr/local/share/cikit" +NOCHECK="--no-requirements-check" +ARGS="$*" -if [ "--no-requirements-check" != "${1}" ]; then +if [[ ! "$ARGS" =~ .*$NOCHECK.* ]]; then MISSING="" for COMMAND in vagrant VBoxManage ansible-playbook; do - if ! \command -v "${COMMAND}" > /dev/null; then - MISSING+="\n- ${COMMAND}" + if ! \command -v "$COMMAND" > /dev/null; then + MISSING+="\n- $COMMAND" fi done - if [ -n "${MISSING}" ]; then - \echo -e "The following software were not found on your machine, so continuation is not possible:${MISSING}" + if [ -n "$MISSING" ]; then + \echo -e "The following software were not found on your machine, so continuation is not possible:$MISSING" \exit 1 fi fi -if [ ! -d "${INSTALL_PATH}" ]; then - sudo \mkdir -p "${INSTALL_PATH}" +ARGS="${ARGS//$NOCHECK/}" +VERSION="$(\tr -d '[:blank:]' <<< "$ARGS")" + +: "${VERSION:="master"}" + +if [ ! -d "$INSTALL_PATH" ]; then + sudo \mkdir -p "$INSTALL_PATH" fi -if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive "${INSTALL_PATH}"; then - sudo \ln -s "${INSTALL_PATH}/lib/cikit" /usr/local/bin/cikit - sudo \chown -R "$(\whoami)" "${INSTALL_PATH}" +if sudo \git clone https://github.com/BR0kEN-/cikit.git --recursive --branch="$VERSION" "$INSTALL_PATH"; then + sudo \ln -s "$INSTALL_PATH/lib/cikit" /usr/local/bin/cikit + sudo \chown -R "$(\whoami)" "$INSTALL_PATH" fi diff --git a/lib/.version b/lib/.version index 21e8796a..ee90284c 100644 --- a/lib/.version +++ b/lib/.version @@ -1 +1 @@ -1.0.3 +1.0.4 diff --git a/lib/platformsh/setup.yml b/lib/platformsh/setup.yml index dfabcfb7..2c78f4dd 100644 --- a/lib/platformsh/setup.yml +++ b/lib/platformsh/setup.yml @@ -1,3 +1,4 @@ +# This playbook runs by Vagrantfile only. --- - hosts: localhost connection: local diff --git a/scripts/roles/cikit-ci/defaults/main.yml b/scripts/roles/cikit-ci/defaults/main.yml new file mode 100644 index 00000000..ebd3f47f --- /dev/null +++ b/scripts/roles/cikit-ci/defaults/main.yml @@ -0,0 +1,7 @@ +--- +cikit_ci: + process_controller: + repo: https://github.com/BR0kEN-/bash-process-controller.git + dest: /var/ci/process-controller + # Either branch name or tag is also allowed. + version: HEAD diff --git a/scripts/roles/cikit-ci/files/bash-process-controller b/scripts/roles/cikit-ci/files/bash-process-controller new file mode 160000 index 00000000..8023c1f6 --- /dev/null +++ b/scripts/roles/cikit-ci/files/bash-process-controller @@ -0,0 +1 @@ +Subproject commit 8023c1f60e4f118b7164aa96316821f558f96b55 diff --git a/scripts/roles/cikit-ci/files/tasks.sh b/scripts/roles/cikit-ci/files/tasks.sh new file mode 100644 index 00000000..422c137e --- /dev/null +++ b/scripts/roles/cikit-ci/files/tasks.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +# Allows continuing the process regardless of an exit code at one of the stages. +set +e + +# ------------------------------------------------------------------------------ +# The required configuration of a process. +# ------------------------------------------------------------------------------ + +declare -rA VARIABLES=( + [BUILD_NUMBER]="either \"stable\" or any custom value, like \"54\" or \"my-pr\"" + [BUILD_MODE]="either \"full\", \"pull\" or whatever you have defined in \"scripts/tasks/reinstall/modes/*.yml\"" + [BUILD_ENV]="the name of an environment to build or \"default\"" + [CIKIT_PROJECT_DIR]="the path to directory where repository clones to" + [CIKIT_PROJECT_HOSTNAME]="the hostname where the project can be accessed" + [RUN_SNIFFERS]="either \"yes\" or whatever" + [RUN_TESTS]="either \"yes\" or whatever" +) + +for VARIABLE in "${!VARIABLES[@]}"; do + if [ -z ${!VARIABLE+x} ]; then + echo "The \"$VARIABLE\" variable is missing! Its value must be ${VARIABLES[$VARIABLE]}." + exit 101 + fi + + # Ensure the variable is available in the subshells. + # shellcheck disable=SC2163 + # https://github.com/koalaman/shellcheck/wiki/SC2163 + export "${VARIABLE}" +done + +# ------------------------------------------------------------------------------ +# Read CIKit configuration. +# ------------------------------------------------------------------------------ + +declare -A CIKIT_PROJECT_CONFIG=() + +for VARIABLE in webroot project build_slug; do + VALUE="$(awk '/^'"$VARIABLE"':/ {print $2}' < "$CIKIT_PROJECT_DIR/.cikit/config.yml")" + + if [ -z "$VALUE" ]; then + echo "The value of \"$VARIABLE\" variable cannot be empty!" + exit 102 + fi + + CIKIT_PROJECT_CONFIG["$VARIABLE"]="$VALUE" +done + +# ------------------------------------------------------------------------------ +# Compute build parameters. +# ------------------------------------------------------------------------------ + +if [ "$BUILD_NUMBER" == "stable" ]; then + export IS_COMMIT=false + BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-$BUILD_ENV" +else + export IS_COMMIT=true + BUILD_ID="${CIKIT_PROJECT_CONFIG['project']}-${CIKIT_PROJECT_CONFIG['build_slug']}-$BUILD_NUMBER" +fi + +# https://docs.python.org/2/using/cmdline.html#cmdoption-u +export PYTHONUNBUFFERED=1 +# Replace underscores by dashes in the ID of a build. +export BUILD_ID="${BUILD_ID//_/-}" +# Form an absolute path to directory where the project is accessible from web. +export DESTINATION="${CIKIT_PROJECT_CONFIG['webroot']}/$BUILD_ID" + +# Print the environment. +env + +# ------------------------------------------------------------------------------ +# Define the runtime. +# ------------------------------------------------------------------------------ + +cikit_run() { + cikit "$1" \ + "${@:2}" \ + --env="$BUILD_ENV" \ + --site-url="https://$BUILD_ID.$CIKIT_PROJECT_HOSTNAME" \ + --build-id="$BUILD_ID" \ + --workspace="$CIKIT_PROJECT_DIR" +} + +cikit_hook() { + local HOOK_PLAYBOOK="$CIKIT_PROJECT_DIR/.cikit/ci/$1.yml" + + if [ -f "$HOOK_PLAYBOOK" ]; then + cikit_run "$HOOK_PLAYBOOK" --dist="$DESTINATION" --rc="$PROCESS_EXIT_CODE" + else + echo "==> There's no \"$HOOK_PLAYBOOK\" hook to run." + fi +} + +export -f cikit_run cikit_hook + +# ------------------------------------------------------------------------------ +# Define the process. +# ------------------------------------------------------------------------------ + +PROCESS_pre() { + cikit_hook pre-deploy +} + +PROCESS_main() { + # Install a project. + if ${IS_COMMIT}; then + cikit_run reinstall --actions="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" + else + cikit_run reinstall --reinstall-mode="$BUILD_MODE" + fi + + # Copy codebase to directory accessible from the web. + sudo rsync -ra --delete --chown=www-data:www-data ./ "$DESTINATION/" + + if [ "$RUN_SNIFFERS" == "yes" ]; then + cikit_run sniffers + fi + + if [ "$RUN_TESTS" == "yes" ]; then + cikit_run tests --run --headless + fi +} + +PROCESS_post() { + cikit_hook post-deploy +} + +PROCESS_clean() { + cikit_hook server-cleaner +} + +PROCESS_finish() { + echo "Restore permissions for \"$USER\"." + # The "$USER" must be either "jenkins" or "gitlab-runner". + sudo chown -R "$USER":"$USER" "$CIKIT_PROJECT_DIR" + # The "$HOME" will be for the "$USER". + sudo chown -R "$USER":"$USER" "$HOME" +} + +export -f PROCESS_pre PROCESS_main PROCESS_post PROCESS_clean PROCESS_finish diff --git a/scripts/roles/cikit-ci/tasks/main.yml b/scripts/roles/cikit-ci/tasks/main.yml index d87f80ae..30e78d1e 100644 --- a/scripts/roles/cikit-ci/tasks/main.yml +++ b/scripts/roles/cikit-ci/tasks/main.yml @@ -1,5 +1,38 @@ --- -- include_role: +- name: Determine CI service + set_fact: # Install Gitlab CI if either "--gitlab-ci-token" or "--gitlab-ci-url" # is passed. Use Jenkins otherwise. - name: "cikit-{{ 'gitlab-ci' if gitlab_ci_token | default or gitlab_ci_url | default else 'jenkins' }}" + cikit_ci_gitlab: "{{ gitlab_ci_token | default or gitlab_ci_url | default }}" + +- include_role: + name: "cikit-{{ 'gitlab-ci' if cikit_ci_gitlab else 'jenkins' }}" + +- name: Determine the group and owner of CI service + set_fact: + cikit_ci_owner: "{{ 'root' if cikit_ci_gitlab else jenkins_data.user }}" + cikit_ci_group: "{{ 'root' if cikit_ci_gitlab else jenkins_data.group }}" + +- name: Ensure the directory for CI scripts exist + file: + path: "{{ cikit_ci.process_controller.dest | dirname }}" + mode: 0755 + state: directory + owner: "{{ cikit_ci_owner }}" + group: "{{ cikit_ci_group }}" + register: cikit_ci_scripts_path + +- name: Install/update CI process controller + git: + repo: "{{ cikit_ci.process_controller.repo }}" + dest: "{{ cikit_ci.process_controller.dest }}" + force: yes + update: yes + version: "{{ cikit_ci.process_controller.version }}" + +- name: Deploy CI process handler + copy: + src: tasks.sh + dest: "{{ cikit_ci_scripts_path.path }}" + owner: "{{ cikit_ci_owner }}" + group: "{{ cikit_ci_group }}" diff --git a/scripts/roles/cikit-env/defaults/main.yml b/scripts/roles/cikit-env/defaults/main.yml index f5caaaf7..5e78d6ad 100644 --- a/scripts/roles/cikit-env/defaults/main.yml +++ b/scripts/roles/cikit-env/defaults/main.yml @@ -1,9 +1,7 @@ --- cikit_env: # The list of per-file variables to unconditionally add everywhere. - global: - /etc/environment: - CIKIT_PROJECT: "{{ project }}" + global: {} # The list of per-file variables to add to development environment only (VM). local: /etc/environment: @@ -12,6 +10,4 @@ cikit_env: CIKIT_PROJECT_URI: "{{ site_url }}" SSH_HOME: "{{ ssh_home if ssh_home | default else webroot }}" # The list of per-file variables for CI environment (remote CI server). - ci: - /etc/environment: - CIKIT_CI: true + ci: {} diff --git a/scripts/roles/cikit-gitlab-ci/tasks/main.yml b/scripts/roles/cikit-gitlab-ci/tasks/main.yml index 57fcdbb6..4a93d0c5 100644 --- a/scripts/roles/cikit-gitlab-ci/tasks/main.yml +++ b/scripts/roles/cikit-gitlab-ci/tasks/main.yml @@ -26,6 +26,11 @@ --tls-cert-file='{{ ssl_folder }}/ssl.crt' \ --registration-token='{{ gitlab_ci_token }}' +- name: Add a user to nopasswd sudoers + lineinfile: + dest: /etc/sudoers + line: "gitlab-runner ALL=(ALL) NOPASSWD:ALL" + - name: Create the configuration in a project template: src: .gitlab-ci.yml.j2 diff --git a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 index 3fc5e113..e04ad244 100644 --- a/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 +++ b/scripts/roles/cikit-gitlab-ci/templates/.gitlab-ci.yml.j2 @@ -1,13 +1,9 @@ variables: - # The name of a project from the ".cikit/config.yml" (underscores - # are replaced by dashes). + # The value of a "project" variable from the ".cikit/config.yml" (underscores + # should be replaced by dashes). PROJECT: "{{ project | replace('_', '-') }}" - # A value of the "webroot" variable from the ".cikit/config.yml". - WEBROOT: "{{ webroot }}" - -after_script: - # Restore possibly broken permissions to prevent failures of next builds. - - sudo chown -R gitlab-runner:gitlab-runner ${CI_PROJECT_DIR} + # The value of a "build_slug" variable from the ".cikit/config.yml". + BUILD_SLUG: "{{ build_slug }}" # Consider changing "CI_BUILD_REF_SLUG" by the "CI_BUILD_ID" if you are # interested in creating a build per commit. The default behavior is to @@ -18,11 +14,10 @@ build/run: # IMPORTANT! Bear in mind that substitution of nested variables is # not working. There's a bug in Gitlab CI. # https://gitlab.com/gitlab-org/gitlab-ce/issues/27098 - url: "https://${PROJECT}-${CI_BUILD_REF_SLUG}.${CI_RUNNER_DESCRIPTION}" + url: "https://$PROJECT-$BUILD_SLUG-$CI_BUILD_REF_SLUG.$CI_RUNNER_DESCRIPTION" # IMPORTANT! Do not use slashes here! They'll be treated as directory # separators. - name: "${PROJECT}-${CI_BUILD_REF_SLUG}" - on_stop: build/stop + name: "$PROJECT-$BUILD_SLUG-$CI_BUILD_REF_SLUG" # Valid branch names: # - "feature/*" # - "bug/*" @@ -31,49 +26,46 @@ build/run: # - "dev" only: - /^(feature|bug|NOJIRA)\/.+?$/ - - master - dev variables: # Display useful verbosity. Max - 4, min - 1. ANSIBLE_VERBOSITY: 2 + BUILD_NUMBER: "$CI_BUILD_REF_SLUG" + BUILD_MODE: "full" + BUILD_ENV: "default" + CIKIT_PROJECT_DIR: "$CI_PROJECT_DIR" + CIKIT_PROJECT_HOSTNAME: "$CI_RUNNER_DESCRIPTION" + RUN_SNIFFERS: "yes" + RUN_TESTS: "no" + # IMPORTANT: Multiline YAML declaration ("- |", or "- >") not working here. + # https://gitlab.com/gitlab-org/gitlab-runner/issues/166#note_3805463 script: - # IMPORTANT: Multiline YAML declaration ("- |", or "- >") not working here. - # https://gitlab.com/gitlab-org/gitlab-runner/issues/166#note_3805463 - - export DESTINATION="${WEBROOT}/${CI_ENVIRONMENT_NAME}" - # Seek for actions definitions in the commit message. - - export BUILD_ACTIONS="$(php -r "echo json_encode(array_map('trim', array_filter(explode(PHP_EOL, '$(git log -n1 --pretty=%B | awk -vRS="]" -vFS="[" '{print $2}')'))));")" - # Dump all available variables for debugging purposes. - - env - # Reinstall a project. - - CIKIT_PROJECT_DIR="${CI_PROJECT_DIR}" cikit reinstall --actions="${BUILD_ACTIONS}" --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${CI_PROJECT_DIR}" - # Copy codebase to the "web" directory. - - sudo rsync -ra --delete --chown=www-data:www-data ./ "${DESTINATION}" - # Run sniffers. - - CIKIT_PROJECT_DIR="${DESTINATION}" cikit sniffers --site-url="${CI_ENVIRONMENT_URL}" --build-id="${CI_ENVIRONMENT_NAME}" --workspace="${DESTINATION}" - # Copy artifacts. - - cp -r "${DESTINATION}/docroot/reports/" "${CI_PROJECT_DIR}/artifacts/" + # The "CI_JOB_ID" is provided by Gitlab CI. + # See https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables + - /var/ci/process-controller/main.sh "/tmp/$CI_JOB_ID" "/var/ci/tasks.sh" artifacts: paths: - artifacts/ expire_in: 1 day -build/stop: +build/demo: stage: deploy environment: - name: "${PROJECT}-${CI_BUILD_REF_SLUG}" - action: stop + url: "https://$PROJECT-$BUILD_ENV.$CI_RUNNER_DESCRIPTION" + name: "$PROJECT-$BUILD_ENV" only: - - /^(feature|bug|NOJIRA)\/.+?$/ - master - - dev variables: - # https://docs.gitlab.com/ee/ci/environments.html#stopping-an-environment - GIT_STRATEGY: none + ANSIBLE_VERBOSITY: 2 + BUILD_NUMBER: "stable" + BUILD_MODE: "full" + BUILD_ENV: "demo" + CIKIT_PROJECT_DIR: "$CI_PROJECT_DIR" + CIKIT_PROJECT_HOSTNAME: "$CI_RUNNER_DESCRIPTION" + RUN_SNIFFERS: "no" + RUN_TESTS: "no" script: - - export DESTINATION="${WEBROOT}/${CI_ENVIRONMENT_NAME}" - - env - - if [ -d "${DESTINATION}" ]; then - - drush sql-drop -r "${DESTINATION}/docroot" -y - - sudo rm -rf "${DESTINATION}" - - fi - when: manual + - /var/ci/process-controller/main.sh "/tmp/$CI_JOB_ID" "/var/ci/tasks.sh" + artifacts: + paths: + - artifacts/ diff --git a/scripts/roles/cikit-jenkins/defaults/main.yml b/scripts/roles/cikit-jenkins/defaults/main.yml index 511566a2..d679fbf6 100644 --- a/scripts/roles/cikit-jenkins/defaults/main.yml +++ b/scripts/roles/cikit-jenkins/defaults/main.yml @@ -8,9 +8,6 @@ jenkins_data: user: jenkins group: jenkins jobs: - - name: SERVER_CLEANER - template: server_cleaner.xml.j2 - - name: "{{ jenkins_job }}_BUILDER" template: builder.xml.j2 diff --git a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 index 71abca39..8613b16c 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/builder.xml.j2 @@ -81,48 +81,16 @@ Pull - do not reinstall an application and just grab latest changes from reposit false - #!/usr/bin/env bash -e + #!/usr/bin/env bash -export PYTHONUNBUFFERED=1 -export PROJECT="$(echo ${JOB_NAME//_BUILDER/} | tr '[:upper:]' '[:lower:]')" -export BUILD_NAME="${PROJECT}_${BUILD_ENV}" -export SITE_URL="https://${BUILD_NAME}.$(php -r "echo parse_url('${JOB_URL}')['host'];")" -export DESTINATION="/var/www/${BUILD_NAME}" -export CIKIT_PROJECT_DIR="${WORKSPACE}" -export ANSIBLE_VERBOSITY=2 - -cikit reinstall \ - --env="${BUILD_ENV}" \ - --site-url="${SITE_URL}" \ - --build-id="${BUILD_NAME}" \ - --workspace="${WORKSPACE}" \ - --reinstall-mode="${BUILD_MODE}" - -# Copy codebase to directory, accessible from web. -sudo rsync --delete -ra ./ "${DESTINATION}/" -sudo chown -R www-data:jenkins $_ +set -e -if ${RUN_SNIFFERS}; then - cikit sniffers \ - --env="${BUILD_ENV}" \ - --site-url="${SITE_URL}" \ - --build-id="${BUILD_NAME}" \ - --workspace="${WORKSPACE}" -fi - -if ${RUN_TESTS}; then - cikit tests \ - --run \ - --headless \ - --env=${BUILD_ENV} \ - --site-url=${SITE_URL} \ - --build-id=${BUILD_NAME} \ - --workspace=${WORKSPACE} -fi +export ANSIBLE_VERBOSITY=2 +export BUILD_NUMBER="stable" +export CIKIT_PROJECT_DIR="$WORKSPACE" +export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")" -# Move reports to the website directory. -sudo cp -r ./docroot/reports/ "${DESTINATION}/docroot/" -sudo chown -R jenkins:jenkins ./ +/var/ci/process-controller/main.sh "/tmp/$BUILD_TAG" "/var/ci/tasks.sh" diff --git a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 index f74e4f73..1db44575 100644 --- a/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 +++ b/scripts/roles/cikit-jenkins/templates/jobs/pr_builder.xml.j2 @@ -93,61 +93,25 @@ false - #!/usr/bin/env bash -e + #!/usr/bin/env bash -export PROJECT=$(echo ${JOB_NAME//_PR_BUILDER/} | tr '[:upper:]' '[:lower:]') -export BUILD_NAME="${PROJECT}_build_${BUILD_NUMBER}" -export SITE_URL="https://${BUILD_NAME}.$(php -r "echo parse_url('${JOB_URL}')['host'];")" -export DESTINATION="/var/www/${BUILD_NAME}" -export CIKIT_PROJECT_DIR="${WORKSPACE}" -export ANSIBLE_VERBOSITY=2 -export BUILD_ACTIONS=$(php -r "echo json_encode(array_map('trim', array_filter( - explode(PHP_EOL, '$(git log --format=%B -n1 ${ghprbActualCommit} | awk -vRS="]" -vFS="[" '{print $2}')') -)));") - -cikit reinstall \ - --actions="${BUILD_ACTIONS}" \ - --site-url="${SITE_URL}" \ - --build-id="${BUILD_NAME}" \ - --workspace="${WORKSPACE}" +set -e -# Copy codebase to directory, accessible from web. -sudo rsync --delete -ra ./ "${DESTINATION}/" -sudo chown -R www-data:jenkins $_ - -cikit sniffers \ - --site-url=${SITE_URL} \ - --build-id=${BUILD_NAME} \ - --workspace=${WORKSPACE} +export ANSIBLE_VERBOSITY=2 +export BUILD_MODE="full" +export BUILD_ENV="default" +export CIKIT_PROJECT_DIR="$WORKSPACE" +export CIKIT_PROJECT_HOSTNAME="$(php -r "echo parse_url('$JOB_URL')['host'];")" +export RUN_SNIFFERS="yes" +export RUN_TESTS="no" -# Move reports to the website directory. -sudo cp -r ./docroot/reports/ "${DESTINATION}/docroot/" -sudo chown -R jenkins:jenkins ./ +# The "BUILD_TAG" is provided by Jenkins. +# See https://wiki.jenkins.io/display/JENKINS/Building+a+software+project +/var/ci/process-controller/main.sh "/tmp/$BUILD_TAG" "/var/ci/tasks.sh" - - - - - - - - - FORCE - false - - - - - SERVER_CLEANER - ALWAYS - false - false - - - - + xterm diff --git a/scripts/roles/cikit-jenkins/templates/jobs/server_cleaner.xml.j2 b/scripts/roles/cikit-jenkins/templates/jobs/server_cleaner.xml.j2 deleted file mode 100644 index 6e40eae2..00000000 --- a/scripts/roles/cikit-jenkins/templates/jobs/server_cleaner.xml.j2 +++ /dev/null @@ -1,50 +0,0 @@ - - - - Remove all files and databases of currently existing builds. Not applicable for "demo", "stage" etc. - false - - - - - FORCE - Forcibly remove all builds files and databases. Otherwise data will be deleted only in case when device has less than 90% of free space. - true - - - - - - true - false - false - false - - - H 23 * * * - - - false - - - COMMAND="mysql -uroot -proot -se" - -# This build could be triggered by one of PR builders. Ansible might change the -# owner of some directories making further builds impossible. -sudo chown -R "${USER}":"${USER}" "${HOME}" - -if ! ${FORCE}; then - # Return "force" state if empty space on device is coming to the end. - FORCE="[ $(df -H | head -2 | tail -1 | awk '{printf "%d", $5}') -gt 90 ]" -fi - -if ${FORCE}; then - ${COMMAND} "SHOW DATABASES" | grep "build" | xargs -I "@@" ${COMMAND} "DROP DATABASE @@" - sudo rm -rf /var/www/*build* -fi - - - - - - diff --git a/scripts/roles/cikit-sniffers/defaults/main.yml b/scripts/roles/cikit-sniffers/defaults/main.yml index 2571dfb6..bef79241 100644 --- a/scripts/roles/cikit-sniffers/defaults/main.yml +++ b/scripts/roles/cikit-sniffers/defaults/main.yml @@ -12,12 +12,16 @@ cikit_sniffers: Drupal: version: 8.x-2.x repo: git://git.drupal.org/project/coder.git + build: composer install DrupalSecure: version: master repo: http://git.drupal.org/sandbox/coltrane/1921926.git Security: version: master repo: https://github.com/FloeDesignTechnologies/phpcs-security-audit.git + PHPCompatibility: + version: 8.1.0 + repo: https://github.com/wimg/PHPCompatibility.git Symfony2: version: master repo: https://github.com/podarok/Symfony2-coding-standard.git @@ -25,5 +29,6 @@ cikit_sniffers: version: master repo: https://github.com/WordPress-Coding-Standards/WordPress-Coding-Standards.git HTML: - version: 2.1.1 + version: 2.2.0 repo: https://github.com/squizlabs/HTML_CodeSniffer.git + build: npm install && ./node_modules/.bin/grunt build diff --git a/scripts/roles/cikit-sniffers/tasks/main.yml b/scripts/roles/cikit-sniffers/tasks/main.yml index 40dfe590..fe62d411 100644 --- a/scripts/roles/cikit-sniffers/tasks/main.yml +++ b/scripts/roles/cikit-sniffers/tasks/main.yml @@ -6,6 +6,13 @@ version: "{{ item.value.version }}" with_dict: "{{ cikit_sniffers.coding_standards.list }}" +- name: "Build '{{ item.key }}'" + shell: "{{ item.value.build }}" + args: + chdir: "{{ cikit_sniffers.coding_standards.dest }}/{{ item.key }}" + when: item.value.build is defined + with_dict: "{{ cikit_sniffers.coding_standards.list }}" + - name: Create HTMLCS binary template: src: htmlcs.j2 diff --git a/scripts/self-update.yml b/scripts/self-update.yml index dbbc3a4c..4d5dc504 100644 --- a/scripts/self-update.yml +++ b/scripts/self-update.yml @@ -53,4 +53,4 @@ # Run the migrations only when the version of CIKit has grown. - include_tasks: tasks/migrations/run.yml # A new version of CIKit is greater than current. - when: cikit_version | version_compare(cikit_current_version, 'gt') + when: cikit_version is version_compare(cikit_current_version, 'gt') diff --git a/scripts/tasks/migrations/1.0.3.yml b/scripts/tasks/migrations/1.0.3.yml index cda4f91f..e59f813b 100644 --- a/scripts/tasks/migrations/1.0.3.yml +++ b/scripts/tasks/migrations/1.0.3.yml @@ -2,8 +2,8 @@ - name: Seeking CIKit projects shell: |- vagrant global-status | awk '{print $5}' | while read -r PROJECT_PATH; do - if [ -d "${PROJECT_PATH}/.cikit/" ]; then - echo ${PROJECT_PATH} + if [ -d "$PROJECT_PATH/.cikit" ]; then + echo "$PROJECT_PATH" fi done register: cikit_projects_list diff --git a/scripts/tasks/migrations/run.yml b/scripts/tasks/migrations/run.yml index 2f947df1..57695256 100644 --- a/scripts/tasks/migrations/run.yml +++ b/scripts/tasks/migrations/run.yml @@ -13,4 +13,4 @@ with_items: "{{ cikit_migrations }}" # This file runs only if a new version of the package is greater than current. This # means we have to run only those tasks, which were added only after the current version. - when: "item | version_compare(cikit_current_version, 'gt')" + when: "item is version_compare(cikit_current_version, 'gt')" diff --git a/tests/travis/runner.sh b/tests/travis/runner.sh deleted file mode 100755 index 2ea755c5..00000000 --- a/tests/travis/runner.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Restrictions: -# - Will not work with BSD "awk" (i.e. on macOS) due to "awk: invalid -v option". -# -# Usage: -# - Run all tests. -# bash runner.sh -# -# - Run non-bash tests. -# TRAVIS_COMMIT_MESSAGE="[skip bash]" bash runner.sh -# -# - List available tests. -# bash runner.sh --list -# -# - List non-bash tests. -# TRAVIS_COMMIT_MESSAGE="[skip bash]" bash runner.sh --list - -cd ./tests/travis -declare -A TESTS=() -declare -r OPTION="$1" - -# Iterate all over subdirectories. -for INTERPRETER in [a-z]*/; do - EXTENSION="$INTERPRETER/.extension" - - # Assume tests are in a directory that has the ".extension" file. - if [ -f "$EXTENSION" ]; then - TESTS["${INTERPRETER%%/}"]="$(head -n1 "$EXTENSION")" - fi -done - -# Parse the commit message that looks like "#120: [skip bash/init][ skip python] Commit name". -# The resulting string will be: "|skipbash/init|skippython|" -if [ -v TRAVIS_COMMIT_MESSAGE ]; then - PARAMS="|$(awk -vRS="]" -vFS="[" '{print $2}' <<< "$TRAVIS_COMMIT_MESSAGE" | head -n -1 | tr '\n' '|' | tr -d '[:space:]')" -fi - -for INTERPRETER in "${!TESTS[@]}"; do - if [[ ! "$PARAMS" =~ \|skip$INTERPRETER\| ]]; then - for TEST in "$INTERPRETER"/[a-z]*."${TESTS[$INTERPRETER]}"; do - if [[ ! "$PARAMS" =~ \|skip$TEST\| ]]; then - if [ "--list" == "$OPTION" ]; then - echo "- $TEST" - else - echo "[$(date --iso-8601=seconds)] -- $TEST" - ${INTERPRETER} "$TEST" - fi - fi - done - fi -done diff --git a/tests/travis/tests-runner b/tests/travis/tests-runner new file mode 160000 index 00000000..0646b8b6 --- /dev/null +++ b/tests/travis/tests-runner @@ -0,0 +1 @@ +Subproject commit 0646b8b62a81f37e2f8351f5c518d344a93c9375