From 32a8877f936ae0d334031a14b0e8bac86e8997f5 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 26 Aug 2021 11:49:28 -0400 Subject: [PATCH 1/3] CI: some tweaks to the steps; switch to miniconda action --- .github/workflows/tests.yml | 112 ++++++++++--------- scripts/start_mongo.sh | 22 +++- scripts/{start_docker.sh => start_sirepo.sh} | 17 +-- 3 files changed, 87 insertions(+), 64 deletions(-) rename scripts/{start_docker.sh => start_sirepo.sh} (84%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ff12ad1a..aceeb7cd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,75 +1,77 @@ -name : tests +name: tests on: push: pull_request: schedule: - - cron: '00 4 * * *' # daily at 4AM + - cron: '00 4 * * *' # daily at 4AM jobs: build: - + name: Test sirepo-bluesky with Python ${{ matrix.python-version }} runs-on: ubuntu-latest - strategy: - matrix: + matrix: python-version: [3.7, 3.8, 3.9] - fail-fast: false - steps: - - uses: actions/checkout@v2 + steps: + - name: Set env.REPOSITORY_NAME # just the repo, as opposed to org/repo + shell: bash -l {0} + run: | + export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} + echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - - name: start MongoDB - uses: supercharge/mongodb-github-action@1.6.0 + - name: Checkout the code + uses: actions/checkout@v2 - - name: start Sirepo-Bluesky container - run: | - set -vxeuo pipefail - . scripts/start_docker.sh -d - export SIREPO_DOCKER_CONTAINER_ID - echo "SIREPO_DOCKER_CONTAINER_ID=${SIREPO_DOCKER_CONTAINER_ID}" >> $GITHUB_ENV + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.6.0 - - name: is Sirepo-Bluesky running? - run: docker ps -a + - name: Start Sirepo Docker container + shell: bash -l {0} + run: | + # For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html. + set -vxeuo pipefail + . scripts/start_sirepo.sh -d + export SIREPO_DOCKER_CONTAINER_ID + echo "SIREPO_DOCKER_CONTAINER_ID=${SIREPO_DOCKER_CONTAINER_ID}" >> $GITHUB_ENV - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - name: Check Sirepo state + run: | + set -vxeuo pipefail + docker images + docker ps -a + docker logs ${SIREPO_DOCKER_CONTAINER_ID} - - name: Create conda environment - run: | - # For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html. - set -vxeuo pipefail - source $CONDA/etc/profile.d/conda.sh - conda create -n test -c conda-forge python=${{ matrix.python-version }} shadow3 srwpy + - name: Set up Python ${{ matrix.python-version }} with conda + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} + auto-update-conda: true + miniconda-version: "latest" + python-version: ${{ matrix.python-version }} - - name: Install - shell: bash -l {0} - run: | - # For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html. - set -vxeuo pipefail - source $CONDA/etc/profile.d/conda.sh - conda activate test - conda env list - pip install --upgrade pip wheel - pip install . - pip install -r requirements-dev.txt - pip list - conda list + - name: Install the package and its dependencies + shell: bash -l {0} + run: | + set -vxeuo pipefail + conda env list + conda install -c conda-forge shadow3 srwpy + pip install --upgrade pip wheel + pip install . + pip install -r requirements-dev.txt + pip list + conda list - - name: Test with pytest - shell: bash -l {0} - run: | - # For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html. - set -vxeuo pipefail - source $CONDA/etc/profile.d/conda.sh - conda activate test - pytest -s -vv sirepo_bluesky/tests - status=$? - echo "Sirepo Docker container id: ${SIREPO_DOCKER_CONTAINER_ID}" - docker logs ${SIREPO_DOCKER_CONTAINER_ID} - if [ $status -gt 0 ]; then - exit $status - fi + - name: Test with pytest + shell: bash -l {0} + run: | + set -vxeuo pipefail + pytest -s -vv + status=$? + echo "Sirepo Docker container id: ${SIREPO_DOCKER_CONTAINER_ID}" + docker logs ${SIREPO_DOCKER_CONTAINER_ID} + if [ $status -gt 0 ]; then + exit $status + fi diff --git a/scripts/start_mongo.sh b/scripts/start_mongo.sh index 204a3245..4fb4069a 100644 --- a/scripts/start_mongo.sh +++ b/scripts/start_mongo.sh @@ -1 +1,21 @@ -docker run -it --rm -p 27017:27017 mongo \ No newline at end of file +#!/bin/bash + +set -vxeuo pipefail + +error_msg="Specify '-it' or '-d' on the command line as a first argument." + +arg="${1:-}" + +if [ -z "${arg}" ]; then + echo "${error_msg}" + exit 1 +elif [ "${arg}" != "-it" -a "${arg}" != "-d" ]; then + echo "${error_msg} Specified argument: ${arg}" + exit 2 +fi + +docker_image="mongo" + +docker pull ${docker_image} +docker images +docker run ${arg} --rm -p 27017:27017 --name mongo ${docker_image} diff --git a/scripts/start_docker.sh b/scripts/start_sirepo.sh similarity index 84% rename from scripts/start_docker.sh rename to scripts/start_sirepo.sh index f423dd1e..34523aa1 100644 --- a/scripts/start_docker.sh +++ b/scripts/start_sirepo.sh @@ -1,14 +1,17 @@ #!/bin/bash # set -vxeuo pipefail +set -e error_msg="Specify '-it' or '-d' on the command line as a first argument." -if [ -z "$1" ]; then +arg="${1:-}" + +if [ -z "${arg}" ]; then echo "${error_msg}" exit 1 -elif [ "$1" != "-it" -a "$1" != "-d" ]; then - echo "${error_msg} Specified argument: ${1}" +elif [ "${arg}" != "-it" -a "${arg}" != "-d" ]; then + echo "${error_msg} Specified argument: ${arg}" exit 2 fi @@ -20,8 +23,6 @@ year=$(date +"%Y") today="${HOME}/tmp/data/${year}/${month}/${day}" -docker_image="radiasoft/sirepo:beta" - if [ -d "${today}" ]; then echo "Directory ${today} exists." else @@ -29,14 +30,14 @@ else mkdir -p "${today}" fi -# ls -l $HOME/tmp +docker_image="radiasoft/sirepo:beta" docker pull ${docker_image} docker images in_docker_cmd="mkdir -v -p /sirepo/ && cp -Rv /SIREPO_SRDB_ROOT/* /sirepo/ && sirepo service http" -cmd="docker run $1 --init --rm --name sirepo \ +cmd="docker run ${arg} --init --rm --name sirepo \ -e SIREPO_AUTH_METHODS=bluesky:guest \ -e SIREPO_AUTH_BLUESKY_SECRET=bluesky \ -e SIREPO_SRDB_ROOT=/sirepo \ @@ -46,7 +47,7 @@ cmd="docker run $1 --init --rm --name sirepo \ ${docker_image} bash -l -c \"${in_docker_cmd}\"" echo -e "Command to run:\n\n${cmd}\n" -if [ "$1" == "-d" ]; then +if [ "${arg}" == "-d" ]; then SIREPO_DOCKER_CONTAINER_ID=$(eval ${cmd}) export SIREPO_DOCKER_CONTAINER_ID echo "Container ID: ${SIREPO_DOCKER_CONTAINER_ID}" From 7573e036725d319e0de408475d1177bac8605f2a Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 26 Aug 2021 12:01:06 -0400 Subject: [PATCH 2/3] CI: do not check for unbound vars --- .github/workflows/tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aceeb7cd..19ed7e99 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -55,7 +55,12 @@ jobs: - name: Install the package and its dependencies shell: bash -l {0} run: | - set -vxeuo pipefail + # set -vxeuo pipefail + # Do not check for unbound variables (the '-u' flag) as it fails on + # conda deactivate command: + # /usr/share/miniconda3/envs/sirepo-bluesky-py3.9/etc/conda/deactivate.d/glib_deactivate.sh: + # line 1: GSETTINGS_SCHEMA_DIR_CONDA_BACKUP: unbound variable + set -vxeo pipefail conda env list conda install -c conda-forge shadow3 srwpy pip install --upgrade pip wheel From 658e493673fbee66b3ebe775f0ba6a779ae68a22 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 26 Aug 2021 12:56:08 -0400 Subject: [PATCH 3/3] CI: remove TravisCI config - not used anymore --- .travis.yml | 91 ----------------------------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d0cdb2eb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,91 +0,0 @@ -os: linux -dist: xenial -language: python -services: - - docker - - mongodb - - xvfb - -env: - global: - - MPLBACKEND: Qt5Agg - -cache: - directories: - - $HOME/.cache/pip - - $HOME/.ccache # https://github.com/travis-ci/travis-ci/issues/5853 - -jobs: - fast_finish: true - include: - - os: linux - python: 3.7 - - os: linux - python: 3.8 - # - os: linux - # python: 3.9 - - os: linux - python: 3.7 - env: - - USE_DOCKER: true - - os: linux - python: 3.8 - env: - - USE_DOCKER: true - # - os: linux - # python: 3.9 - # env: - # - USE_DOCKER: true - -before_install: - - | - set -e - if [ "$TRAVIS_OS_NAME" == "linux" ]; then - arch="Linux" - elif [ "$TRAVIS_OS_NAME" == "osx" ]; then - arch="MacOSX" - else - echo "Unknown arch $TRAVIS_OS_NAME" - exit 1 - fi - wget https://repo.continuum.io/miniconda/Miniconda3-latest-${arch}-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - ./miniconda.sh -b -p ~/mc - source ~/mc/etc/profile.d/conda.sh - conda update conda --yes - # export CONDARC=ci/condarc - -install: - - env | sort -u - - conda create -n testenv python=$TRAVIS_PYTHON_VERSION -y - - conda activate testenv - - conda install shadow3 -c nsls2forge -c defaults --override-channels -y - # Install this package and the packages listed in requirements.txt. - - pip install . - # Install extra requirements for running tests and building docs. - - pip install -r requirements-dev.txt - - pip install --upgrade numpy pip - - pip list - # Removed for testing entry point feature. - #- mkdir -p $HOME/.config/databroker/ && cp -v examples/local.yml $HOME/.config/databroker/local.yml - - | - if [ ! -z "$USE_DOCKER" ]; then - # echo ${DOCKERHUB_TOKEN} | docker login --username ${DOCKERHUB_USER} --password-stdin - docker pull radiasoft/sirepo:beta - fi - -script: - - flake8 # Enforce code style ('relaxed' line length limit is set in .flake8 config file). - - | - if [ ! -z "$USE_DOCKER" ]; then - container_id=$(docker run -d -t --init --rm --name sirepo -e SIREPO_AUTH_METHODS=bluesky:guest -e SIREPO_AUTH_BLUESKY_SECRET=bluesky -e SIREPO_SRDB_ROOT=/sirepo -e SIREPO_COOKIE_IS_SECURE=false -p 8000:8000 -v $PWD/sirepo_bluesky/tests/SIREPO_SRDB_ROOT:/SIREPO_SRDB_ROOT:ro,z radiasoft/sirepo:beta bash -l -c "mkdir -v -p /sirepo/ && cp -Rv /SIREPO_SRDB_ROOT/* /sirepo/ && sirepo service http") - pytest -vvvv -m "docker" # Run the tests and check for test coverage. - status=$? - docker logs $container_id - if [ $status -gt 0 ]; then - exit $status - fi - else - pytest -vvvv -m "not docker" # Run the tests and check for test coverage. - fi - - make -C docs html # Build the documentation.