Skip to content
Tests

Storage migrations #5111

Sign in to view logs

Storage migrations

Storage migrations #5111

Workflow file for this run

.github/workflows/main.yml at d861c09
name: Tests
on:
pull_request:
push:
branches:
- staging
- trying
- master
jobs:
linting:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/lint.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Get yarn cache
id: yarn_cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: ${{ steps.yarn_cache.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- uses: actions/cache@v2
with:
path: node_modules
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set up Python
uses: actions/setup-python@v1
with:
python-version: '3.11'
- name: Set up Node
uses: actions/setup-node@v1
with:
node-version: '16'
- name: Install
run: |
python -m pip install --upgrade pip
pip install -r requirements/docs.txt -r requirements/lint.txt
pip check
yarn install --frozen-lockfile
- name: doc8
run: |
doc8 ESSArch_Core/docs
- name: flake8
run: |
flake8
- name: isort
run: |
isort --check-only --diff .
- name: prettier
run: |
yarn prettier --check "ESSArch_Core/frontend/static/frontend/**/*.{ts,js,scss,html}"
- name: stylelint
run: |
yarn stylelint
build-frontend:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get yarn cache
id: yarn_cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: ${{ steps.yarn_cache.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- uses: actions/cache@v2
with:
path: node_modules
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set up Node
uses: actions/setup-node@v1
with:
node-version: '16'
- name: Build
run: |
yarn install --frozen-lockfile
yarn build:prod
build-docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/lint.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Set up Python
uses: actions/setup-python@v1
with:
python-version: '3.11'
- name: Set PYTHONPATH
run: |
echo "PYTHONPATH=/home/runner/work/ESSArch/ESSArch/config" >> $GITHUB_ENV
echo "ESSARCH_DIR=/home/runner/work/ESSArch/ESSArch" >> $GITHUB_ENV
- name: Install
run: |
mkdir -p log
mkdir -p config
python -m pip install --upgrade pip wheel setuptools
git config user.email test
git config user.name test
t="0+`git rev-parse --short HEAD`"
git tag -a $t -m $t
pip install -e .[docs]
essarch settings generate -qp config/local_essarch_settings.py
- name: Build
run: |
cd ESSArch_Core/docs
make html SPHINXOPTS="-W"
build-docker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Build
run: |
cd docker
docker-compose build essarch
test-windows:
runs-on: windows-latest
env:
DATABASE_URL_ESSARCH: 'mssql://SA:MyPassword42@localhost:1433/essarch?isolation_level=read committed&driver=ODBC Driver 17 for SQL Server'
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: ~\AppData\Local\pip\Cache
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/**.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Set up Python
uses: actions/setup-python@v1
with:
python-version: '3.11'
- name: Set PYTHONPATH
run: |
echo "PYTHONPATH=D:\a\ESSArch\ESSArch\config" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
echo "ESSARCH_DIR=D:\a\ESSArch\ESSArch" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
- name: Get yarn cache
id: yarn_cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: ${{ steps.yarn_cache.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- uses: actions/cache@v2
with:
path: node_modules
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set up Node
uses: actions/setup-node@v1
with:
node-version: '16'
- name: Install
run: |
powershell wget https://github.com/tschoonj/GTK-for-Windows-Runtime-Environment-Installer/releases/download/2021-04-29/gtk3-runtime-3.24.29-2021-04-29-ts-win64.exe -OutFile gtk3-runtime-3.24.29-2021-04-29-ts-win64.exe
powershell 'Start-Process -FilePath "gtk3-runtime-3.24.29-2021-04-29-ts-win64.exe" -Wait -PassThru -ArgumentList /S'
powershell wget https://download.microsoft.com/download/1/a/4/1a4a49b8-9fe6-4237-be0d-a6b8f2d559b5/en-US/18.0.1.1/x64/msodbcsql.msi -OutFile msodbcsql.msi
powershell "Start-Process msiexec.exe -PassThru -Wait -ArgumentList '/I msodbcsql.msi IACCEPTMSODBCSQLLICENSETERMS=YES ADDLOCAL=ALL /quiet /passive /qn /norestart'"
choco install sql-server-express -o -ia "'/IACCEPTSQLSERVERLICENSETERMS /Q /ACTION=install /INSTANCEID=MSSQLSERVER /INSTANCENAME=MSSQLSERVER /UPDATEENABLED=FALSE /TCPENABLED=1 /SECURITYMODE=SQL /SAPWD=MyPassword42'" -f -y
choco install imagemagick
choco install redis-64 --version 3.0.503
python -m pip install --upgrade pip wheel setuptools
git config user.email test
git config user.name test
$t=&git rev-parse --short HEAD
git tag -a "0+$t" -m "0+$t"
pip install .[mssql]
pip install -r requirements/tests.txt
pip check
mkdir config
mkdir log
essarch settings generate -qp config/local_essarch_settings.py
# Add geckodriver to PATH
echo "$env:GeckoWebDrive" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
# Make all PATH additions global
echo "C:\Program Files\GTK3-Runtime Win64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Build frontend
run: |
yarn install --frozen-lockfile
yarn build:prod
- name: Test
run: |
start redis-server
coverage run manage.py test --verbosity=2 --exclude-tag=requires-elasticsearch
- name: Report code coverage
run: |
coverage xml
- name: Codecov
uses: codecov/codecov-action@v3
with:
name: Windows
files: ./coverage.xml
flags: backend
test-linux:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
db: [sqlite, mariadb, mysql, postgres, mssql]
include:
- db: sqlite
database_url: 'sqlite:///db.sqlite'
image: busybox
port: 1234
health_cmd: true
- db: mariadb
database_url: 'mysql://root:password@127.0.0.1:13307/essarch?isolation_level=read committed'
image: busybox
port: 1234
health_cmd: true
- db: mysql
database_url: 'mysql://root:password@127.0.0.1:13306/essarch?isolation_level=read committed'
image: 'mysql:8.0.12'
port: 3306
health_cmd: mysqladmin ping --user=root -ppassword
- db: postgres
database_url: 'postgres://postgres:postgres@127.0.0.1:13306/essarch?isolation_level=1'
image: postgres
port: 5432
health_cmd: pg_isready
- db: mssql
database_url: 'mssql://SA:MyPassword42@127.0.0.1:13306/essarch?isolation_level=read committed&driver=ODBC Driver 17 for SQL Server'
image: mcr.microsoft.com/mssql/server:2019-latest
port: 1433
health_cmd: true
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
ports:
- 19200:9200
options: >-
-e="discovery.type=single-node"
--name="elasticsearch"
--health-cmd="curl http://localhost:9200/_cluster/health"
--health-interval=10s
--health-timeout=5s
--health-retries=10
redis:
image: redis
ports:
- 6379:6379
${{ matrix.db }}:
image: ${{ matrix.image }}
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: essarch
MYSQL_DATABASE: essarch
MYSQL_USER: arkiv
MYSQL_PASSWORD: password
MYSQL_ROOT_PASSWORD: password
MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
ACCEPT_EULA: Y
SA_PASSWORD: MyPassword42
ports:
- 13306:${{ matrix.port }}
options: >-
-it
--name="${{ matrix.db }}1"
--health-cmd="${{ matrix.health_cmd }}"
--health-interval 2s
--health-timeout 10s
--health-retries 10
steps:
- name: Start MariaDB
uses: getong/mariadb-action@v1.1
if: matrix.db == 'mariadb'
with:
host port: 13307
mysql database: essarch
mysql user: arkiv
mysql password: password
mysql root password: password
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/**.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.11'
- name: Set PYTHONPATH
run: |
echo "PYTHONPATH=/home/runner/work/ESSArch/ESSArch/config" >> $GITHUB_ENV
echo "ESSARCH_DIR=/home/runner/work/ESSArch/ESSArch" >> $GITHUB_ENV
- name: Get yarn cache
id: yarn_cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: ${{ steps.yarn_cache.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- uses: actions/cache@v2
with:
path: node_modules
key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set up Node
uses: actions/setup-node@v1
with:
node-version: '16'
- name: Setup apache http schemaserver
run: |
sudo apt-get update && sudo apt-get install apache2
sudo a2enmod speling
sudo wget https://files.essarch.org/dist/ESSArch_schema-3.1.2_linux.tar.gz
sudo mkdir /var/www/schema
sudo tar -x -v -f ESSArch_schema-3.1.2_linux.tar.gz -C /var/www/schema
sudo wget https://files.essarch.org/dist/httpd-schema.conf
sudo sed -e "s?\${ESSARCH_DIR}?\/var\/www?g" --in-place httpd-schema.conf
sudo mv httpd-schema.conf /etc/apache2/sites-enabled/
sudo echo "Adding entrys for locale schemaserver to /etc/hosts"
sudo echo "127.0.0.1 www.w3.org schema.arkivverket.no arkivverket.no www.loc.gov xml.ra.se www.arkivverket.no www.dasboard.eu xml.essarch.org" | sudo tee -a /etc/hosts
sudo service apache2 restart
- name: Install
run: |
docker exec elasticsearch elasticsearch-plugin install --batch ingest-attachment
docker restart elasticsearch
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
curl https://packages.microsoft.com/config/ubuntu/22.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
sudo apt-get update
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 g++ unixodbc-dev default-libmysqlclient-dev libpq-dev
python -m pip install --upgrade pip wheel setuptools
git config user.email test
git config user.name test
t="0+`git rev-parse --short HEAD`"
git tag -a $t -m $t
pip install .[mssql,mysql,postgres]
pip install -r requirements/tests.txt
pip check
mkdir config
mkdir log
essarch settings generate --overwrite -qp config/local_essarch_settings.py
wget https://github.com/mozilla/geckodriver/releases/download/v0.32.2/geckodriver-v0.32.2-linux64.tar.gz
tar -xvzf geckodriver*
sudo mv geckodriver /usr/local/bin
- name: Build frontend
run: |
yarn install --frozen-lockfile
yarn build:prod
- name: Test
env:
DATABASE_URL_ESSARCH: ${{ matrix.database_url }}
run: |
coverage run manage.py test --verbosity=2
- name: Report code coverage
run: |
coverage xml
- name: Codecov
uses: codecov/codecov-action@v3
with:
name: Linux (${{matrix.db}})
files: ./coverage.xml
flags: backend