Skip to content

Commit

Permalink
Merge pull request eXist-db#5091 from reinhapa/backport-build
Browse files Browse the repository at this point in the history
[6.x.x] Backport build scripts
  • Loading branch information
adamretter authored Nov 6, 2023
2 parents 672fcf3 + 9ed64ee commit c5c1a0d
Show file tree
Hide file tree
Showing 11 changed files with 340 additions and 86 deletions.
89 changes: 89 additions & 0 deletions .github/actions/install-mvnd/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
name: 'install-mvnd'
description: 'Install the Maven Daemon'
inputs:
version:
description: 'The version of the Maven Daemon to install'
required: true
default: '0.9.0'
file-version-suffix:
description: 'A suffix to append to the version of the download file of Maven Daemon to install'
required: false
default: ''
install-path:
description: 'The folder in which Maven Daemon will be installed as a sub-folder'
required: true
default: '/tmp'
cache:
description: 'Set to true to cache Maven Daemon artifacts per-platform-architecture'
required: true
default: 'true'
mvnd-connect-timeout:
description: 'The timeout (as a duration, e.g. `90 seconds`) for connecting to the Maven Daemon'
required: true
default: '90 seconds'
outputs:
mvnd-dir:
description: "The directory where the command mvnd is located"
value: ${{ steps.mvnd-location.outputs.mvnd-dir }}
runs:
using: "composite"
steps:
- name: Determine mvnd platform and architecture
shell: bash
run: |
if [ "$RUNNER_OS" == "Linux" ]; then
MVND_PLATFORM="linux"
elif [ "$RUNNER_OS" == "macOS" ]; then
MVND_PLATFORM="darwin"
elif [ "$RUNNER_OS" == "Windows" ]; then
MVND_PLATFORM="windows"
else
"echo Unknown platform: $RUNNER_OS"
exit 1
fi
MVND_ARCHITECTURE="amd64"
echo "MVND_PLATFORM=${MVND_PLATFORM}" >> $GITHUB_ENV
echo "MVND_ARCHITECTURE=${MVND_ARCHITECTURE}" >> $GITHUB_ENV
echo "MVND_NAME=maven-mvnd-${{ inputs.version }}${{ inputs.file-version-suffix }}-${MVND_PLATFORM}-${MVND_ARCHITECTURE}" >> $GITHUB_ENV
- name: Cache mvnd
if: inputs.cache == 'true'
id: cache-mvnd
uses: actions/cache@v3
with:
path: |
${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip
${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256
${{ inputs.install-path }}/${{ env.MVND_NAME }}
key: setup-${{ env.MVND_NAME }}
- name: Download mvnd
if: steps.cache-mvnd.outputs.cache-hit != 'true'
shell: bash
run: |
curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip
curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256 https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip.sha256
- name: Install sha256sum (macOS)
if: ${{ runner.os == 'macOS' }}
shell: bash
run: brew install coreutils
- name: Verify mvnd sha256 checksum
shell: bash
run: echo "$(cat ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256) ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip" | sha256sum --check
- name: Unzip mvnd
if: steps.cache-mvnd.outputs.cache-hit != 'true'
shell: bash
run: unzip ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip -d ${{ inputs.install-path }}/
- name: Show Maven Daemon version
shell: bash
run: |
${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin/mvnd -D'mvnd.connectTimeout=${{ inputs.mvnd-connect-timeout }}' --version
${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin/mvnd -D'mvnd.connectTimeout=${{ inputs.mvnd-connect-timeout }}' --status
- name: Set mvnd-dir
id: mvnd-location
shell: bash
run: |
MVND_BIN_DIR="${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin"
if [ "$RUNNER_OS" == "Windows" ]; then
MVND_BIN_DIR="$(cygpath --absolute --long-name --windows $MVND_BIN_DIR)"
fi
echo "MVND_BIN_DIR=${MVND_BIN_DIR}" >> $GITHUB_ENV
echo "mvnd-dir=${MVND_BIN_DIR}" >> $GITHUB_OUTPUT
23 changes: 15 additions & 8 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
version: 2
updates:
- package-ecosystem: maven

- package-ecosystem: "github-actions"
target-branch: "develop-6.x.x"
directory: "/"
schedule:
interval: daily
interval: "weekly"
- package-ecosystem: "maven"
target-branch: "develop-6.x.x"
directory: "/"
schedule:
interval: "daily"
time: "03:00"
open-pull-requests-limit: 10
ignore:
Expand Down Expand Up @@ -36,23 +43,23 @@ updates:
- "> 1.9.4, < 1.10"
- dependency-name: net.sf.saxon:Saxon-HE
versions:
> "9"
- "> 9"
- dependency-name: org.hsqldb:hsqldb
versions:
- 2.6.0
- "2.6.0"
- dependency-name: com.ibm.icu:icu4j
versions:
- "69.1"
- dependency-name: com.sun.mail:jakarta.mail
versions:
- 2.0.1
- "2.0.1"
- dependency-name: com.sun.activation:jakarta.activation
versions:
- 2.0.1
- "2.0.1"
- dependency-name: com.mycila:license-maven-plugin
versions:
- "4.0"
- 4.0.rc2
- "4.0.rc2"
- dependency-name: org.owasp:dependency-check-maven
versions:
- 6.1.3
- "6.1.3"
16 changes: 8 additions & 8 deletions .github/workflows/ci-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,29 +7,29 @@ jobs:
# NOTE (DP): Publish on develop and master, test on PRs against these
if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master' || github.base_ref == 'develop' || github.base_ref == 'master'
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Set up JDK 8
uses: actions/setup-java@v2
uses: actions/setup-java@v3
with:
distribution: temurin
java-version: '8'
- name: Make buildkit default
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v3
id: buildx
with:
install: true
- name: Cache Maven packages
uses: actions/cache@v2
uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
key: deploy-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: deploy-${{ runner.os }}-maven
- name: Install bats
run: sudo apt-get install bats
- name: Build images
run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true clean package
run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true -P !mac-dmg-on-unix,!installer,!concurrency-stress-tests,!micro-benchmarks,skip-build-dist-archives clean package
- name: Check local images
run: docker image ls
- name: Check license headers
Expand Down Expand Up @@ -66,6 +66,6 @@ jobs:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
# DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
# run: mvn -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push
# run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push
# working-directory: ./exist-docker

31 changes: 0 additions & 31 deletions .github/workflows/ci-docs.yml

This file was deleted.

73 changes: 54 additions & 19 deletions .github/workflows/ci-test.yml
Original file line number Diff line number Diff line change
@@ -1,45 +1,80 @@
name: Test
name: Test & documentation
on: [push, pull_request]
permissions:
contents: read
env:
MAVEN_OPTS: -XX:StartFlightRecording=maxsize=5g,disk=true,dumponexit=true,settings=default,filename=./ -DtrimStackTrace=false -D'maven.resolver.transport=wagon'
DEV_JDK: '8'
jobs:
license:
name: License check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: temurin
java-version: ${{ env.DEV_JDK }}
cache: 'maven'
- run: mvn -V -B license:check
timeout-minutes: 60
dependencies:
name: Dependency checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: temurin
java-version: ${{ env.DEV_JDK }}
cache: 'maven'
- run: mvn -V -B dependency-check:check
timeout-minutes: 60
test:
name: (JDK ${{ matrix.jdk }} / ${{ matrix.os }}) Test
env:
MAVEN_OPTS: -XX:+IgnoreUnrecognizedVMOptions --illegal-access=debug -XX:StartFlightRecording=maxsize=5g,disk=true,dumponexit=true,settings=default,filename=./
name: ${{ matrix.os }} Test
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
jdk: ['8', '11']
include:
- os: ubuntu-latest
jdk: '16'
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v2
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: temurin
java-version: ${{ matrix.jdk }}
java-version: ${{ env.DEV_JDK }}
cache: 'maven'
- name: Install Maven Daemon
id: install-mvnd
uses: ./.github/actions/install-mvnd
with:
version: '1.0-m7'
file-version-suffix: '-m39'
cache: 'true'
- name: Maven Build
timeout-minutes: 10
run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -T 1C compile test-compile -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip'
- name: Maven Test
timeout-minutes: 60
run: mvn -V -B -DtrimStackTrace=false clean verify
continue-on-error: ${{ matrix.jdk == '16' }}
- name: Maven Code Coverage
if: ${{ github.ref == 'refs/heads/develop' && matrix.jdk == '8' && matrix.os == 'ubuntu-latest' }}
run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B verify -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip' -D'mvnd.maxLostKeepAlive=6000'
- name: Javadoc (Linux only)
if: ${{ matrix.os == 'ubuntu-latest' }}
run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -q -T 1C install javadoc:javadoc -DskipTests -D'dependency-check.skip' -D'license.skip' --projects '!exist-distribution,!exist-installer' --also-make
- name: Maven Code Coverage (Develop branch on Linux only)
if: ${{ github.ref == 'refs/heads/develop' && matrix.os == 'ubuntu-latest' }}
env:
CI_NAME: github
BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }}
CI_BUILD_NUMBER: ${{ github.run_id }}
CI_BUILD_URL: https://github.com/${{ github.repository }}/commit/${{ github.event.after }}/checks
COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
run: mvn -V -B jacoco:report coveralls:report
run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B jacoco:report coveralls:report
- name: Archive build logs
if: always()
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: ${{ runner.os }}-jdk${{ matrix.jdk }}-build-logs
name: ${{ runner.os }}-build-logs
retention-days: 5
path: |
**/*.jfr
Expand Down
46 changes: 40 additions & 6 deletions .github/workflows/ci-xqts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,58 @@ jobs:
name: W3C XQuery Test Suite
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
- name: Set up JDK 8
uses: actions/setup-java@v2
uses: actions/setup-java@v3
with:
distribution: temurin
java-version: '8'
cache: 'maven'
- name: Cache Maven packages
uses: actions/cache@v3
with:
path: ~/.m2
key: xqts-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: xqts-${{ runner.os }}-maven
- name: Maven XQTS Build
run: mvn -V -B clean package -DskipTests -Ddependency-check.skip=true --projects exist-xqts --also-make
- name: Run XQTS
timeout-minutes: 60
env:
JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication
run: find exist-xqts/target -name exist-xqts-runner.sh -exec {} --xqts-version HEAD --output-dir /tmp/xqts-output \;
JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication -Xmx6g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -XX:+ExitOnOutOfMemoryError
run: find exist-xqts/target -name exist-xqts-runner.sh -exec {} --xqts-version HEAD --output-dir /tmp/xqts-output --exclude-test-case RangeExpr-411d,RangeExpr-409d,RangeExpr-408d,RangeExpr-409c,RangeExpr-408c,GenCompEq-21 \;
- name: Check for HeapDump
id: check_heapdump
uses: andstor/file-existence-action@v2
with:
files: "/tmp/*.hprof"
- name: Compress HeapDump
if: steps.check_heapdump.outputs.files_exists == 'true'
run: zstd --rm -9 --progress -T0 /tmp/*.hprof
- name: Attach HeapDump artifact
if: steps.check_heapdump.outputs.files_exists == 'true'
uses: actions/upload-artifact@v3
with:
name: exist-xqts-runner-hprof
retention-days: 1
path: /tmp/*.hprof.zst
- name: Archive XQTS Logs
if: always()
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: xqts-logs
retention-days: 14
path: /tmp/xqts-output
- name: Get Previous XQTS Logs Artifacts JSON
run: 'curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/exist-db/exist/actions/artifacts?name=xqts-logs > /tmp/previous-xqts-logs-artifacts.json'
- name: Extract Previous XQTS Logs Artifact JSON
run: cat /tmp/previous-xqts-logs-artifacts.json | jq -r "[.artifacts[] | select(.workflow_run.head_branch == \"develop\")][1].archive_download_url" > /tmp/previous-xqts-logs-artifact.json
- name: Get Previous XQTS Logs Artifact
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: 'cat /tmp/previous-xqts-logs-artifact.json | xargs curl -H "Authorization: Bearer ${GITHUB_TOKEN}" --location --output /tmp/previous-xqts-output.zip'
- name: Extract Previous XQTS Logs Artifact
run: mkdir /tmp/previous-xqts-output && unzip /tmp/previous-xqts-output.zip -d /tmp/previous-xqts-output
- name: Compare Previous and Current XQTS Logs
run: java -jar ~/.m2/repository/net/sf/saxon/Saxon-HE/9.9.1-8/Saxon-HE-9.9.1-8.jar -xsl:exist-xqts/src/main/xslt/compare-results.xslt -it:compare-results -o:/tmp/comparison-results.xml xqts.previous.junit-data-path=/tmp/previous-xqts-output/junit/data xqts.current.junit-data-path=/tmp/xqts-output/junit/data
- name: Show Comparison Results
run: cat /tmp/comparison-results.xml
Loading

0 comments on commit c5c1a0d

Please sign in to comment.