diff --git a/.github/actions/install-mvnd/action.yml b/.github/actions/install-mvnd/action.yml new file mode 100644 index 00000000000..9769860c218 --- /dev/null +++ b/.github/actions/install-mvnd/action.yml @@ -0,0 +1,89 @@ +name: 'install-mvnd' +description: 'Install the Maven Daemon' +inputs: + version: + description: 'The version of the Maven Daemon to install' + required: true + default: '0.9.0' + file-version-suffix: + description: 'A suffix to append to the version of the download file of Maven Daemon to install' + required: false + default: '' + install-path: + description: 'The folder in which Maven Daemon will be installed as a sub-folder' + required: true + default: '/tmp' + cache: + description: 'Set to true to cache Maven Daemon artifacts per-platform-architecture' + required: true + default: 'true' + mvnd-connect-timeout: + description: 'The timeout (as a duration, e.g. `90 seconds`) for connecting to the Maven Daemon' + required: true + default: '90 seconds' +outputs: + mvnd-dir: + description: "The directory where the command mvnd is located" + value: ${{ steps.mvnd-location.outputs.mvnd-dir }} +runs: + using: "composite" + steps: + - name: Determine mvnd platform and architecture + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + MVND_PLATFORM="linux" + elif [ "$RUNNER_OS" == "macOS" ]; then + MVND_PLATFORM="darwin" + elif [ "$RUNNER_OS" == "Windows" ]; then + MVND_PLATFORM="windows" + else + "echo Unknown platform: $RUNNER_OS" + exit 1 + fi + MVND_ARCHITECTURE="amd64" + echo "MVND_PLATFORM=${MVND_PLATFORM}" >> $GITHUB_ENV + echo "MVND_ARCHITECTURE=${MVND_ARCHITECTURE}" >> $GITHUB_ENV + echo "MVND_NAME=maven-mvnd-${{ inputs.version }}${{ inputs.file-version-suffix }}-${MVND_PLATFORM}-${MVND_ARCHITECTURE}" >> $GITHUB_ENV + - name: Cache mvnd + if: inputs.cache == 'true' + id: cache-mvnd + uses: actions/cache@v3 + with: + path: | + ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip + ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256 + ${{ inputs.install-path }}/${{ env.MVND_NAME }} + key: setup-${{ env.MVND_NAME }} + - name: Download mvnd + if: steps.cache-mvnd.outputs.cache-hit != 'true' + shell: bash + run: | + curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip + curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256 https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip.sha256 + - name: Install sha256sum (macOS) + if: ${{ runner.os == 'macOS' }} + shell: bash + run: brew install coreutils + - name: Verify mvnd sha256 checksum + shell: bash + run: echo "$(cat ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256) ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip" | sha256sum --check + - name: Unzip mvnd + if: steps.cache-mvnd.outputs.cache-hit != 'true' + shell: bash + run: unzip ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip -d ${{ inputs.install-path }}/ + - name: Show Maven Daemon version + shell: bash + run: | + ${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin/mvnd -D'mvnd.connectTimeout=${{ inputs.mvnd-connect-timeout }}' --version + ${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin/mvnd -D'mvnd.connectTimeout=${{ inputs.mvnd-connect-timeout }}' --status + - name: Set mvnd-dir + id: mvnd-location + shell: bash + run: | + MVND_BIN_DIR="${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin" + if [ "$RUNNER_OS" == "Windows" ]; then + MVND_BIN_DIR="$(cygpath --absolute --long-name --windows $MVND_BIN_DIR)" + fi + echo "MVND_BIN_DIR=${MVND_BIN_DIR}" >> $GITHUB_ENV + echo "mvnd-dir=${MVND_BIN_DIR}" >> $GITHUB_OUTPUT diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a072bff0d98..688e37d2cbb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,9 +1,16 @@ version: 2 updates: -- package-ecosystem: maven + +- package-ecosystem: "github-actions" + target-branch: "develop-6.x.x" directory: "/" schedule: - interval: daily + interval: "weekly" +- package-ecosystem: "maven" + target-branch: "develop-6.x.x" + directory: "/" + schedule: + interval: "daily" time: "03:00" open-pull-requests-limit: 10 ignore: @@ -36,23 +43,23 @@ updates: - "> 1.9.4, < 1.10" - dependency-name: net.sf.saxon:Saxon-HE versions: - > "9" + - "> 9" - dependency-name: org.hsqldb:hsqldb versions: - - 2.6.0 + - "2.6.0" - dependency-name: com.ibm.icu:icu4j versions: - "69.1" - dependency-name: com.sun.mail:jakarta.mail versions: - - 2.0.1 + - "2.0.1" - dependency-name: com.sun.activation:jakarta.activation versions: - - 2.0.1 + - "2.0.1" - dependency-name: com.mycila:license-maven-plugin versions: - "4.0" - - 4.0.rc2 + - "4.0.rc2" - dependency-name: org.owasp:dependency-check-maven versions: - - 6.1.3 + - "6.1.3" diff --git a/.github/workflows/ci-deploy.yml b/.github/workflows/ci-deploy.yml index 51f6aa49537..d148e04524f 100644 --- a/.github/workflows/ci-deploy.yml +++ b/.github/workflows/ci-deploy.yml @@ -7,29 +7,29 @@ jobs: # NOTE (DP): Publish on develop and master, test on PRs against these if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master' || github.base_ref == 'develop' || github.base_ref == 'master' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up JDK 8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: distribution: temurin java-version: '8' - name: Make buildkit default - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 id: buildx with: install: true - name: Cache Maven packages - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + key: deploy-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: deploy-${{ runner.os }}-maven - name: Install bats run: sudo apt-get install bats - name: Build images - run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true clean package + run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true -P !mac-dmg-on-unix,!installer,!concurrency-stress-tests,!micro-benchmarks,skip-build-dist-archives clean package - name: Check local images run: docker image ls - name: Check license headers @@ -66,6 +66,6 @@ jobs: # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} # DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - # run: mvn -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push + # run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push # working-directory: ./exist-docker diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml deleted file mode 100644 index e7897c6ee94..00000000000 --- a/.github/workflows/ci-docs.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Javadoc -on: [push, pull_request] -permissions: - contents: read - -jobs: - test: - name: ${{ matrix.jdk }} Javadocs - strategy: - matrix: - jdk: ['8','11', '16'] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up JDK - uses: actions/setup-java@v2 - with: - distribution: temurin - java-version: ${{ matrix.jdk }} - - name: Cache Maven packages - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Maven Javadoc - run: mvn -V -B -q -T 2C javadoc:javadoc - continue-on-error: ${{ matrix.jdk != '8' }} - \ No newline at end of file diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 24ca6e5c8fd..829d282d44e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -1,45 +1,80 @@ -name: Test +name: Test & documentation on: [push, pull_request] +permissions: + contents: read +env: + MAVEN_OPTS: -XX:StartFlightRecording=maxsize=5g,disk=true,dumponexit=true,settings=default,filename=./ -DtrimStackTrace=false -D'maven.resolver.transport=wagon' + DEV_JDK: '8' jobs: + license: + name: License check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: ${{ env.DEV_JDK }} + cache: 'maven' + - run: mvn -V -B license:check + timeout-minutes: 60 + dependencies: + name: Dependency checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: ${{ env.DEV_JDK }} + cache: 'maven' + - run: mvn -V -B dependency-check:check + timeout-minutes: 60 test: - name: (JDK ${{ matrix.jdk }} / ${{ matrix.os }}) Test - env: - MAVEN_OPTS: -XX:+IgnoreUnrecognizedVMOptions --illegal-access=debug -XX:StartFlightRecording=maxsize=5g,disk=true,dumponexit=true,settings=default,filename=./ + name: ${{ matrix.os }} Test strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - jdk: ['8', '11'] - include: - - os: ubuntu-latest - jdk: '16' runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v2 + - uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v3 with: distribution: temurin - java-version: ${{ matrix.jdk }} + java-version: ${{ env.DEV_JDK }} cache: 'maven' + - name: Install Maven Daemon + id: install-mvnd + uses: ./.github/actions/install-mvnd + with: + version: '1.0-m7' + file-version-suffix: '-m39' + cache: 'true' + - name: Maven Build + timeout-minutes: 10 + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -T 1C compile test-compile -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip' - name: Maven Test timeout-minutes: 60 - run: mvn -V -B -DtrimStackTrace=false clean verify - continue-on-error: ${{ matrix.jdk == '16' }} - - name: Maven Code Coverage - if: ${{ github.ref == 'refs/heads/develop' && matrix.jdk == '8' && matrix.os == 'ubuntu-latest' }} + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B verify -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip' -D'mvnd.maxLostKeepAlive=6000' + - name: Javadoc (Linux only) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -q -T 1C install javadoc:javadoc -DskipTests -D'dependency-check.skip' -D'license.skip' --projects '!exist-distribution,!exist-installer' --also-make + - name: Maven Code Coverage (Develop branch on Linux only) + if: ${{ github.ref == 'refs/heads/develop' && matrix.os == 'ubuntu-latest' }} env: CI_NAME: github BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }} CI_BUILD_NUMBER: ${{ github.run_id }} CI_BUILD_URL: https://github.com/${{ github.repository }}/commit/${{ github.event.after }}/checks COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }} - run: mvn -V -B jacoco:report coveralls:report + run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B jacoco:report coveralls:report - name: Archive build logs if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: - name: ${{ runner.os }}-jdk${{ matrix.jdk }}-build-logs + name: ${{ runner.os }}-build-logs retention-days: 5 path: | **/*.jfr diff --git a/.github/workflows/ci-xqts.yml b/.github/workflows/ci-xqts.yml index b071d077baf..3d8729faa2b 100644 --- a/.github/workflows/ci-xqts.yml +++ b/.github/workflows/ci-xqts.yml @@ -8,24 +8,58 @@ jobs: name: W3C XQuery Test Suite runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK 8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: distribution: temurin java-version: '8' - cache: 'maven' + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: xqts-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: xqts-${{ runner.os }}-maven - name: Maven XQTS Build run: mvn -V -B clean package -DskipTests -Ddependency-check.skip=true --projects exist-xqts --also-make - name: Run XQTS timeout-minutes: 60 env: - JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication - run: find exist-xqts/target -name exist-xqts-runner.sh -exec {} --xqts-version HEAD --output-dir /tmp/xqts-output \; + JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication -Xmx6g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -XX:+ExitOnOutOfMemoryError + run: find exist-xqts/target -name exist-xqts-runner.sh -exec {} --xqts-version HEAD --output-dir /tmp/xqts-output --exclude-test-case RangeExpr-411d,RangeExpr-409d,RangeExpr-408d,RangeExpr-409c,RangeExpr-408c,GenCompEq-21 \; + - name: Check for HeapDump + id: check_heapdump + uses: andstor/file-existence-action@v2 + with: + files: "/tmp/*.hprof" + - name: Compress HeapDump + if: steps.check_heapdump.outputs.files_exists == 'true' + run: zstd --rm -9 --progress -T0 /tmp/*.hprof + - name: Attach HeapDump artifact + if: steps.check_heapdump.outputs.files_exists == 'true' + uses: actions/upload-artifact@v3 + with: + name: exist-xqts-runner-hprof + retention-days: 1 + path: /tmp/*.hprof.zst - name: Archive XQTS Logs if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: xqts-logs retention-days: 14 path: /tmp/xqts-output + - name: Get Previous XQTS Logs Artifacts JSON + run: 'curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/exist-db/exist/actions/artifacts?name=xqts-logs > /tmp/previous-xqts-logs-artifacts.json' + - name: Extract Previous XQTS Logs Artifact JSON + run: cat /tmp/previous-xqts-logs-artifacts.json | jq -r "[.artifacts[] | select(.workflow_run.head_branch == \"develop\")][1].archive_download_url" > /tmp/previous-xqts-logs-artifact.json + - name: Get Previous XQTS Logs Artifact + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: 'cat /tmp/previous-xqts-logs-artifact.json | xargs curl -H "Authorization: Bearer ${GITHUB_TOKEN}" --location --output /tmp/previous-xqts-output.zip' + - name: Extract Previous XQTS Logs Artifact + run: mkdir /tmp/previous-xqts-output && unzip /tmp/previous-xqts-output.zip -d /tmp/previous-xqts-output + - name: Compare Previous and Current XQTS Logs + run: java -jar ~/.m2/repository/net/sf/saxon/Saxon-HE/9.9.1-8/Saxon-HE-9.9.1-8.jar -xsl:exist-xqts/src/main/xslt/compare-results.xslt -it:compare-results -o:/tmp/comparison-results.xml xqts.previous.junit-data-path=/tmp/previous-xqts-output/junit/data xqts.current.junit-data-path=/tmp/xqts-output/junit/data + - name: Show Comparison Results + run: cat /tmp/comparison-results.xml diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index f0ad2ecbe89..8809372cdc7 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -11,26 +11,26 @@ jobs: name: SonarCloud Analysis runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Set up JDK 11 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: distribution: temurin java-version: 11 - name: Cache SonarCloud packages - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar + key: sonarcloud-${{ runner.os }}-cache-${{ hashFiles('**/pom.xml') }} + restore-keys: sonarcloud-${{ runner.os }}-cache - name: Cache Maven packages - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + key: sonarcloud-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: sonarcloud-${{ runner.os }}-maven - name: Analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any diff --git a/exist-core/src/test/java/org/exist/storage/lock/CollectionLocksTest.java b/exist-core/src/test/java/org/exist/storage/lock/CollectionLocksTest.java index 8126cbc1240..f7024d597b1 100644 --- a/exist-core/src/test/java/org/exist/storage/lock/CollectionLocksTest.java +++ b/exist-core/src/test/java/org/exist/storage/lock/CollectionLocksTest.java @@ -100,7 +100,7 @@ public class CollectionLocksTest { * * Note: this should likely be greater than the period required to acquire a lock under contention of CONCURRENCY_LEVEL threads */ - private static final int SINGLE_WRITER_THREAD_SLEEP = 10; // 10ms + private static final int SINGLE_WRITER_THREAD_SLEEP = 15; // 15ms /** * The maximum amount of time we should allow singleWriter diff --git a/exist-parent/pom.xml b/exist-parent/pom.xml index eceed8d96e6..fbf24bf2102 100644 --- a/exist-parent/pom.xml +++ b/exist-parent/pom.xml @@ -563,11 +563,10 @@ org.owasp dependency-check-maven - 8.0.2 + 8.4.2 - - https://nvd.mirror.evolvedbinary.com/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz - https://nvd.mirror.evolvedbinary.com/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz + + true false false diff --git a/exist-xqts/pom.xml b/exist-xqts/pom.xml index 6f3f90f5cb4..6a7afbdbc9a 100644 --- a/exist-xqts/pom.xml +++ b/exist-xqts/pom.xml @@ -54,7 +54,7 @@ org.exist-db exist-xqts-runner_2.13 - 1.1.0 + 1.2.1 diff --git a/exist-xqts/src/main/xslt/compare-results.xslt b/exist-xqts/src/main/xslt/compare-results.xslt new file mode 100644 index 00000000000..5872dc35c28 --- /dev/null +++ b/exist-xqts/src/main/xslt/compare-results.xslt @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file