NIFI-12178 add integration-tests and docker-tests github actions

apache · Oct 13, 2023 · 93aab41 · 93aab41
1 parent 22ad7d5
commit 93aab41
Show file tree

Hide file tree

Showing 7 changed files with 513 additions and 61 deletions.
diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml
@@ -21,7 +21,7 @@ env:
   MAVEN_COMMAND: ./mvnw
   MAVEN_COMMAND_WINDOWS: ./mvnw.cmd
   DEFAULT_MAVEN_OPTS: >-
-    -Xmx3g
+    -Xmx4g
     -XX:ReservedCodeCacheSize=1g
     -XX:+UseG1GC
     -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN
@@ -39,6 +39,7 @@ env:
     --fail-fast
     -pl -:minifi-c2-integration-tests
     -pl -:minifi-integration-tests
+    -pl -:minifi-c2-assembly
     -pl -:minifi-assembly
     -pl -:nifi-assembly
     -pl -:nifi-kafka-connector-assembly
@@ -75,7 +76,7 @@ env:
     -pl -nifi-toolkit/nifi-toolkit-assembly
 
 concurrency:
-  group: ${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 permissions:
@@ -88,7 +89,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Cache Maven Modules
         uses: actions/cache@v3
         with:
@@ -122,7 +123,7 @@ jobs:
           cat /proc/meminfo
           df
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Cache Node Modules
         uses: actions/cache@v3
         with:
@@ -185,7 +186,7 @@ jobs:
           sysctl machdep.cpu
           df
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Cache Node Modules
         uses: actions/cache@v3
         with:
@@ -250,7 +251,7 @@ jobs:
           git config --global core.autocrlf false
           git config --global core.longpaths true
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Cache Node Modules
         uses: actions/cache@v3
         with:

diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml
@@ -0,0 +1,189 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: docker-tests
+
+on:
+  # Run every day at 02:00
+  schedule:
+    - cron: "0 2 * * *"
+  push:
+    paths:
+      - '.github/workflows/docker-tests.yml'
+      - 'nifi-assembly/**'
+      - 'nifi-docker/**'
+      - 'nifi-registry/nifi-registry-assembly/**'
+      - 'nifi-registry/nifi-registry-docker-maven/**'
+      - 'nifi-toolkit/nifi-toolkit-assembly/**'
+      - 'nifi-toolkit/nifi-toolkit-cli/**'
+      - 'nifi-toolkit/nifi-toolkit-encrypt-config/**'
+      - 'nifi-toolkit/nifi-toolkit-tls/**'
+      - 'minifi/minifi-assembly/**'
+      - 'minifi/minifi-docker/**'
+      - 'minifi/minifi-c2/minifi-c2-assembly/**'
+      - 'minifi/minifi-c2/minifi-c2-docker/**'
+  pull_request:
+    paths:
+      - '.github/workflows/docker-tests.yml'
+      - 'nifi-assembly/**'
+      - 'nifi-docker/**'
+      - 'nifi-registry/nifi-registry-assembly/**'
+      - 'nifi-registry/nifi-registry-docker-maven/**'
+      - 'nifi-toolkit/nifi-toolkit-assembly/**'
+      - 'nifi-toolkit/nifi-toolkit-cli/**'
+      - 'nifi-toolkit/nifi-toolkit-encrypt-config/**'
+      - 'nifi-toolkit/nifi-toolkit-tls/**'
+      - 'minifi/minifi-assembly/**'
+      - 'minifi/minifi-docker/**'
+      - 'minifi/minifi-c2/minifi-c2-assembly/**'
+      - 'minifi/minifi-c2/minifi-c2-docker/**'
+
+env:
+  DEFAULT_MAVEN_OPTS: >-
+    -Xmx4g
+    -XX:ReservedCodeCacheSize=1g
+    -XX:+UseG1GC
+    -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN
+    -Daether.connector.http.retryHandler.count=5
+    -Daether.connector.http.connectionMaxTtl=30
+  MAVEN_COMMAND: >-
+    ./mvnw
+    -V
+    -nsu
+    -ntp
+    -ff
+
+# build assemblies (including binaries) for use in Docker Image builds
+  MAVEN_BUILD_ARGUMENTS: >-
+    -am
+    -D skipTests
+    -T 2.0C
+    install
+  MAVEN_BUILD_PROJECTS: >-
+    -pl nifi-assembly
+    -pl nifi-registry/nifi-registry-assembly
+    -pl nifi-toolkit/nifi-toolkit-assembly
+    -pl minifi/minifi-assembly
+    -pl minifi/minifi-c2/minifi-c2-assembly
+
+  MAVEN_CLEAN_ARGUMENTS: >-
+    -T 2.0C
+    clean
+  MAVEN_CLEAN_PROJECTS: >-
+    -pl -nifi-assembly
+    -pl -nifi-registry/nifi-registry-assembly
+    -pl -nifi-toolkit/nifi-toolkit-assembly
+    -pl -nifi-toolkit/nifi-toolkit-cli
+    -pl -nifi-toolkit/nifi-toolkit-encrypt-config
+    -pl -nifi-toolkit/nifi-toolkit-tls
+    -pl -minifi/minifi-assembly
+    -pl -minifi/minifi-c2/minifi-c2-assembly
+
+# build and test Docker Images
+  MAVEN_DOCKER_ARGUMENTS: >-
+    verify
+    -P docker
+  MAVEN_DOCKER_PROJECTS: >-
+    -pl minifi/minifi-docker
+    -pl minifi/minifi-c2/minifi-c2-docker
+    -pl nifi-docker/dockermaven
+    -pl nifi-registry/nifi-registry-docker-maven/dockermaven
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build_and_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: macos-latest doesn't come with Docker and Colima seems to have problems with the nifi-docker integration_test scripts - see NIFI-12177/NIFI-12191
+        os: [ ubuntu-latest ]
+        version: [ 21 ]
+    timeout-minutes: 120
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }} Java ${{ matrix.version }}
+    env:
+      # default to Amazon Corretto JDK
+      JAVA_DISTRIBUTION: corretto
+    steps:
+      # Zulu JDK appears to be more reliable for MacOS
+      - name: Use Java distribution Zulu
+        if: ${{ runner.os == 'macOS' }}
+        run: echo "JAVA_DISTRIBUTION=zulu" >> "$GITHUB_ENV"
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Java ${{ env.JAVA_DISTRIBUTION }} ${{ matrix.version }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: ${{ env.JAVA_DISTRIBUTION }}
+          java-version: ${{ matrix.version }}
+          cache: 'maven'
+
+      - name: Set up Docker
+        if: ${{ runner.os == 'macOS' }}
+        run: |-
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          brew install docker
+          colima start
+
+          sudo ln -s ~/.colima/docker.sock /var/run/docker.sock
+      - name: System Information
+        run: |
+          hostname
+          if [ "${{ runner.os }}" = "macOS" ]; then top -l 1 | grep PhysMem && sysctl machdep.cpu; else cat /proc/cpuinfo && cat /proc/meminfo; fi
+          df
+          docker info
+
+      - name: Build Assemblies
+        env:
+          MAVEN_OPTS: >-
+            ${{ env.DEFAULT_MAVEN_OPTS }}
+        run: >
+          ${{ env.MAVEN_COMMAND }}
+          ${{ env.MAVEN_BUILD_ARGUMENTS }}
+          ${{ env.MAVEN_BUILD_PROJECTS }}
+      # run nifi-toolkit docker build and tests before cleaning modules
+      - name: Run NiFi Toolkit Docker Tests
+        env:
+          MAVEN_OPTS: >-
+            ${{ env.DEFAULT_MAVEN_OPTS }}
+        run: >
+          ${{ env.MAVEN_COMMAND }}
+          ${{ env.MAVEN_DOCKER_ARGUMENTS }}
+          -pl nifi-toolkit/nifi-toolkit-assembly
+      - name: Clean Modules
+        env:
+          MAVEN_OPTS: >-
+            ${{ env.DEFAULT_MAVEN_OPTS }}
+        # clean non-assembly modules before running Docker builds and delete unpacked binaries, otherwise we'll run out of disk space
+        run: >
+          ${{ env.MAVEN_COMMAND }}
+          ${{ env.MAVEN_CLEAN_ARGUMENTS }}
+          ${{ env.MAVEN_CLEAN_PROJECTS }}
+          && rm -rf nifi-assembly/target/nifi-*-bin/ nifi-registry/nifi-registry-assembly/target/nifi-registry-*-bin/
+          && rm -rf ~/.m2/repository/*
+      - name: Run Docker Tests
+        env:
+          MAVEN_OPTS: >-
+            ${{ env.DEFAULT_MAVEN_OPTS }}
+        run: >
+          ${{ env.MAVEN_COMMAND }}
+          ${{ env.MAVEN_DOCKER_ARGUMENTS }}
+          ${{ env.MAVEN_DOCKER_PROJECTS }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -0,0 +1,143 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: integration-tests
+
+on:
+  # Run every day at 03:00
+  schedule:
+    - cron: "0 3 * * *"
+  push:
+    paths:
+      - '.github/workflows/integration-tests.yml'
+      - '**/test/**/*IT.java'
+      - '**/test/**/*ITCase.java'
+      - '**/test/**/IT*.java'
+  pull_request:
+    paths:
+      - '.github/workflows/integration-tests.yml'
+      - '**/test/**/*IT.java'
+      - '**/test/**/*ITCase.java'
+      - '**/test/**/IT*.java'
+
+env:
+  DEFAULT_MAVEN_OPTS: >-
+    -Xmx4g
+    -XX:ReservedCodeCacheSize=1g
+    -XX:+UseG1GC
+    -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN
+    -Daether.connector.http.retryHandler.count=5
+    -Daether.connector.http.connectionMaxTtl=30
+    -Duser.language=en
+    -Duser.country=GB
+    -Duser.timezone=UTC
+  MAVEN_COMMAND: >-
+    ./mvnw
+    -V
+    -nsu
+    -ntp
+    -ff
+
+# skip assemblies and modules covered separately by the system-tests workflow
+# skip unit tests (surefire), run integration tests (failsafe), python ITs included as part of the system-tests workflow
+  MAVEN_BUILD_ARGUMENTS: >-
+    verify
+    -P skip-unit-tests
+    -P integration-tests
+    -P integration-tests-ci
+    -P nifi-registry-integration-tests
+  MAVEN_BUILD_EXCLUDE_PROJECTS: >-
+    -pl -:minifi-assembly
+    -pl -:minifi-c2-assembly
+    -pl -:nifi-assembly
+    -pl -:nifi-toolkit-assembly
+    -pl -:nifi-registry-assembly
+    -pl -:nifi-registry-toolkit-assembly
+    -pl -:nifi-runtime-manifest
+    -pl -:nifi-runtime-manifest-test
+    -pl -:nifi-stateless-assembly
+    -pl -:nifi-stateless-processor-tests
+    -pl -:nifi-stateless-system-test-suite
+    -pl -:nifi-system-test-suite
+    -pl -:nifi-nar-provider-assembly
+    -pl -:nifi-py4j-integration-tests
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build_and_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: macos-latest doesn't come with Docker and Colima seems to have problems with Test Containers - see NIFI-12191
+        os: [ ubuntu-latest ]
+        version: [ 21 ]
+    timeout-minutes: 120
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }} Java ${{ matrix.version }}
+    env:
+      # default to Amazon Corretto JDK
+      JAVA_DISTRIBUTION: corretto
+    steps:
+      # Zulu JDK appears to be more reliable for MacOS
+      - name: Use Java distribution Zulu
+        if: ${{ runner.os == 'macOS' }}
+        run: echo "JAVA_DISTRIBUTION=zulu" >> "$GITHUB_ENV"
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Java ${{ env.JAVA_DISTRIBUTION }} ${{ matrix.version }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: ${{ env.JAVA_DISTRIBUTION }}
+          java-version: ${{ matrix.version }}
+          cache: 'maven'
+
+      # many integration-tests use Docker to run Testcontainers; MacOS doesn't come with Docker, so we need to install it
+      - name: Set up Docker
+        if: ${{ runner.os == 'macOS' }}
+        run: |-
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          brew install docker
+          colima start
+
+          sudo ln -s ~/.colima/docker.sock /var/run/docker.sock
+      - name: System Information
+        run: |
+          hostname
+          if [ "${{ runner.os }}" = "macOS" ]; then top -l 1 | grep PhysMem && sysctl machdep.cpu; else cat /proc/cpuinfo && cat /proc/meminfo; fi
+          df
+          docker info
+
+      - name: Build and Run Tests
+        env:
+          MAVEN_OPTS: >-
+            ${{ env.DEFAULT_MAVEN_OPTS }}
+        run: >
+          ${{ env.MAVEN_COMMAND }}
+          ${{ env.MAVEN_BUILD_ARGUMENTS }}
+          ${{ env.MAVEN_BUILD_EXCLUDE_PROJECTS }}
+      - name: Upload Troubleshooting Logs
+        if: failure() || cancelled()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.os }}-${{ matrix.version }}-failsafe-logs
+          path: |
+            **/target/failsafe-reports/**/*.txt
+          retention-days: 7