WIP

stackrox · Dec 14, 2023 · 8a12c68 · 8a12c68
1 parent 16ca57e
commit 8a12c68
Show file tree

Hide file tree

Showing 6 changed files with 256 additions and 13 deletions.
diff --git a/.github/actions/cache-go-dependencies/action.yaml b/.github/actions/cache-go-dependencies/action.yaml
@@ -0,0 +1,29 @@
+name: Cache Go Dependencies
+description: Cache Go Dependencies
+runs:
+  using: composite
+  steps:
+    - name: Determine Go cache paths
+      id: cache-paths
+      run: |
+        echo "GOCACHE=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
+        echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
+      shell: bash
+
+    - name: Cache Go Dependencies
+      uses: actions/cache@v3
+      with:
+        path: |
+          ${{ steps.cache-paths.outputs.GOMODCACHE }}
+        key: go-mod-v1-${{ hashFiles('**/go.sum') }}
+
+    - name: Cache Go Build
+      uses: actions/cache@v3
+      with:
+        path: |
+          ${{ steps.cache-paths.outputs.GOCACHE }}
+        key: go-build-v1-${{ github.job }}-${{ hashFiles('**/go.sum') }}
+
+    - name: Download Go modules
+      run: make deps --always-make
+      shell: bash
diff --git a/.github/actions/generate-gensis-dump/action.yaml b/.github/actions/generate-gensis-dump/action.yaml
@@ -0,0 +1,17 @@
+name: Generate Genesis Dump
+description: Generate Genesis Dump
+runs:
+  using: composite
+  steps:
+    - name: Recover docker image cache space
+      run: |
+        df --si /
+        docker system prune --force --all
+        df --si /
+      shell: bash
+
+    - name: Ignore dubious repository ownership
+      run: |
+        # Prevent fatal error "detected dubious ownership in repository" from recent git.
+        git config --global --add safe.directory "$(pwd)"
+      shell: bash
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -10,7 +10,7 @@ on:
       - synchronize
 
 jobs:
-  scanner-build:
+  pre-build-updater:
     runs-on: ubuntu-latest
     container:
       image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61
@@ -23,25 +23,175 @@ jobs:
 
       - uses: ./.github/actions/job-preamble
 
+      - name: Cache Go dependencies
+        uses: ./.github/actions/cache-go-dependencies
+
+      - name: Build updater
+        run: make build-updater
+
+      - name: Bundle the build to preserve permissions
+        run: tar -cvzf updater-build.tgz
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: updater-build
+          path: updater-build.tgz
+
+  pre-build-scanner:
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: ./.github/actions/job-preamble
+
+      - name: Cache Go dependencies
+        uses: ./.github/actions/cache-go-dependencies
+
+      - name: Build Scanner
+        run: make scanner-build-nodeps
+
+      - name: Bundle the build to preserve permissions
+        run: tar -cvzf scanner-build.tgz
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: scanner-build
+          path: scanner-build.tgz
+
+  generate-genesis-dump:
+    runs-on: ubuntu-latest
+    needs:
+      - pre-build-updater
+    container:
+      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: ./.github/actions/job-preamble
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: updater-build
+
+      - name: Unpack updater build
+        run: |
+          tar xvzf updater-build.tgz
+
       - name: genesis-dump
         run: |
           source ./scripts/ci/lib.sh
           generate_genesis_dump
 
+  generate-db-dump:
+    runs-on: ubuntu-latest
+    needs:
+      - pre-build-updater # do we really need this?
+      - generate-genesis-dump
+    container:
+      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: ./.github/actions/job-preamble
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: updater-build
+
+      - name: Unpack updater build
+        run: |
+          tar xvzf updater-build.tgz
+
       - name: db-dump
         run: |
           source ./scripts/ci/lib.sh
           generate_db_dump
 
-      - name: build-bundle
-        run: |
-          source ./scripts/ci/lib.sh
-          build_bundle
+  generate-scanner-bundle:
+    runs-on: ubuntu-latest
+    needs:
+      - generate-genesis-dump
+      - pre-build-scanner
+    container:
+      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: scanner-db-bundle
-        run: |
-          source ./scripts/ci/lib.sh
-          build_db_bundle
+      - uses: ./.github/actions/job-preamble
+
+      - name: Make bundle
+        run: make image/scanner/rhel/bundle.tar.gz
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: scanner-bundle
+          path: image/scanner/rhel/bundle.tar.gz
+
+  generate-scanner-db-bundle:
+    runs-on: ubuntu-latest
+    needs:
+      - generate-db-dump
+    container:
+      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: ./.github/actions/job-preamble
+
+      - name: Make db bundle
+        run: make image/db/rhel/bundle.tar.gz
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: scanner-db-bundle
+          path: image/db/rhel/bundle.tar.gz
+
+#  build-and-push-main:
+  build-images:
+    runs-on: ubuntu-latest
+    needs:
+        - generate-scanner-bundle
+        - generate-scanner-db-bundle
+    container:
+      image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: ./.github/actions/job-preamble
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: scanner-bundle
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: scanner-db-bundle
 
       - name: Build scanner image
         run: |
@@ -57,4 +207,47 @@ jobs:
 
       - name: Build scanner-db-slim image
         run: |
-          docker build -t scanner-db-slim:"$(make --quiet tag)" -f image/db/rhel/Dockerfile.slim image/db/rhel
+          docker build -t scanner-db-slim:"$(make --quiet tag)" -f image/db/rhel/Dockerfile.slim image/db/rhel
+
+#        # needed for docs ensure_image.sh initial pull with RHACS_BRANDING
+#      - name: Docker login
+#        # Skip for external contributions.
+#        if: |
+#          github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+#        run: |
+#          docker login -u "${QUAY_RHACS_ENG_RO_USERNAME}" --password-stdin quay.io <<<"${QUAY_RHACS_ENG_RO_PASSWORD}"
+#
+#      - name: Push images
+#        # Skip for external contributions.
+#        if: |
+#          github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+#        run: |
+#          source ./scripts/ci/lib.sh
+#          echo "Will determine context from: ${{ github.event_name }} & ${{ github.ref_name }}"
+#          push_context=""
+#          if [[ "${{ github.event_name }}" == "push" && "${{ github.ref_name }}" == "master" ]]; then
+#            push_context="merge-to-master"
+#          fi
+#          push_main_image_set "$push_context" "${{ env.ROX_PRODUCT_BRANDING }}" "${{ matrix.arch }}"
+#
+#      - name: Push matching collector and scanner images
+#        # Skip for external contributions.
+#        if: |
+#          github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+#        run: |
+#          # Need to free up some space before push_matching_collector_scanner_images() does its pull.
+#          docker system prune --all --force
+#          source ./scripts/ci/lib.sh
+#          push_matching_collector_scanner_images "${{ env.ROX_PRODUCT_BRANDING }}" "${{ matrix.arch }}"
+#
+#      - name: diff-dumps
+#        run: |
+#          ./scripts/ci/jobs/diff-dumps.sh
+#
+#      - name: diff-dumps
+#        run: |
+#          ./scripts/ci/jobs/diff-dumps.sh
+#
+#      - name: store-db-dump
+#        run: |
+#          ./scripts/ci/jobs/store-db-dump.sh
diff --git a/Makefile b/Makefile
@@ -208,6 +208,9 @@ endif
 scanner-build-nodeps:
 	$(BUILD_FLAGS) $(BUILD_CMD)
 
+print-curdir:
+	@echo $(CURDIR)
+
 .PHONY: $(CURDIR)/image/scanner/rhel/bundle.tar.gz
 $(CURDIR)/image/scanner/rhel/bundle.tar.gz:
 	$(CURDIR)/image/scanner/rhel/create-bundle.sh $(CURDIR)/image/scanner $(CURDIR)/image/scanner/rhel

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
@@ -820,9 +820,6 @@ send_slack_notice_for_vuln_check_failure() {
 }
 
 generate_genesis_dump() {
-    info "Building updater"
-    make build-updater
-
     info "Generating genesis dump"
     mkdir -p /tmp/genesis-dump
     bin/updater generate-dump --out-file /tmp/genesis-dump/genesis-dump.zip

diff --git a/scripts/lib.sh b/scripts/lib.sh
@@ -52,6 +52,10 @@ is_OPENSHIFT_CI() {
     [[ "${OPENSHIFT_CI:-}" == "true" ]]
 }
 
+is_GITHUB_ACTIONS() {
+    [[ -n "${GITHUB_ACTION:-}" ]]
+}
+
 is_darwin() {
     uname -a | grep -i darwin >/dev/null 2>&1
 }