From 446290fc8e4021433cb37ff70535d4bfd9000bc0 Mon Sep 17 00:00:00 2001 From: Brad Lugo Date: Tue, 12 Dec 2023 12:40:44 -0800 Subject: [PATCH] WIP --- .github/workflows/build.yaml | 69 +------------------------------ scripts/ci/lib.sh | 78 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c36c7e305..1b22a83f6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -26,74 +26,9 @@ jobs: - name: genesis-dump run: | source ./scripts/ci/lib.sh - - generate_genesis_dump() { - info "Building updater" - make build-updater - - info "Generating genesis dump" - mkdir -p /tmp/genesis-dump - bin/updater generate-dump --out-file /tmp/genesis-dump/genesis-dump.zip - ls -lrt /tmp/genesis-dump - - info "Printing some stats" - bin/updater print-stats /tmp/genesis-dump/genesis-dump.zip - - info "Extracting dumps" - mkdir -p /tmp/vuln-dump - zip /tmp/genesis-dump/genesis-dump.zip 'nvd/*' --copy --out /tmp/vuln-dump/nvd-definitions.zip - zip /tmp/genesis-dump/genesis-dump.zip 'k8s/*' --copy --out /tmp/vuln-dump/k8s-definitions.zip - zip /tmp/genesis-dump/genesis-dump.zip 'istio/*' --copy --out /tmp/vuln-dump/istio-definitions.zip - zip /tmp/genesis-dump/genesis-dump.zip 'rhelv2/repository-to-cpe.json' --copy --out /tmp/vuln-dump/repo2cpe.zip - } - generate_genesis_dump + - name: build-bundle run: | source ./scripts/ci/lib.sh - - cleanup_image() { - info "Reducing the image size" - - set +e - rm -rf /go/{bin,pkg} - rm -rf /root/{.cache,.npm} - rm -rf /usr/local/share/.cache - rm -rf .git - rm -rf image/scanner/bin" - rm -rf image/scanner/rhel/THIRD_PARTY_NOTICES" - set -e - } - - get_genesis_dump() { - info "Retrieving Genesis dump" - - ls -lrt /tmp/vuln-dump || info "No local genesis dump" - - unzip -d image/scanner/dump /tmp/vuln-dump/nvd-definitions.zip - unzip -d image/scanner/dump /tmp/vuln-dump/k8s-definitions.zip - unzip -d image/scanner/dump /tmp/vuln-dump/repo2cpe.zip - unzip -d image/scanner/dump /tmp/vuln-dump/istio-definitions.zip - } - - build_bundle() { - # avoid a -dirty tag - info "Reset to remove Dockerfile modification by OpenShift CI" - git restore . - git status - - info "Building Scanner binary" - make scanner-build-nodeps - - info "Making THIRD_PARTY_NOTICES" - make ossls-notice - - get_genesis_dump - - info "Creating Scanner bundle" - image/scanner/rhel/create-bundle.sh image/scanner image/scanner/rhel - - cleanup_image - } - - build_bundle \ No newline at end of file + build_bundle diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh index c74e8f844..9ee9ba683 100755 --- a/scripts/ci/lib.sh +++ b/scripts/ci/lib.sh @@ -819,6 +819,84 @@ send_slack_notice_for_vuln_check_failure() { curl -XPOST -d @- -H 'Content-Type: application/json' "$webhook_url" } +generate_genesis_dump() { + info "Building updater" + make build-updater + + info "Generating genesis dump" + mkdir -p /tmp/genesis-dump + "$ROOT/bin/updater" generate-dump --out-file /tmp/genesis-dump/genesis-dump.zip + ls -lrt /tmp/genesis-dump + + info "Printing some stats" + "$ROOT/bin/updater" print-stats /tmp/genesis-dump/genesis-dump.zip + + info "Extracting dumps" + mkdir -p /tmp/vuln-dump + zip /tmp/genesis-dump/genesis-dump.zip 'nvd/*' --copy --out /tmp/vuln-dump/nvd-definitions.zip + zip /tmp/genesis-dump/genesis-dump.zip 'k8s/*' --copy --out /tmp/vuln-dump/k8s-definitions.zip + zip /tmp/genesis-dump/genesis-dump.zip 'istio/*' --copy --out /tmp/vuln-dump/istio-definitions.zip + zip /tmp/genesis-dump/genesis-dump.zip 'rhelv2/repository-to-cpe.json' --copy --out /tmp/vuln-dump/repo2cpe.zip +} + +get_genesis_dump() { + info "Retrieving Genesis dump" + + ls -lrt /tmp/vuln-dump || info "No local genesis dump" + + if is_in_PR_context && ! pr_has_label "generate-dumps-on-pr"; then + info "Label generate-dumps-on-pr not set. Pulling dumps from GCS bucket" + mkdir -p /tmp/vuln-dump + gsutil cp gs://stackrox-scanner-ci-vuln-dump/nvd-definitions.zip /tmp/vuln-dump/nvd-definitions.zip + gsutil cp gs://stackrox-scanner-ci-vuln-dump/k8s-definitions.zip /tmp/vuln-dump/k8s-definitions.zip + gsutil cp gs://stackrox-scanner-ci-vuln-dump/istio-definitions.zip /tmp/vuln-dump/istio-definitions.zip + gsutil cp gs://stackrox-scanner-ci-vuln-dump/repo2cpe.zip /tmp/vuln-dump/repo2cpe.zip + fi + + unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/nvd-definitions.zip + unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/k8s-definitions.zip + unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/istio-definitions.zip + unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/repo2cpe.zip +} + +cleanup_image() { + if [[ -z "${OPENSHIFT_BUILD_NAME:-}" ]]; then + info "This is not an OpenShift build, will not reduce the image" + return + fi + + info "Reducing the image size" + + set +e + rm -rf /go/{bin,pkg} + rm -rf /root/{.cache,.npm} + rm -rf /usr/local/share/.cache + rm -rf .git + rm -rf "$ROOT/image/scanner/bin" + rm -rf "$ROOT/image/scanner/rhel/THIRD_PARTY_NOTICES" + set -e +} + +build_bundle() { + # avoid a -dirty tag + info "Reset to remove Dockerfile modification by OpenShift CI" + git restore . + git status + + info "Building Scanner binary" + make scanner-build-nodeps + + info "Making THIRD_PARTY_NOTICES" + make ossls-notice + + get_genesis_dump + + info "Creating Scanner bundle" + "$ROOT/image/scanner/rhel/create-bundle.sh" "$ROOT/image/scanner" "$ROOT/image/scanner/rhel" + + cleanup_image +} + if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then if [[ "$#" -lt 1 ]]; then die "When invoked at the command line a method is required."