From 179881276df6587359e6617cff849f506844d202 Mon Sep 17 00:00:00 2001 From: arewm Date: Fri, 20 Dec 2024 15:02:34 -0500 Subject: [PATCH] chore: identify all unreleased related images for an FBC fragment The set of related images will only continue to grow for a FBC fragment as bundles are added to the packages' channels. Once fragments are released and become part of the target index, we no longer need to consider the validity of these related images. Therefore, we want to be able to determine the set of unique unreleased related images for an FBC fragment so that we know the minimal set of pullspecs that might need to be further inspected and/or verified. Signed-off-by: arewm --- test/utils.sh | 103 +++++++++++++++++++++++++++++++++ unittests_bash/test_utils.bats | 56 +++++++++++++++++- 2 files changed, 156 insertions(+), 3 deletions(-) diff --git a/test/utils.sh b/test/utils.sh index 826efeb..2127758 100644 --- a/test/utils.sh +++ b/test/utils.sh @@ -270,6 +270,28 @@ extract_unique_bundles_from_catalog() { echo "$RENDER_OUT" | tr -d '\000-\031' | jq -r "$jq_unique_bundles" } +# Given output of `opm render` command and package name, this function returns +# all related images for the given package in the catalog +extract_related_images_from_catalog() { + local RENDER_OUT="$1" + local PACKAGE_NAME="$2" + + if [ -z "$RENDER_OUT" ]; then + echo "Missing 'opm render' output for the image" >&2 + exit 2 + fi + + if [ -z "$PACKAGE_NAME" ]; then + echo "Missing package name" >&2 + exit 2 + fi + + # Jq query to extract related images from `opm render` command output + local jq_related_images='select( .package == "'$PACKAGE_NAME'" ) | select(.schema == "olm.bundle") | select( [.properties[]|select(.type == "olm.deprecated")] == []) | ["\(.relatedImages[]? | .image)"]' + # flatten the lists into one and determine the unique values + echo "$RENDER_OUT" | tr -d '\000-\031' | jq "$jq_related_images" | jq -s "flatten(1) | unique" +} + # Given output of `opm render` command and package name, this function returns # unique package names in the catalog extract_unique_package_names_from_catalog() { @@ -359,6 +381,87 @@ get_unreleased_bundle() { } +# This function will be used by tekton tasks in build-definitions +# It returns a list of unreleased related images as indicated in a FBC fragment. +# It compares the provided FBC fragment against the corresponding production index image +get_unreleased_fbc_related_images() { + # FBC fragment containing the unreleased bundle + local FBC_FRAGMENT="" + local INDEX_IMAGE="registry.redhat.io/redhat/redhat-operator-index" + + get_unreleased_fbc_related_images_usage() + { + echo " + get_unreleased_fbc_related_images -i FBC_FRAGMENT [-b INDEX_IMAGE] + " >&2 + exit 2 + } + + local opt + while getopts "i:b:" opt; do + case "${opt}" in + i) + FBC_FRAGMENT="${OPTARG}" ;; + b) + INDEX_IMAGE="${OPTARG}" ;; + *) + get_unreleased_fbc_related_images_usage ;; + esac + done + + if [ -z "$FBC_FRAGMENT" ]; then + echo "Missing parameter FBC_FRAGMENT" >&2 + exit 2 + fi + + # If the index image is provided and has a tag, remove it. + # The target ocp version is determined from the fragment + if [[ "$INDEX_IMAGE" == *:* ]]; then + INDEX_IMAGE="${INDEX_IMAGE%%:*}" + fi + + #Get target ocp version from the fragment + local ocp_version + if ! ocp_version=$(get_ocp_version_from_fbc_fragment "$FBC_FRAGMENT"); then + echo "Could not get ocp version for the fragment" >&2 + exit 1 + fi + + # Run opm render on the FBC fragment to extract package names + local render_out_fbc unique_bundles_fbc package_names + if ! render_out_fbc=$(opm render "$FBC_FRAGMENT"); then + echo "Could not render image $FBC_FRAGMENT" >&2 + exit 1 + fi + package_names=$(extract_unique_package_names_from_catalog "$render_out_fbc") + + # Run opm render on the index image + local render_out_index unique_bundles_index tagged_index + tagged_index="${INDEX_IMAGE}:${ocp_version}" + if ! render_out_index=$(opm render "$tagged_index"); then + echo "Could not render image $tagged_index" >&2 + exit 1 + fi + + # Get unique bundles for each package from the fragment and the index + for package_name in $package_names; do + related_images_fbc+="$(extract_related_images_from_catalog "$render_out_fbc" "$package_name")"$'\n' + related_images_index+="$(extract_related_images_from_catalog "$render_out_index" "$package_name")"$'\n' + done + + # Ensure that the jq arrays are flattened and unique + related_images_fbc=$(echo "$related_images_fbc" | jq -s "flatten(1) | unique") + related_images_index=$(echo "$related_images_index" | jq -s "flatten(1) | unique") + + # Get the images that are only in the fbc fragment + local unreleased_related_images + unreleased_related_images=$(jq -n --argjson released "$related_images_index" --argjson unreleased "$related_images_fbc" '{"released": $released,"unreleased":$unreleased} | .unreleased-.released') + + # output as json array + echo -n "${unreleased_related_images}" + +} + # This function will be used by tekton tasks in build-definitions # It returns a list of labels on the image get_image_labels() { diff --git a/unittests_bash/test_utils.bats b/unittests_bash/test_utils.bats index d9f2ebd..046e46c 100644 --- a/unittests_bash/test_utils.bats +++ b/unittests_bash/test_utils.bats @@ -37,6 +37,9 @@ setup() { elif [[ $1 == "inspect" && $2 == "--no-tags" && $3 == "--raw" && $4 == "docker://valid-fragment-fbc-success-2" ]]; then echo '{"annotations": {"org.opencontainers.image.base.name": "registry.redhat.io/openshift4/ose-operator-registry:v4.20"}}' return 0 + elif [[ $1 == "inspect" && $2 == "--no-tags" && $3 == "--raw" && $4 == "docker://valid-fbc-fragment-isolated" ]]; then + echo '{"annotations": {"org.opencontainers.image.base.name": "registry.redhat.io/openshift4/ose-operator-registry:v4.15"}}' + return 0 else echo 'Unrecognized call to mock skopeo' return 1 @@ -45,12 +48,17 @@ setup() { opm() { if [[ $1 == "render" && $2 == "valid-fragment-fbc" || $1 == "render" && $2 == "valid-fragment-fbc-success" || $1 == "render" && $2 == "valid-fragment-fbc-success-2" ]]; then - echo '{"invalid-control-char": "This is an invalid control char \\t", "schema": "olm.package", "name": "rhbk-operator"}{"schema": "olm.bundle", "package": "rhbk-operator", "image": "registry.redhat.io/rhbk/keycloak-operator-bundle@my-sha", "properties":[]}{"schema": "olm.package", "name": "not-rhbk-operator"}{"schema": "olm.bundle", "package": "not-rhbk-operator", "image": "registry.redhat.io/not-rhbk/operator-bundle@my-other-sha", "properties":[]}' + echo '{"invalid-control-char": "This is an invalid control char \\t", "schema": "olm.package", "name": "rhbk-operator"}{"schema": "olm.bundle", "package": "rhbk-operator", "image": "registry.redhat.io/rhbk/keycloak-operator-bundle@my-sha", "properties":[], "relatedImages": [{"name": "foo-bar", "image": "registry.redhat.io/foo/bar@sha256:my-bar-sha"}, {"name": "foo-baz", "image": "registry.redhat.io/foo/baz@sha256:my-sha"}]}{"schema": "olm.package", "name": "not-rhbk-operator"}{"schema": "olm.bundle", "package": "not-rhbk-operator", "image": "registry.redhat.io/not-rhbk/operator-bundle@my-other-sha", "properties":[], "relatedImages": [{"name": "foo-baz", "image": "registry.redhat.io/foo/baz@sha256:my-sha"}]}' return 0 + elif [[ $1 == "render" && $2 == "valid-fbc-fragment-isolated" ]]; then + echo '{"invalid-control-char": "This is an invalid control char \\t", "schema": "olm.package", "name": "rhbk-operator"}{"schema": "olm.bundle", "package": "rhbk-operator", "image": "registry.redhat.io/rhbk/keycloak-operator-bundle@my-sha", "properties":[], "relatedImages": [{"name": "foo-bar", "image": "registry.redhat.io/foo/bar@sha256:my-bar-sha"}, {"name": "foo-baz", "image": "registry.redhat.io/foo/baz@sha256:my-sha"}]}' elif [[ $1 == "render" && $2 == "valid-operator-bundle-1" ]]; then echo '{"schema":"olm.bundle", "relatedImages": [{"name": "", "image": "quay.io/securesign/rhtas-operator:something"}]}' elif [[ $1 == "render" && $2 == "registry.redhat.io/redhat/redhat-operator-index:v4.15" ]]; then - echo '{"schema": "olm.package", "name": "rhbk-operator"}{"schema": "olm.bundle", "package": "rhbk-operator", "image": "registry.redhat.io/rhbk/keycloak-operator-bundle@random-image", "properties":[]}{"schema": "olm.package", "name": "not-rhbk-operator"}{"schema": "olm.bundle", "package": "not-rhbk-operator", "image": "registry.redhat.io/not-rhbk/operator-bundle@not-my-other-sha", "properties":[]}' + echo '{"schema": "olm.package", "name": "rhbk-operator"}{"schema": "olm.bundle", "package": "rhbk-operator", "image": "registry.redhat.io/rhbk/keycloak-operator-bundle@random-image", "properties":[], "relatedImages": [{"name": "foo-baz", "image": "registry.redhat.io/foo/baz@sha256:my-sha"}]}{"schema": "olm.package", "name": "not-rhbk-operator"}{"schema": "olm.bundle", "package": "not-rhbk-operator", "image": "registry.redhat.io/not-rhbk/operator-bundle@not-my-other-sha", "properties":[], "relatedImages": [{"name": "foo-baz", "image": "registry.redhat.io/foo/bar@sha256:my-bar-sha"}]}' + return 0 + elif [[ $1 == "render" && $2 == "registry.io/random-index:v4.15" ]]; then + echo '{"schema": "olm.package", "name": "rhbk-operator"}{"schema": "olm.bundle", "package": "rhbk-operator", "image": "registry.redhat.io/rhbk/keycloak-operator-bundle@random-image", "properties":[], "relatedImages": [{"name": "foo-bar", "image": "registry.redhat.io/foo/bar@sha256:my-bar-sha"}, {"name": "foo-baz", "image": "registry.redhat.io/foo/baz@sha256:my-sha"}]}{"schema": "olm.package", "name": "not-rhbk-operator"}{"schema": "olm.bundle", "package": "not-rhbk-operator", "image": "registry.redhat.io/not-rhbk/operator-bundle@my-other-sha", "properties":[]}' return 0 elif [[ $1 == "render" && $2 == "registry.io/random-index:v4.20" ]]; then echo '{"schema": "olm.package", "name": "rhbk-operator"}{"schema": "olm.bundle", "package": "rhbk-operator", "image": "registry.redhat.io/rhbk/keycloak-operator-bundle@random-image", "properties":[]}{"schema": "olm.package", "name": "not-rhbk-operator"}{"schema": "olm.bundle", "package": "not-rhbk-operator", "image": "registry.redhat.io/not-rhbk/operator-bundle@my-other-sha", "properties":[]}' @@ -206,7 +214,7 @@ setup() { } @test "Get Unreleased Bundle: valid-fragment-fbc-success and index with tag" { - run get_unreleased_bundle -i valid-fragment-fbc-success -b registry.redhat.io/redhat/redhat-operator-index:v4.27@randomsha256 + run get_unreleased_bundle -i valid-fragment-fbc-success -b registry.redhat.io/redhat/redhat-operator-index:v4.15@randomsha256 EXPECTED_RESPONSE=$(echo "registry.redhat.io/rhbk/keycloak-operator-bundle@my-sha registry.redhat.io/not-rhbk/operator-bundle@my-other-sha" | tr ' ' '\n') [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 0 ]] } @@ -217,6 +225,48 @@ setup() { [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 0 ]] } +@test "Get Unreleased FBC related images: missing FBC_FRAGMENT" { + run get_unreleased_fbc_related_images + EXPECTED_RESPONSE='Missing parameter FBC_FRAGMENT' + [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 2 ]] +} + +@test "Get Unreleased FBC related images: invalid-url" { + run get_unreleased_fbc_related_images -i invalid-url + EXPECTED_RESPONSE='Could not get ocp version for the fragment' + [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 1 ]] +} + +@test "Get Unreleased FBC related images: invalid-fragment-fbc" { + run get_unreleased_fbc_related_images -i invalid-fragment-fbc + EXPECTED_RESPONSE='Could not render image invalid-fragment-fbc' + [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 1 ]] +} + +@test "Get Unreleased FBC related images: valid-fragment-fbc and invalid index" { + run get_unreleased_fbc_related_images -i valid-fragment-fbc + EXPECTED_RESPONSE='Could not render image registry.redhat.io/redhat/redhat-operator-index:v4.12' + [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 1 ]] +} + +@test "Get Unreleased FBC related images: valid-fbc-fragment-isolated and missing index" { + run get_unreleased_fbc_related_images -i valid-fbc-fragment-isolated + EXPECTED_RESPONSE=$(echo "[ **\"registry.redhat.io/foo/bar@sha256:my-bar-sha\" ]" | tr ' ' '\n' | tr '*' ' ') + [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 0 ]] +} + +@test "Get Unreleased FBC related images: valid-fbc-fragment-isolated and index with tag" { + run get_unreleased_fbc_related_images -i valid-fbc-fragment-isolated -b registry.redhat.io/redhat/redhat-operator-index:v4.15@randomsha256 + EXPECTED_RESPONSE=$(echo "[ **\"registry.redhat.io/foo/bar@sha256:my-bar-sha\" ]" | tr ' ' '\n' | tr '*' ' ') + [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 0 ]] +} + +@test "Get Unreleased FBC related images: valid-fbc-fragment-isolated and custom index" { + run get_unreleased_fbc_related_images -i valid-fbc-fragment-isolated -b registry.io/random-index:v4.20 + EXPECTED_RESPONSE="[]" + [[ "${EXPECTED_RESPONSE}" = "${output}" && "$status" -eq 0 ]] +} + @test "Get Image Labels: valid-image-manifest-url-2" { run get_image_labels valid-image-manifest-url-2 EXPECTED_RESPONSE="architecture=arm64"