From b299f95a8b9950a71e9691b6a0b11c36486fe4b8 Mon Sep 17 00:00:00 2001 From: Brian Cook Date: Thu, 21 Nov 2024 11:47:59 -0500 Subject: [PATCH] RHSM integration for prefetch task This adds steps to the prefetch task to detect when a Red Hat subscription activation key is provided. When prefetch is configured for RPM package manager and an acivation key is provided, the pod will be registered with Red Hat's subscription management service so that protected content can be fetched. The activation key is provided via the param ACTIVATION_KEY. This is expected to be the name of a secret with two keys: org and activationkey. For more information see https://access.redhat.com/solutions/3341191. The task modifies the prefetch input on the fly in order to inject the necessary entitlement files used for mTLS auth. For example, for simple input like 'rpm', the input will first be transformed to: [ { "type": "rpm", "options": { "ssl": { "client_key": null, "client_cert": null, "ca_bundle": null, "verify": 1 } } } ] After this the entitelement certificate information will be added to ALL instances of rpm package manager present (in case the input is a JSON array.) After prefetch the container is unregistered. --- .../README.md | 1 + pipelines/docker-build-oci-ta/README.md | 1 + pipelines/docker-build/README.md | 1 + pipelines/tekton-bundle-builder/README.md | 1 + .../0.1/README.md | 1 + .../0.1/prefetch-dependencies-oci-ta.yaml | 196 ++++++++++++++++- .../0.1/prefetch-dependencies.yaml | 206 +++++++++++++++++- 7 files changed, 395 insertions(+), 12 deletions(-) diff --git a/pipelines/docker-build-multi-platform-oci-ta/README.md b/pipelines/docker-build-multi-platform-oci-ta/README.md index 6b731b0b18..af0041575a 100644 --- a/pipelines/docker-build-multi-platform-oci-ta/README.md +++ b/pipelines/docker-build-multi-platform-oci-ta/README.md @@ -135,6 +135,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito ### prefetch-dependencies-oci-ta:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.clone-repository.results.SOURCE_ARTIFACT)'| |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | diff --git a/pipelines/docker-build-oci-ta/README.md b/pipelines/docker-build-oci-ta/README.md index 1bc86b739a..1cfdc67dd5 100644 --- a/pipelines/docker-build-oci-ta/README.md +++ b/pipelines/docker-build-oci-ta/README.md @@ -132,6 +132,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito ### prefetch-dependencies-oci-ta:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.clone-repository.results.SOURCE_ARTIFACT)'| |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | diff --git a/pipelines/docker-build/README.md b/pipelines/docker-build/README.md index 50815a8e52..82a6faa560 100644 --- a/pipelines/docker-build/README.md +++ b/pipelines/docker-build/README.md @@ -131,6 +131,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito ### prefetch-dependencies:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | |config-file-content| Pass configuration to cachi2. Note this needs to be passed as a YAML-formatted config dump, not as a file path! | | | diff --git a/pipelines/tekton-bundle-builder/README.md b/pipelines/tekton-bundle-builder/README.md index 5cd3595543..a0b065a8cc 100644 --- a/pipelines/tekton-bundle-builder/README.md +++ b/pipelines/tekton-bundle-builder/README.md @@ -66,6 +66,7 @@ ### prefetch-dependencies:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | |config-file-content| Pass configuration to cachi2. Note this needs to be passed as a YAML-formatted config dump, not as a file path! | | | diff --git a/task/prefetch-dependencies-oci-ta/0.1/README.md b/task/prefetch-dependencies-oci-ta/0.1/README.md index 48065f95ea..73e8ebb5d5 100644 --- a/task/prefetch-dependencies-oci-ta/0.1/README.md +++ b/task/prefetch-dependencies-oci-ta/0.1/README.md @@ -26,6 +26,7 @@ params: ## Parameters |name|description|default value|required| |---|---|---|---| +|ACTIVATION_KEY|Name of secret which contains subscription activation key|activation-key|false| |SOURCE_ARTIFACT|The Trusted Artifact URI pointing to the artifact with the application source code.||true| |caTrustConfigMapKey|The name of the key in the ConfigMap that contains the CA bundle data.|ca-bundle.crt|false| |caTrustConfigMapName|The name of the ConfigMap to read CA bundle data from.|trusted-ca|false| diff --git a/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml b/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml index 4a7c138739..cc6352dd4c 100644 --- a/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml +++ b/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml @@ -33,6 +33,10 @@ spec: [available configuration parameters]: https://github.com/containerbuildsystem/cachi2?tab=readme-ov-file#available-configuration-parameters params: + - name: ACTIVATION_KEY + description: Name of secret which contains subscription activation key + type: string + default: activation-key - name: SOURCE_ARTIFACT description: The Trusted Artifact URI pointing to the artifact with the application source code. @@ -79,8 +83,16 @@ spec: the application source code. type: string volumes: + - name: activation-key + secret: + optional: true + secretName: $(params.ACTIVATION_KEY) - name: config emptyDir: {} + - name: etc-pki-entitlement + emptyDir: {} + - name: shared + emptyDir: {} - name: trusted-ca configMap: items: @@ -110,6 +122,8 @@ spec: volumeMounts: - mountPath: /mnt/config name: config + - mountPath: /shared + name: shared - mountPath: /var/workdir name: workdir steps: @@ -143,15 +157,164 @@ spec: # https://github.com/containerbuildsystem/cachi2/issues/577 yq 'del(.goproxy_url)' <<<"${CONFIG_FILE_CONTENT}" >/mnt/config/config.yaml fi +<<<<<<< HEAD - name: prefetch-dependencies image: quay.io/redhat-appstudio/cachi2:0.14.0@sha256:3088e307f41894c0654b69bb199a4648ba31721d6173099cf6ca8ff259c0e457 +======= + - name: check-prefetch-input + image: quay.io/redhat-appstudio/cachi2:0.13.0@sha256:eb34cfe3fea20997eebd8164dc93eedb2fd7a60dc1fb4afcc1b1ff43df9d6667 + env: + - name: INPUT + value: $(params.input) + script: | + if [ -z "${INPUT}" ]; then + # Confirm input was provided though it's likely the whole task would be skipped if it wasn't + echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + echo "skip" >/shared/skip + fi + - name: register-red-hat + image: quay.io/redhat-appstudio/cachi2@sha256:eb34cfe3fea20997eebd8164dc93eedb2fd7a60dc1fb4afcc1b1ff43df9d6667 + results: + - name: registered + type: string + volumeMounts: + - mountPath: /activation-key + name: activation-key + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + echo "false" >/shared/registered + ACTIVATION_KEY_PATH="/activation-key" + + mkdir -p /shared/rhsm/entitlement + mkdir -p /shared/rhsm/consumer + + if [ -e /activation-key/org ]; then + cp -r --preserve=mode "$ACTIVATION_KEY_PATH" /tmp/activation-key + + echo "Registering with Red Hat subscription manager." + subscription-manager register --org "$(cat /tmp/activation-key/org)" --activationkey "$(cat /tmp/activation-key/activationkey)" + + # copy generated certificates to /shared/rhsm + cp /etc/pki/entitlement/*.pem /shared/rhsm/entitlement/ + cp /etc/pki/consumer/*.pem /shared/rhsm/consumer/ + + file="$(find /shared/rhsm/entitlement -regextype egrep -regex '.*[0-9]+\.pem' -printf %f)" + echo "file: $file" + basename "$file" .pem >/shared/RHSM_ID + echo "./RHSM_ID:" + cat /shared/RHSM_ID + + # trust the CA used for Red Hat CDN + cp /etc/rhsm-host/ca/redhat-uep.pem /shared/rhsm/redhat-uep.pem + fi + - name: preprocess-input + image: quay.io/redhat-appstudio/cachi2@sha256:eb34cfe3fea20997eebd8164dc93eedb2fd7a60dc1fb4afcc1b1ff43df9d6667 + args: + - $(params.input) + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + script: | + #!/bin/python3 + import json + import os + import sys + + + def string_to_json(input: str): + if input in ['bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn-classic', 'yarn']: + input = '{"type": "%s"}' % input + print("json: %s" % input) + return input + + + def json_to_list(input: str): + input = json.loads(input) + if type(input) is dict: + input = [input] + return json.dumps(input) + + + def inject_certs(input: str, rhsm_id: str): + input_list: list = json.loads(input) + + cert = ("/shared/rhsm/entitlement/%s.pem" % rhsm_id) + key = ("/shared/rhsm/entitlement/%s-key.pem" % rhsm_id) + ca_bundle = os.getenv("CA_BUNDLE", None) + for pkg_man in input_list: + if pkg_man["type"] == "rpm": + + # preserve verify setting + verify = \ + pkg_man.get("options", {}).get("ssl", {}).get("ssl_verify", 1) + + # preserve other options + options: dict = pkg_man.get('options', {}) + + ssl_options = { + "client_key": key, + "client_cert": cert, + "ca_bundle": ca_bundle, + "ssl_verify": verify} + + options['ssl'] = ssl_options + pkg_man["options"] = options + return (json.dumps(input_list)) + + + def convert_input(input, rhsm_id): + input = string_to_json(input) + input = json_to_list(input) + input = inject_certs(input, rhsm_id) + return input + + + if __name__ == '__main__': + + if os.path.isfile("/shared/skip"): + sys.exit() + + rhsm_id = "" + input = "" + + try: + f = open("/shared/RHSM_ID", "r") + rhsm_id = f.read().strip("\n") + except FileNotFoundError: + print("No RHSM ID found.") + + if rhsm_id == "": + input = sys.argv[1] + else: + print("RHSM Cert ID is: %s" % rhsm_id) + print("Called with args: %s" % str(sys.argv)) + input = convert_input(sys.argv[1], rhsm_id) + + print("Preprocessing result: %s" % input) + with open('/shared/rhsm/preprocessed_input', 'w') as f: + f.write(input) + - name: prefetch-dependencies + image: quay.io/redhat-appstudio/cachi2@sha256:eb34cfe3fea20997eebd8164dc93eedb2fd7a60dc1fb4afcc1b1ff43df9d6667 +>>>>>>> 04d5e91c (RHSM integration for prefetch task) volumeMounts: - mountPath: /mnt/trusted-ca name: trusted-ca readOnly: true + - mountPath: /activation-key + name: activation-key env: - - name: INPUT - value: $(params.input) - name: DEV_PACKAGE_MANAGERS value: $(params.dev-package-managers) - name: LOG_LEVEL @@ -165,9 +328,10 @@ spec: - name: WORKSPACE_NETRC_PATH value: $(workspaces.netrc.path) script: | - if [ -z "${INPUT}" ]; then - # Confirm input was provided though it's likely the whole task would be skipped if it wasn't - echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + #!/bin/bash + + if [ -f /shared/skip ]; then + echo "Skipping." exit 0 fi @@ -183,6 +347,16 @@ spec: dev_pacman_flag="" fi + INPUT=$(cat /shared/rhsm/preprocessed_input) + export INPUT + + # trust Red Hat CA cert used for Red Hat CDN + if [ -f /shared/rhsm/redhat-uep.pem ]; then + echo "Adding Red Hat CA certificate to trusted roots." + cp /shared/rhsm/redhat-uep.pem /etc/pki/ca-trust/source/anchors/ + update-ca-trust + fi + # Copied from https://github.com/konflux-ci/build-definitions/blob/main/task/git-clone/0.1/git-clone.yaml if [ "${WORKSPACE_GIT_AUTH_BOUND}" = "true" ]; then if [ -f "${WORKSPACE_GIT_AUTH_PATH}/.git-credentials" ] && [ -f "${WORKSPACE_GIT_AUTH_PATH}/.gitconfig" ]; then @@ -225,6 +399,18 @@ spec: cachi2 --log-level="$LOG_LEVEL" inject-files /var/workdir/cachi2/output \ --for-output-dir=/cachi2/output + - name: unregister-rhsm + image: quay.io/redhat-appstudio/cachi2@sha256:eb34cfe3fea20997eebd8164dc93eedb2fd7a60dc1fb4afcc1b1ff43df9d6667 + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + cp /shared/rhsm/consumer/* /etc/pki/consumer/ + cp /shared/rhsm/entitlement/* /etc/pki/entitlement/ + subscription-manager unregister || true - name: create-trusted-artifact image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:81c4864dae6bb11595f657be887e205262e70086a05ed16ada827fd6391926ac args: diff --git a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml index cdf99f0560..09ab34d24c 100644 --- a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml +++ b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml @@ -53,6 +53,10 @@ spec: type: string description: The name of the key in the ConfigMap that contains the CA bundle data. default: ca-bundle.crt + - name: ACTIVATION_KEY + default: activation-key + description: Name of secret which contains subscription activation key + type: string stepTemplate: env: @@ -61,7 +65,8 @@ spec: volumeMounts: - name: config mountPath: /mnt/config - + - mountPath: /shared + name: shared steps: - name: sanitize-cachi2-config-file-with-yq image: quay.io/konflux-ci/yq:latest@sha256:343c2ca0a347ae87fe43750ee0873e1fe813f77eff56e9722c840bb75d97fef2 @@ -74,13 +79,164 @@ spec: yq 'del(.goproxy_url)' <<< "${CONFIG_FILE_CONTENT}" > /mnt/config/config.yaml fi - - image: quay.io/redhat-appstudio/cachi2:0.14.0@sha256:3088e307f41894c0654b69bb199a4648ba31721d6173099cf6ca8ff259c0e457 + - name: check-prefetch-input + image: quay.io/redhat-appstudio/cachi2:0.14.0@sha256:3088e307f41894c0654b69bb199a4648ba31721d6173099cf6ca8ff259c0e457 # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting # the cluster will set imagePullPolicy to IfNotPresent - name: prefetch-dependencies env: - name: INPUT value: $(params.input) + script: | + if [ -z "${INPUT}" ] + then + # Confirm input was provided though it's likely the whole task would be skipped if it wasn't + echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + echo "skip" > /shared/skip + fi + + - name: register-red-hat + image: quay.io/redhat-appstudio/cachi2@sha256:eb34cfe3fea20997eebd8164dc93eedb2fd7a60dc1fb4afcc1b1ff43df9d6667 + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + volumeMounts: + - mountPath: /activation-key + name: activation-key + results: + - name: registered + type: string + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + echo "false" > /shared/registered + ACTIVATION_KEY_PATH="/activation-key" + + mkdir -p /shared/rhsm/entitlement + mkdir -p /shared/rhsm/consumer + + if [ -e /activation-key/org ]; then + cp -r --preserve=mode "$ACTIVATION_KEY_PATH" /tmp/activation-key + + echo "Registering with Red Hat subscription manager." + subscription-manager register --org "$(cat /tmp/activation-key/org)" --activationkey "$(cat /tmp/activation-key/activationkey)" + + # copy generated certificates to /shared/rhsm + cp /etc/pki/entitlement/*.pem /shared/rhsm/entitlement/ + cp /etc/pki/consumer/*.pem /shared/rhsm/consumer/ + + file="$(find /shared/rhsm/entitlement -regextype egrep -regex '.*[0-9]+\.pem' -printf %f)" + echo "file: $file" + basename "$file" .pem > /shared/RHSM_ID + echo "./RHSM_ID:" + cat /shared/RHSM_ID + + # trust the CA used for Red Hat CDN + cp /etc/rhsm-host/ca/redhat-uep.pem /shared/rhsm/redhat-uep.pem + fi + + - name: preprocess-input + image: quay.io/redhat-appstudio/cachi2@sha256:eb34cfe3fea20997eebd8164dc93eedb2fd7a60dc1fb4afcc1b1ff43df9d6667 + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + args: ["$(params.input)"] + script: | + #!/bin/python3 + import json + import os + import sys + + + def string_to_json(input: str): + if input in ['bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn-classic', 'yarn']: + input = '{"type": "%s"}' % input + print("json: %s" % input) + return input + + + def json_to_list(input: str): + input = json.loads(input) + if type(input) is dict: + input = [input] + return json.dumps(input) + + + def inject_certs(input: str, rhsm_id: str): + input_list: list = json.loads(input) + + cert = ("/shared/rhsm/entitlement/%s.pem" % rhsm_id) + key = ("/shared/rhsm/entitlement/%s-key.pem" % rhsm_id) + ca_bundle = os.getenv("CA_BUNDLE", None) + for pkg_man in input_list: + if pkg_man["type"] == "rpm": + + # preserve verify setting + verify = \ + pkg_man.get("options", {}).get("ssl", {}).get("ssl_verify", 1) + + # preserve other options + options: dict = pkg_man.get('options', {}) + + ssl_options = { + "client_key": key, + "client_cert": cert, + "ca_bundle": ca_bundle, + "ssl_verify": verify} + + options['ssl'] = ssl_options + pkg_man["options"] = options + return (json.dumps(input_list)) + + + def convert_input(input, rhsm_id): + input = string_to_json(input) + input = json_to_list(input) + input = inject_certs(input, rhsm_id) + return input + + + if __name__ == '__main__': + + if os.path.isfile("/shared/skip"): + sys.exit() + + rhsm_id = "" + input = "" + + try: + f = open("/shared/RHSM_ID", "r") + rhsm_id = f.read().strip("\n") + except FileNotFoundError: + print("No RHSM ID found.") + + if rhsm_id == "": + input = sys.argv[1] + else: + print("RHSM Cert ID is: %s" % rhsm_id) + print("Called with args: %s" % str(sys.argv)) + input = convert_input(sys.argv[1], rhsm_id) + + print("Preprocessing result: %s" % input) + with open('/shared/rhsm/preprocessed_input', 'w') as f: + f.write(input) + + + - name: prefetch-dependencies + image: quay.io/redhat-appstudio/cachi2:0.14.0@sha256:3088e307f41894c0654b69bb199a4648ba31721d6173099cf6ca8ff259c0e457 + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + env: - name: DEV_PACKAGE_MANAGERS value: $(params.dev-package-managers) - name: LOG_LEVEL @@ -97,11 +253,13 @@ spec: - name: trusted-ca mountPath: /mnt/trusted-ca readOnly: true + - mountPath: /activation-key + name: activation-key script: | - if [ -z "${INPUT}" ] - then - # Confirm input was provided though it's likely the whole task would be skipped if it wasn't - echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + #!/bin/bash + + if [ -f /shared/skip ]; then + echo "Skipping." exit 0 fi @@ -117,6 +275,16 @@ spec: dev_pacman_flag="" fi + INPUT=$(cat /shared/rhsm/preprocessed_input) + export INPUT + + # trust Red Hat CA cert used for Red Hat CDN + if [ -f /shared/rhsm/redhat-uep.pem ]; then + echo "Adding Red Hat CA certificate to trusted roots." + cp /shared/rhsm/redhat-uep.pem /etc/pki/ca-trust/source/anchors/ + update-ca-trust + fi + # Copied from https://github.com/konflux-ci/build-definitions/blob/main/task/git-clone/0.1/git-clone.yaml if [ "${WORKSPACE_GIT_AUTH_BOUND}" = "true" ] ; then if [ -f "${WORKSPACE_GIT_AUTH_PATH}/.git-credentials" ] && [ -f "${WORKSPACE_GIT_AUTH_PATH}/.gitconfig" ]; then @@ -159,6 +327,22 @@ spec: cachi2 --log-level="$LOG_LEVEL" inject-files $(workspaces.source.path)/cachi2/output \ --for-output-dir=/cachi2/output + + - name: unregister-rhsm + image: quay.io/redhat-appstudio/cachi2:0.14.0@sha256:3088e307f41894c0654b69bb199a4648ba31721d6173099cf6ca8ff259c0e457 + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + cp /shared/rhsm/consumer/* /etc/pki/consumer/ + cp /shared/rhsm/entitlement/* /etc/pki/entitlement/ + subscription-manager unregister || true + workspaces: - name: source description: Workspace with the source code, cachi2 artifacts will be stored on the workspace as well @@ -175,6 +359,14 @@ spec: performing http(s) requests. optional: true volumes: + - name: shared + emptyDir: {} + - name: etc-pki-entitlement + emptyDir: {} + - name: activation-key + secret: + optional: true + secretName: $(params.ACTIVATION_KEY) - name: trusted-ca configMap: name: $(params.caTrustConfigMapName)