From 5ed4fc3d32f0f6639412263132d03a105cd772c3 Mon Sep 17 00:00:00 2001 From: Frederik Rothenberger Date: Thu, 24 Aug 2023 11:43:28 +0200 Subject: [PATCH] Add support for dfx deps (#1798) * Add support for dfx pull This PR adds support for DFX pull. Supporting dfx pull creates a dependency between the dev build and the prod build, because the wasm metadata needs to include the dev build hash. Furthermore, it is not only the prod_build but also the dev build that needs to include additional metadata. To make the feature work this PR: - changes the build order of the wasm modules so that the prod buid is built after the dev build to include the dev build hash - all II wasm files include additional metadata, _if_ the commit corresponds to a release (since no valid metadata can be generated for non-released wasms) - a script has been added to generate the above metadata - a script for wasm hash verification has been added since verification of wasm hashes is no more difficult (as it entails building II twice and including the hash of one module in the other) Future work: - Refactor build scripts to separate building the wasm from adding metadata. This would allow to build all wasms in parallel again, shortening CI times. - Try to move to `dfx` for wasm post-processing. This would remove a substantial amount of scripts / logic from our build process. * Adjust metadata init type and guide * Fix shell expansion issues and improve init guide * Improve comment * Update release build check with metadata * Implement review feedback Changes: - Change `dfx pull` to `dfx deps` - Fix a misleading comment (including the location I copied it from) - Introduced helper functions to print in color - Fix typos --- .github/actions/check-build/action.yml | 13 ++- .github/actions/release/run.sh | 5 +- .github/workflows/canister-tests.yml | 102 +++++++++++++++--- .github/workflows/release-build-check.yml | 20 +++- Dockerfile | 5 +- scripts/build | 21 +++- scripts/dfx-metadata | 94 +++++++++++++++++ scripts/docker-build | 13 ++- scripts/verify-hash | 122 ++++++++++++++++++++++ 9 files changed, 369 insertions(+), 26 deletions(-) create mode 100755 scripts/dfx-metadata create mode 100755 scripts/verify-hash diff --git a/.github/actions/check-build/action.yml b/.github/actions/check-build/action.yml index b99426f197..526f034e14 100644 --- a/.github/actions/check-build/action.yml +++ b/.github/actions/check-build/action.yml @@ -2,7 +2,10 @@ name: 'Check build' description: This action performs a clean, non-Docker build of II, and optionally checks the gzipped Wasm module sha256 against the 'sha256' argument. Nothing is cached except for the bootstrap environment. inputs: sha256: - description: The expected sha256 of the final Wasm module + description: The expected sha256 of the final production Wasm module + required: false + dfx-metadata: + description: The dfx metadata to include in the production build required: false runs: using: "composite" @@ -12,7 +15,13 @@ runs: # run the build - run: npm ci shell: bash - - run: ./scripts/build + - name: Run build + env: + # use an env variable so that the GitHub templating does not cause issues with string escaping + # see: https://github.com/orgs/community/discussions/32012 + METADATA: ${{ inputs.dfx-metadata }} + run: | + ./scripts/build ${METADATA:+"--dfx-metadata" "$METADATA"} shell: bash # check the hash diff --git a/.github/actions/release/run.sh b/.github/actions/release/run.sh index 770abe3d37..d1e99a0b42 100755 --- a/.github/actions/release/run.sh +++ b/.github/actions/release/run.sh @@ -38,10 +38,7 @@ To build the wasm modules yourself and verify their hashes, run the following co \`\`\` git pull # to ensure you have the latest changes. git checkout $GITHUB_SHA -./scripts/docker-build -sha256sum internet_identity.wasm.gz -./scripts/docker-build --archive -sha256sum archive.wasm.gz +./scripts/verify-hash --ii-hash $(shasum -a 256 "$PRODUCTION_ASSET" | cut -d ' ' -f1) \`\`\` EOF diff --git a/.github/workflows/canister-tests.yml b/.github/workflows/canister-tests.yml index 2ae61a94b2..12aba06458 100644 --- a/.github/workflows/canister-tests.yml +++ b/.github/workflows/canister-tests.yml @@ -45,12 +45,8 @@ jobs: # NOTE: if you modify the flavors, update the #flavors table in README.md matrix: include: - # The production build - - name: internet_identity_production.wasm.gz - II_FETCH_ROOT_KEY: 0 - II_DUMMY_CAPTCHA: 0 - II_DUMMY_AUTH: 0 - II_INSECURE_REQUESTS: 0 + # The production build is built later because it has a dependency on the dev build (for dfx deps) + # See job: docker-build-internet_identity_production # No captcha and fetching the root key, used in (our) tests, backend and # selenium. @@ -78,6 +74,13 @@ jobs: echo "Inferred version: '$version'" echo "version=$version" >> "$GITHUB_OUTPUT" + - name: "Create dfx metadata for the dfx deps feature" + id: dfx-metadata + run: | + dfx_metadata_json="$(./scripts/dfx-metadata --asset-name ${{ matrix.name }} )" + echo "using dfx metadata $dfx_metadata_json" + echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT" + - name: Set up docker buildx uses: docker/setup-buildx-action@v2 @@ -92,6 +95,7 @@ jobs: II_DUMMY_CAPTCHA=${{ matrix.II_DUMMY_CAPTCHA }} II_INSECURE_REQUESTS=${{ matrix.II_INSECURE_REQUESTS }} II_VERSION=${{ steps.version.outputs.version }} + DFX_METADATA=${{ steps.dfx-metadata.outputs.metadata }} cache-from: type=gha,scope=cached-stage # Exports the artefacts from the final stage outputs: ./out @@ -105,9 +109,68 @@ jobs: # name is the name used to display and retrieve the artifact name: ${{ matrix.name }} # path is the name used as the file to upload and the name of the - # downloaded file + # file when downloaded path: ${{ matrix.name }} + # Build the production version of internet identity. + # The production build is separately because it has a dependency on the dev build (for dfx deps) + # + # Note: do not rename this job as it needs to contain the file name of the produced asset (without extension) + # in order for the release script action to work correctly. + docker-build-internet_identity_production: + runs-on: ubuntu-latest + needs: docker-build-ii + steps: + - uses: actions/checkout@v3 + + - name: Infer version + id: version + run: | + version="$(./scripts/version)" + echo "Inferred version: '$version'" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: 'Download dev build II wasm.gz' + uses: actions/download-artifact@v3 + with: + name: internet_identity_dev.wasm.gz + path: . + + - name: "Create dfx metadata for the dfx deps feature" + id: dfx-metadata + run: | + sha256="$(shasum -a 256 ./internet_identity_dev.wasm.gz | cut -d ' ' -f1)" + dfx_metadata_json="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz --wasm-hash $sha256)" + echo "using dfx metadata $dfx_metadata_json" + echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT" + + - name: Set up docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Build internet_identity_production.wasm.gz + uses: docker/build-push-action@v3 + with: + context: . + file: Dockerfile + build-args: | + II_VERSION=${{ steps.version.outputs.version }} + DFX_METADATA=${{ steps.dfx-metadata.outputs.metadata }} + cache-from: type=gha,scope=cached-stage + # Exports the artefacts from the final stage + outputs: ./out + target: scratch_internet_identity + + - run: mv out/internet_identity.wasm.gz internet_identity_production.wasm.gz + - run: sha256sum internet_identity_production.wasm.gz + - name: 'Upload internet_identity_production.wasm.gz' + uses: actions/upload-artifact@v3 + with: + # name is the name used to display and retrieve the artifact + name: internet_identity_production.wasm.gz + # path is the name used as the file to upload and the name of the + # file when downloaded + path: internet_identity_production.wasm.gz + docker-build-archive: runs-on: ubuntu-latest needs: docker-build-base @@ -140,7 +203,7 @@ jobs: wasm-size: runs-on: ubuntu-latest - needs: docker-build-ii + needs: docker-build-internet_identity_production steps: - uses: actions/checkout@v3 - name: 'Download wasm' @@ -487,7 +550,7 @@ jobs: deploy: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/release-') - needs: [docker-build-ii, docker-build-archive] + needs: [docker-build-internet_identity_production, docker-build-archive] steps: - uses: actions/checkout@v3 @@ -531,7 +594,7 @@ jobs: release: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/release-') - needs: [docker-build-ii, docker-build-archive] + needs: [docker-build-internet_identity_production, docker-build-archive] steps: - uses: actions/checkout@v3 @@ -629,7 +692,7 @@ jobs: clean-build: runs-on: ${{ matrix.os }} - needs: docker-build-ii + needs: [docker-build-internet_identity_production, docker-build-ii] strategy: matrix: # On main, we run the checks across all platforms. On other branches, in order to speed up checks (on PRs) we skip most platforms @@ -639,11 +702,25 @@ jobs: os: ${{ github.ref == 'refs/heads/main' && fromJson('[ "ubuntu-22.04", "ubuntu-20.04", "macos-11", "macos-12" ]') || fromJson('[ "ubuntu-22.04" ]') }} steps: - uses: actions/checkout@v3 - - name: 'Download wasm.gz' + - name: 'Download production build wasm.gz' uses: actions/download-artifact@v3 with: name: internet_identity_production.wasm.gz path: . + - name: 'Download dev build II wasm.gz' + uses: actions/download-artifact@v3 + with: + name: internet_identity_dev.wasm.gz + path: . + + - name: "Create dfx metadata for the dfx deps feature" + id: dfx-metadata + run: | + sha256="$(shasum -a 256 ./internet_identity_dev.wasm.gz | cut -d ' ' -f1)" + dfx_metadata_json="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz --wasm-hash $sha256)" + echo "using dfx metadata $dfx_metadata_json" + echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT" + - run: | sha256=$(shasum -a 256 ./internet_identity_production.wasm.gz | cut -d ' ' -f1) echo "sha256=$sha256" >> "$GITHUB_OUTPUT" @@ -653,6 +730,7 @@ jobs: with: # we check that ubuntu builds match the docker build sha256: ${{ startsWith(matrix.os, 'ubuntu') && steps.sha256.outputs.sha256 || '' }} + dfx-metadata: ${{ steps.dfx-metadata.outputs.metadata }} interface-compatibility: runs-on: ubuntu-latest diff --git a/.github/workflows/release-build-check.yml b/.github/workflows/release-build-check.yml index bc1f8fd76b..ea5b53f7cd 100644 --- a/.github/workflows/release-build-check.yml +++ b/.github/workflows/release-build-check.yml @@ -31,11 +31,15 @@ jobs: exit 1 fi curl --silent -SL "https://github.com/dfinity/internet-identity/releases/download/$latest_release_ref/internet_identity_production.wasm.gz" -o internet_identity_previous.wasm.gz - latest_release_sha256=$(shasum -a 256 ./internet_identity_previous.wasm.gz | cut -d ' ' -f1) + curl --silent -SL "https://github.com/dfinity/internet-identity/releases/download/$latest_release_ref/internet_identity_dev.wasm.gz" -o internet_identity_dev.wasm.gz + latest_prod_release_sha256=$(shasum -a 256 ./internet_identity_previous.wasm.gz | cut -d ' ' -f1) + latest_dev_release_sha256=$(shasum -a 256 ./internet_identity_previous.wasm.gz | cut -d ' ' -f1) echo latest release is "$latest_release_ref" - echo latest release sha256 is "$latest_release_sha256" + echo latest prod release sha256 is "$latest_prod_release_sha256" + echo latest dev release sha256 is "$latest_dev_release_sha256" echo "ref=$latest_release_ref" >> "$GITHUB_OUTPUT" - echo "sha256=$latest_release_sha256" >> "$GITHUB_OUTPUT" + echo "prod_sha256=$latest_prod_release_sha256" >> "$GITHUB_OUTPUT" + echo "dev_sha256=$latest_dev_release_sha256" >> "$GITHUB_OUTPUT" id: release # Then perform the build, using the release as checkout @@ -50,10 +54,18 @@ jobs: with: ref: "refs/tags/${{ needs.latest-release.outputs.ref }}" + - name: "Create dfx metadata" + id: dfx-metadata + run: | + dfx_metadata_json="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz --wasm-hash ${{ needs.latest-release.outputs.dev_sha256 }})" + echo "using dfx metadata $dfx_metadata_json" + echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT" + - uses: ./.github/actions/check-build with: # we check that ubuntu builds match the latest release build - sha256: ${{ startsWith(matrix.os, 'ubuntu') && needs.latest-release.outputs.sha256 || '' }} + sha256: ${{ startsWith(matrix.os, 'ubuntu') && needs.latest-release.outputs.prod_sha256 || '' }} + dfx-metadata: ${{ steps.dfx-metadata.outputs.metadata }} # Since the release build check is a scheduled job, a failure won't be shown on any # PR status. To notify the team, we send a message to our Slack channel on failure. diff --git a/Dockerfile b/Dockerfile index 1291774c9e..f7929ce987 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,12 +73,15 @@ ARG II_DUMMY_CAPTCHA= ARG II_DUMMY_AUTH= ARG II_INSECURE_REQUESTS= +# DFX specific metadata for dfx deps +ARG DFX_METADATA= + RUN touch src/internet_identity/src/lib.rs RUN touch src/internet_identity_interface/src/lib.rs RUN touch src/canister_tests/src/lib.rs RUN npm ci -RUN ./scripts/build +RUN ./scripts/build ${DFX_METADATA:+"--dfx-metadata" "$DFX_METADATA"} RUN sha256sum /internet_identity.wasm.gz FROM deps as build_archive diff --git a/scripts/build b/scripts/build index 7f85456139..4fbfb550c9 100755 --- a/scripts/build +++ b/scripts/build @@ -18,12 +18,13 @@ function usage() { cat << EOF Usage: - $0 [--only-dependencies] [--internet-identity] [--archive] + $0 [--only-dependencies] [--internet-identity] [--archive] [--dfx-metadata METADATA] Options: --only-dependencies only build rust dependencies (no js build, no wasm optimization) --internet-identity build the internet_identity canister (alongside other specifically mentioned canisters), defaults to --internet-identity --archive build the archive canister (alongside other specifically mentioned canisters), defaults to --internet-identity + --dfx-metadata METADATA DFX metadata to include in the canister public metadata section EOF } @@ -39,6 +40,7 @@ EOF ONLY_DEPS= CANISTERS=() +DFX_METADATA= while [[ $# -gt 0 ]] do @@ -61,6 +63,11 @@ do CANISTERS+=("archive") shift ;; + --dfx-metadata) + DFX_METADATA="${2:?missing value for '--dfx-metadata'}" + shift; # shift past --dfx-metadata and value + shift; + ;; *) echo "ERROR: unknown argument $1" usage @@ -129,8 +136,20 @@ function build_canister() { -o "./$canister.wasm" \ shrink ic-wasm "$canister.wasm" -o "$canister.wasm" metadata candid:service -f "$SRC_DIR/$canister.did" -v public + # indicate support for certificate version 1 and 2 in the canister metadata ic-wasm "$canister.wasm" -o "$canister.wasm" metadata supported_certificate_versions -d "1,2" -v public + + if [ "$canister" == "internet_identity" ] + then + # indicate the II canister init argument type + ic-wasm "$canister.wasm" -o "$canister.wasm" metadata candid:args -d "(opt InternetIdentityInit)" -v public + fi + + if [ -n "$DFX_METADATA" ] + then + ic-wasm "$canister.wasm" -o "$canister.wasm" metadata dfx -d "$DFX_METADATA" -v public + fi gzip --no-name --force "$canister.wasm" fi } diff --git a/scripts/dfx-metadata b/scripts/dfx-metadata new file mode 100755 index 0000000000..41c1d4fba0 --- /dev/null +++ b/scripts/dfx-metadata @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Make sure we always run from the root +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPTS_DIR/.." + +######### +# USAGE # +######### + +function title() { + echo "Assemble DFX metadata" +} + +function usage() { + cat << EOF + +Usage: + $0 --asset-name ASSET_NAME [--wasm-hash SHA256] + +Options: + --asset-name ASSET_NAME Name of the asset to be used in the metadata. + --wasm-hash SHA256 Optional: SHA256 hash of the II wasm pointed at by the wasm_url. + It is only needed if the wasm_url is pointing to a different wasm. +EOF +} + +function help() { + cat << EOF + +Assembles the dfx metadata to be put into the II canister wasm. +If the current build is not a release build (and no alternative value is supplied using the II_RELEASE env variable), +this will return an empty string, as no valid metadata can be generated for builds that are not published. +EOF +} + +WASM_SHA256= +ASSET_NAME= + +while [[ $# -gt 0 ]] +do + case "$1" in + -h|--help) + title + usage + help + exit 0 + ;; + --asset-name) + ASSET_NAME="${2:?missing value for '--asset-name'}" + shift; # shift past --asset-name and value + shift; + ;; + --wasm-hash) + WASM_SHA256="${2:?missing value for '--wasm-hash'}" + shift; # shift past --wasm-hash and value + shift; + ;; + *) + echo "ERROR: unknown argument $1" + usage + echo + echo "Use 'build --help' for more information" + exit 1 + ;; + esac +done + +if [ -z "$ASSET_NAME" ] +then + echo "no asset name provided" + usage + exit 1 +fi + +IFS=, +read -r -a version_parts <<< "$(./scripts/version)" +release="${II_RELEASE:-${version_parts[1]}}" + +if [ -z "$release" ] +then + echo -n "" + exit 0 +fi + +metadata_json="{\"pullable\":{\ +\"wasm_url\": \"https://github.com/dfinity/internet-identity/releases/download/$release/$ASSET_NAME\",\ +${WASM_SHA256:+"\"wasm_hash\":" "\"$WASM_SHA256\","}\ +\"dependencies\": [],\ +\"init_guide\": \"Use '(null)' for sensible defaults. See the candid interface for more details.\"}}" + +echo -n "$metadata_json" diff --git a/scripts/docker-build b/scripts/docker-build index 4779aa4c37..af01c3653f 100755 --- a/scripts/docker-build +++ b/scripts/docker-build @@ -17,11 +17,12 @@ function usage() { cat << EOF Usage: - $0 [--internet-identity] [--archive] + $0 [--internet-identity] [--archive] [--dfx-metadata METADATA] Options: --internet-identity build the internet_identity canister (alongside other specifically mentioned canisters), defaults to --internet-identity --archive build the archive canister (alongside other specifically mentioned canisters), defaults to --internet-identity + --dfx-metadata METADATA DFX metadata to include in the canister public metadata section Environment: II_FETCH_ROOT_KEY When set to "1", enable the "II_FETCH_ROOT_KEY" feature. @@ -61,6 +62,7 @@ function check_feature() { # CANISTER: possible values: [internet_identity, archive] function build() { local canister="$1" + local dfx_metadata="$2" # image name and build args, made global because they're used in # check_feature() @@ -84,6 +86,7 @@ function build() { set -x DOCKER_BUILDKIT=1 docker build \ --build-arg II_VERSION="$version" \ + --build-arg DFX_METADATA="$dfx_metadata" \ "${docker_build_args[@]}" \ --output "$tmp_outdir" set +x @@ -98,6 +101,7 @@ function build() { # ARGUMENT PARSING CANISTERS=() +DFX_METADATA= while [[ $# -gt 0 ]] do case $1 in @@ -115,6 +119,11 @@ do CANISTERS+=("archive") shift ;; + --dfx-metadata) + DFX_METADATA="${2:?missing value for '--dfx-metadata'}" + shift; # shift past --dfx-metadata and value + shift; + ;; *) echo "ERROR: unknown argument $1" usage @@ -132,5 +141,5 @@ fi for canister in "${CANISTERS[@]}" do - build "$canister" + build "$canister" "$DFX_METADATA" done diff --git a/scripts/verify-hash b/scripts/verify-hash new file mode 100755 index 0000000000..5fec61990c --- /dev/null +++ b/scripts/verify-hash @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Make sure we always run from the root +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPTS_DIR/.." + + +declare expected_ii_hash= +declare expected_archive_hash= +GREEN=$(tput setaf 2) +RED=$(tput setaf 1) +NC=$(tput sgr0) + +print_red() { + echo "${RED}$*${NC}" +} + +print_green() { + echo "${GREEN}$*${NC}" +} + +######### +# USAGE # +######### + +function title() { + echo "Verifies the hash of the II and archive wasm files" +} + +function usage() { + cat << EOF + +Usage: + $0 --ii-hash II_HASH [--archive-hash ARCHIVE_HASH] + +Options: + --ii-hash HASH Expected II SHA256 hash of the II wasm. + --archive-hash ARCHIVE_HASH Optional archive hash, if the archive needs to be verified as well. +EOF +} + +function help() { + cat << EOF + +Builds the II canister (and optionally also the archive) and compares the hash of the built wasm with the expected hash. +EOF +} + +# ARGUMENT PARSING + +while [[ $# -gt 0 ]] +do + case $1 in + --help) + title + usage + help + exit 0 + ;; + --ii-hash) + expected_ii_hash="${2:?missing value for '--ii-hash'}" + shift; # shift past --ii-hash and value + shift; + ;; + --archive-hash) + expected_archive_hash="${2:?missing value for '--archive-hash'}" + shift; # shift past --archive-hash and value + shift; + ;; + *) + echo "ERROR: unknown argument $1" + usage + echo + echo "Use 'release --help' for more information." + exit 1 + ;; + esac +done + +if [ -z "$expected_ii_hash" ] +then + echo no II hash provided + usage + exit 1 +fi + +# dev build +dfx_metadata_dev="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz)" +II_FETCH_ROOT_KEY=1 II_DUMMY_CAPTCHA=1 II_DUMMY_AUTH=1 II_INSECURE_REQUESTS=1 ./scripts/docker-build ${dfx_metadata_dev:+"--dfx-metadata" "$dfx_metadata_dev"} +dev_build_sha256="$(shasum -a 256 ./internet_identity.wasm.gz | cut -d ' ' -f1)" +# delete the dev wasm file +rm ./internet_identity.wasm.gz + +# prod build +dfx_metadata_prod="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz --wasm-hash "$dev_build_sha256")" +./scripts/docker-build ${dfx_metadata_prod:+"--dfx-metadata" "$dfx_metadata_prod"} +prod_build_sha256="$(shasum -a 256 ./internet_identity.wasm.gz | cut -d ' ' -f1)" + +if [ "$prod_build_sha256" == "$expected_ii_hash" ] +then + print_green "internet_identity.wasm.gz sha256 matches expected hash $expected_ii_hash" +else + print_red "sha mismatch: $prod_build_sha256 /= $expected_ii_hash" + exit 1 +fi + +if [ -n "$expected_archive_hash" ] +then + ./scripts/docker-build.sh --archive + archive_sha256="$(shasum -a 256 ./archive.wasm.gz | cut -d ' ' -f1)" + if [ "$archive_sha256" == "$expected_archive_hash" ] + then + print_green "archive.wasm.gz sha256 matches expected hash $expected_archive_hash" + else + print_red "sha mismatch: $archive_sha256 /= $expected_archive_hash" + exit 1 + fi +fi + +print_green "Wasm verification successful!"