Add support for dfx deps (#1798)

* Add support for dfx pull This PR adds support for DFX pull. Supporting dfx pull creates a dependency between the dev build and the prod build, because the wasm metadata needs to include the dev build hash. Furthermore, it is not only the prod_build but also the dev build that needs to include additional metadata. To make the feature work this PR: - changes the build order of the wasm modules so that the prod buid is built after the dev build to include the dev build hash - all II wasm files include additional metadata, _if_ the commit corresponds to a release (since no valid metadata can be generated for non-released wasms) - a script has been added to generate the above metadata - a script for wasm hash verification has been added since verification of wasm hashes is no more difficult (as it entails building II twice and including the hash of one module in the other) Future work: - Refactor build scripts to separate building the wasm from adding metadata. This would allow to build all wasms in parallel again, shortening CI times. - Try to move to `dfx` for wasm post-processing. This would remove a substantial amount of scripts / logic from our build process. * Adjust metadata init type and guide * Fix shell expansion issues and improve init guide * Improve comment * Update release build check with metadata * Implement review feedback Changes: - Change `dfx pull` to `dfx deps` - Fix a misleading comment (including the location I copied it from) - Introduced helper functions to print in color - Fix typos
dfinity · Aug 24, 2023 · 5ed4fc3 · 5ed4fc3
1 parent 73e6ad3
commit 5ed4fc3
Show file tree

Hide file tree

Showing 9 changed files with 369 additions and 26 deletions.
diff --git a/.github/actions/check-build/action.yml b/.github/actions/check-build/action.yml
@@ -2,7 +2,10 @@ name: 'Check build'
 description: This action performs a clean, non-Docker build of II, and optionally checks the gzipped Wasm module sha256 against the 'sha256' argument. Nothing is cached except for the bootstrap environment.
 inputs:
   sha256:
-    description: The expected sha256 of the final Wasm module
+    description: The expected sha256 of the final production Wasm module
+    required: false
+  dfx-metadata:
+    description: The dfx metadata to include in the production build
     required: false
 runs:
   using: "composite"
@@ -12,7 +15,13 @@ runs:
     # run the build
     - run: npm ci
       shell: bash
-    - run: ./scripts/build
+    - name: Run build
+      env:
+        # use an env variable so that the GitHub templating does not cause issues with string escaping
+        # see: https://github.com/orgs/community/discussions/32012
+        METADATA: ${{ inputs.dfx-metadata }}
+      run: |
+        ./scripts/build ${METADATA:+"--dfx-metadata" "$METADATA"}
       shell: bash
 
     # check the hash

diff --git a/.github/actions/release/run.sh b/.github/actions/release/run.sh
@@ -38,10 +38,7 @@ To build the wasm modules yourself and verify their hashes, run the following co
 \`\`\`
 git pull # to ensure you have the latest changes.
 git checkout $GITHUB_SHA
-./scripts/docker-build
-sha256sum internet_identity.wasm.gz
-./scripts/docker-build --archive
-sha256sum archive.wasm.gz
+./scripts/verify-hash --ii-hash $(shasum -a 256 "$PRODUCTION_ASSET" | cut -d ' ' -f1)
 \`\`\`
 EOF
 

diff --git a/.github/workflows/canister-tests.yml b/.github/workflows/canister-tests.yml
@@ -45,12 +45,8 @@ jobs:
       # NOTE: if you modify the flavors, update the #flavors table in README.md
       matrix:
         include:
-          # The production build
-          - name: internet_identity_production.wasm.gz
-            II_FETCH_ROOT_KEY: 0
-            II_DUMMY_CAPTCHA: 0
-            II_DUMMY_AUTH: 0
-            II_INSECURE_REQUESTS: 0
+          # The production build is built later because it has a dependency on the dev build (for dfx deps)
+          # See job: docker-build-internet_identity_production
 
           # No captcha and fetching the root key, used in (our) tests, backend and
           # selenium.
@@ -78,6 +74,13 @@ jobs:
           echo "Inferred version: '$version'"
           echo "version=$version" >> "$GITHUB_OUTPUT"
 
+      - name: "Create dfx metadata for the dfx deps feature"
+        id: dfx-metadata
+        run: |
+          dfx_metadata_json="$(./scripts/dfx-metadata --asset-name ${{ matrix.name }} )"
+          echo "using dfx metadata $dfx_metadata_json"
+          echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT"
+
       - name: Set up docker buildx
         uses: docker/setup-buildx-action@v2
 
@@ -92,6 +95,7 @@ jobs:
             II_DUMMY_CAPTCHA=${{ matrix.II_DUMMY_CAPTCHA }}
             II_INSECURE_REQUESTS=${{ matrix.II_INSECURE_REQUESTS }}
             II_VERSION=${{ steps.version.outputs.version }}
+            DFX_METADATA=${{ steps.dfx-metadata.outputs.metadata }}
           cache-from: type=gha,scope=cached-stage
           # Exports the artefacts from the final stage
           outputs: ./out
@@ -105,9 +109,68 @@ jobs:
           # name is the name used to display and retrieve the artifact
           name: ${{ matrix.name }}
           # path is the name used as the file to upload and the name of the
-          # downloaded file
+          # file when downloaded
           path: ${{ matrix.name }}
 
+  # Build the production version of internet identity.
+  # The production build is separately because it has a dependency on the dev build (for dfx deps)
+  #
+  # Note: do not rename this job as it needs to contain the file name of the produced asset (without extension)
+  # in order for the release script action to work correctly.
+  docker-build-internet_identity_production:
+    runs-on: ubuntu-latest
+    needs: docker-build-ii
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Infer version
+        id: version
+        run: |
+          version="$(./scripts/version)"
+          echo "Inferred version: '$version'"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
+      - name: 'Download dev build II wasm.gz'
+        uses: actions/download-artifact@v3
+        with:
+          name: internet_identity_dev.wasm.gz
+          path: .
+
+      - name: "Create dfx metadata for the dfx deps feature"
+        id: dfx-metadata
+        run: |
+          sha256="$(shasum -a 256 ./internet_identity_dev.wasm.gz | cut -d ' ' -f1)"
+          dfx_metadata_json="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz --wasm-hash $sha256)"
+          echo "using dfx metadata $dfx_metadata_json"
+          echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT"
+
+      - name: Set up docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build internet_identity_production.wasm.gz
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: Dockerfile
+          build-args: |
+            II_VERSION=${{ steps.version.outputs.version }}
+            DFX_METADATA=${{ steps.dfx-metadata.outputs.metadata }}
+          cache-from: type=gha,scope=cached-stage
+          # Exports the artefacts from the final stage
+          outputs: ./out
+          target: scratch_internet_identity
+
+      - run: mv out/internet_identity.wasm.gz internet_identity_production.wasm.gz
+      - run: sha256sum internet_identity_production.wasm.gz
+      - name: 'Upload internet_identity_production.wasm.gz'
+        uses: actions/upload-artifact@v3
+        with:
+          # name is the name used to display and retrieve the artifact
+          name: internet_identity_production.wasm.gz
+          # path is the name used as the file to upload and the name of the
+          # file when downloaded
+          path: internet_identity_production.wasm.gz
+
   docker-build-archive:
     runs-on: ubuntu-latest
     needs: docker-build-base
@@ -140,7 +203,7 @@ jobs:
 
   wasm-size:
     runs-on: ubuntu-latest
-    needs: docker-build-ii
+    needs: docker-build-internet_identity_production
     steps:
       - uses: actions/checkout@v3
       - name: 'Download wasm'
@@ -487,7 +550,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/release-')
-    needs: [docker-build-ii, docker-build-archive]
+    needs: [docker-build-internet_identity_production, docker-build-archive]
     steps:
       - uses: actions/checkout@v3
 
@@ -531,7 +594,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/release-')
-    needs: [docker-build-ii, docker-build-archive]
+    needs: [docker-build-internet_identity_production, docker-build-archive]
 
     steps:
       - uses: actions/checkout@v3
@@ -629,7 +692,7 @@ jobs:
 
   clean-build:
     runs-on: ${{ matrix.os }}
-    needs: docker-build-ii
+    needs: [docker-build-internet_identity_production, docker-build-ii]
     strategy:
       matrix:
         # On main, we run the checks across all platforms. On other branches, in order to speed up checks (on PRs) we skip most platforms
@@ -639,11 +702,25 @@ jobs:
         os: ${{ github.ref == 'refs/heads/main' && fromJson('[ "ubuntu-22.04", "ubuntu-20.04", "macos-11", "macos-12" ]') || fromJson('[ "ubuntu-22.04" ]') }}
     steps:
       - uses: actions/checkout@v3
-      - name: 'Download wasm.gz'
+      - name: 'Download production build wasm.gz'
         uses: actions/download-artifact@v3
         with:
           name: internet_identity_production.wasm.gz
           path: .
+      - name: 'Download dev build II wasm.gz'
+        uses: actions/download-artifact@v3
+        with:
+          name: internet_identity_dev.wasm.gz
+          path: .
+
+      - name: "Create dfx metadata for the dfx deps feature"
+        id: dfx-metadata
+        run: |
+          sha256="$(shasum -a 256 ./internet_identity_dev.wasm.gz | cut -d ' ' -f1)"
+          dfx_metadata_json="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz --wasm-hash $sha256)"
+          echo "using dfx metadata $dfx_metadata_json"
+          echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT"
+
       - run: |
           sha256=$(shasum -a 256 ./internet_identity_production.wasm.gz | cut -d ' ' -f1)
           echo "sha256=$sha256" >> "$GITHUB_OUTPUT"
@@ -653,6 +730,7 @@ jobs:
         with:
           # we check that ubuntu builds match the docker build
           sha256: ${{ startsWith(matrix.os, 'ubuntu') && steps.sha256.outputs.sha256 || '' }}
+          dfx-metadata: ${{ steps.dfx-metadata.outputs.metadata }}
 
   interface-compatibility:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/release-build-check.yml b/.github/workflows/release-build-check.yml
@@ -31,11 +31,15 @@ jobs:
             exit 1
           fi
           curl --silent -SL "https://github.com/dfinity/internet-identity/releases/download/$latest_release_ref/internet_identity_production.wasm.gz" -o internet_identity_previous.wasm.gz
-          latest_release_sha256=$(shasum -a 256 ./internet_identity_previous.wasm.gz | cut -d ' ' -f1)
+          curl --silent -SL "https://github.com/dfinity/internet-identity/releases/download/$latest_release_ref/internet_identity_dev.wasm.gz" -o internet_identity_dev.wasm.gz
+          latest_prod_release_sha256=$(shasum -a 256 ./internet_identity_previous.wasm.gz | cut -d ' ' -f1)
+          latest_dev_release_sha256=$(shasum -a 256 ./internet_identity_previous.wasm.gz | cut -d ' ' -f1)
           echo latest release is "$latest_release_ref"
-          echo latest release sha256 is "$latest_release_sha256"
+          echo latest prod release sha256 is "$latest_prod_release_sha256"
+          echo latest dev release sha256 is "$latest_dev_release_sha256"
           echo "ref=$latest_release_ref" >> "$GITHUB_OUTPUT"
-          echo "sha256=$latest_release_sha256" >> "$GITHUB_OUTPUT"
+          echo "prod_sha256=$latest_prod_release_sha256" >> "$GITHUB_OUTPUT"
+          echo "dev_sha256=$latest_dev_release_sha256" >> "$GITHUB_OUTPUT"
         id: release
 
   # Then perform the build, using the release as checkout
@@ -50,10 +54,18 @@ jobs:
         with:
           ref: "refs/tags/${{ needs.latest-release.outputs.ref }}"
 
+      - name: "Create dfx metadata"
+        id: dfx-metadata
+        run: |
+          dfx_metadata_json="$(./scripts/dfx-metadata --asset-name internet_identity_dev.wasm.gz --wasm-hash ${{ needs.latest-release.outputs.dev_sha256 }})"
+          echo "using dfx metadata $dfx_metadata_json"
+          echo "metadata=$dfx_metadata_json" >> "$GITHUB_OUTPUT"
+
       - uses: ./.github/actions/check-build
         with:
           # we check that ubuntu builds match the latest release build
-          sha256: ${{ startsWith(matrix.os, 'ubuntu') && needs.latest-release.outputs.sha256 || '' }}
+          sha256: ${{ startsWith(matrix.os, 'ubuntu') && needs.latest-release.outputs.prod_sha256 || '' }}
+          dfx-metadata: ${{ steps.dfx-metadata.outputs.metadata }}
 
         # Since the release build check is a scheduled job, a failure won't be shown on any
         # PR status. To notify the team, we send a message to our Slack channel on failure.

diff --git a/Dockerfile b/Dockerfile
@@ -73,12 +73,15 @@ ARG II_DUMMY_CAPTCHA=
 ARG II_DUMMY_AUTH=
 ARG II_INSECURE_REQUESTS=
 
+# DFX specific metadata for dfx deps
+ARG DFX_METADATA=
+
 RUN touch src/internet_identity/src/lib.rs
 RUN touch src/internet_identity_interface/src/lib.rs
 RUN touch src/canister_tests/src/lib.rs
 RUN npm ci
 
-RUN ./scripts/build
+RUN ./scripts/build ${DFX_METADATA:+"--dfx-metadata" "$DFX_METADATA"}
 RUN sha256sum /internet_identity.wasm.gz
 
 FROM deps as build_archive

diff --git a/scripts/build b/scripts/build
@@ -18,12 +18,13 @@ function usage() {
     cat << EOF
 
 Usage:
-  $0 [--only-dependencies] [--internet-identity] [--archive]
+  $0 [--only-dependencies] [--internet-identity] [--archive] [--dfx-metadata METADATA]
 
 Options:
   --only-dependencies          only build rust dependencies (no js build, no wasm optimization)
   --internet-identity          build the internet_identity canister (alongside other specifically mentioned canisters), defaults to --internet-identity
   --archive                    build the archive canister (alongside other specifically mentioned canisters), defaults to --internet-identity
+  --dfx-metadata METADATA      DFX metadata to include in the canister public metadata section
 EOF
 }
 
@@ -39,6 +40,7 @@ EOF
 
 ONLY_DEPS=
 CANISTERS=()
+DFX_METADATA=
 
 while [[ $# -gt 0  ]]
 do
@@ -61,6 +63,11 @@ do
             CANISTERS+=("archive")
             shift
             ;;
+        --dfx-metadata)
+            DFX_METADATA="${2:?missing value for '--dfx-metadata'}"
+            shift; # shift past --dfx-metadata and value
+            shift;
+            ;;
         *)
             echo "ERROR: unknown argument $1"
             usage
@@ -129,8 +136,20 @@ function build_canister() {
             -o "./$canister.wasm" \
             shrink
         ic-wasm "$canister.wasm" -o "$canister.wasm" metadata candid:service -f "$SRC_DIR/$canister.did" -v public
+
         # indicate support for certificate version 1 and 2 in the canister metadata
         ic-wasm "$canister.wasm" -o "$canister.wasm" metadata supported_certificate_versions -d "1,2" -v public
+
+        if [ "$canister" == "internet_identity" ]
+        then
+          # indicate the II canister init argument type
+          ic-wasm "$canister.wasm" -o "$canister.wasm" metadata candid:args -d "(opt InternetIdentityInit)" -v public
+        fi
+
+        if [ -n "$DFX_METADATA" ]
+        then
+          ic-wasm "$canister.wasm" -o "$canister.wasm" metadata dfx -d "$DFX_METADATA" -v public
+        fi
         gzip --no-name --force "$canister.wasm"
     fi
 }