Merge remote-tracking branch 'origin/master' into feat/add-hooks-for-…

…utility-batch
paritytech · Jan 13, 2025 · d3dd6fe · d3dd6fe
2 parents 6db7778 + dc6e5f7
commit d3dd6fe
Show file tree

Hide file tree

Showing 562 changed files with 17,204 additions and 8,919 deletions.
diff --git a/.config/nextest.toml b/.config/nextest.toml
@@ -21,7 +21,6 @@ retries = 5
 # The number of threads to run tests with. Supported values are either an integer or
 # the string "num-cpus". Can be overridden through the `--test-threads` option.
 # test-threads = "num-cpus"
-
 test-threads = 20
 
 # The number of threads required for each test. This is generally used in overrides to

diff --git a/.github/env b/.github/env
@@ -1 +1 @@
-IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-11-19-v202411281558"
diff --git a/.github/scripts/common/lib.sh b/.github/scripts/common/lib.sh
@@ -237,24 +237,52 @@ fetch_release_artifacts() {
   popd > /dev/null
 }
 
-# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# Fetch deb package from S3. Assumes the ENV are set:
 # - RELEASE_ID
 # - GITHUB_TOKEN
 # - REPO in the form paritytech/polkadot
-fetch_release_artifacts_from_s3() {
+fetch_debian_package_from_s3() {
   BINARY=$1
   echo "Version    : $VERSION"
   echo "Repo       : $REPO"
   echo "Binary     : $BINARY"
+  echo "Tag        : $RELEASE_TAG"
   OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
   echo "OUTPUT_DIR : $OUTPUT_DIR"
 
   URL_BASE=$(get_s3_url_base $BINARY)
   echo "URL_BASE=$URL_BASE"
 
-  URL_BINARY=$URL_BASE/$VERSION/$BINARY
-  URL_SHA=$URL_BASE/$VERSION/$BINARY.sha256
-  URL_ASC=$URL_BASE/$VERSION/$BINARY.asc
+  URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb
+
+  mkdir -p "$OUTPUT_DIR"
+  pushd "$OUTPUT_DIR" > /dev/null
+
+  echo "Fetching deb package..."
+
+  echo "Fetching %s" "$URL"
+  curl --progress-bar -LO "$URL" || echo "Missing $URL"
+
+  pwd
+  ls -al --color
+  popd > /dev/null
+
+}
+
+# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# inputs: binary (polkadot), target(aarch64-apple-darwin)
+fetch_release_artifacts_from_s3() {
+  BINARY=$1
+  TARGET=$2
+  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${TARGET}/${BINARY}"}
+  echo "OUTPUT_DIR : $OUTPUT_DIR"
+
+  URL_BASE=$(get_s3_url_base $BINARY)
+  echo "URL_BASE=$URL_BASE"
+
+  URL_BINARY=$URL_BASE/$VERSION/$TARGET/$BINARY
+  URL_SHA=$URL_BASE/$VERSION/$TARGET/$BINARY.sha256
+  URL_ASC=$URL_BASE/$VERSION/$TARGET/$BINARY.asc
 
   # Fetch artifacts
   mkdir -p "$OUTPUT_DIR"
@@ -269,23 +297,34 @@ fetch_release_artifacts_from_s3() {
   pwd
   ls -al --color
   popd > /dev/null
-
+  unset OUTPUT_DIR
 }
 
 # Pass the name of the binary as input, it will
 # return the s3 base url
 function get_s3_url_base() {
     name=$1
     case $name in
-    polkadot | polkadot-execute-worker | polkadot-prepare-worker | staking-miner)
+      polkadot | polkadot-execute-worker | polkadot-prepare-worker )
         printf "https://releases.parity.io/polkadot"
         ;;
 
-    polkadot-parachain)
-        printf "https://releases.parity.io/cumulus"
+      polkadot-parachain)
+        printf "https://releases.parity.io/polkadot-parachain"
+        ;;
+
+      polkadot-omni-node)
+        printf "https://releases.parity.io/polkadot-omni-node"
         ;;
 
-    *)
+      chain-spec-builder)
+        printf "https://releases.parity.io/chain-spec-builder"
+        ;;
+
+      frame-omni-bencher)
+        printf "https://releases.parity.io/frame-omni-bencher"
+        ;;
+      *)
         printf "UNSUPPORTED BINARY $name"
         exit 1
         ;;
@@ -468,3 +507,16 @@ validate_stable_tag() {
         exit 1
     fi
 }
+
+# Prepare docker stable tag form the polkadot stable tag
+# input: tag (polkaodot-stableYYMM(-X) or polkadot-stableYYMM(-X)-rcX)
+# output: stableYYMM(-X) or stableYYMM(-X)-rcX
+prepare_docker_stable_tag() {
+  tag="$1"
+  if [[ "$tag" =~ stable[0-9]{4}(-[0-9]+)?(-rc[0-9]+)? ]]; then
+      echo "${BASH_REMATCH[0]}"
+  else
+      echo "Tag is invalid: $tag"
+      exit 1
+  fi
+}
diff --git a/.github/scripts/release/build-linux-release.sh b/.github/scripts/release/build-linux-release.sh
@@ -3,6 +3,8 @@
 # This is used to build our binaries:
 # - polkadot
 # - polkadot-parachain
+# - polkadot-omni-node 
+#
 # set -e
 
 BIN=$1
@@ -21,7 +23,7 @@ time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PAC
 echo "Artifact target: $ARTIFACTS"
 
 cp ./target/$PROFILE/$BIN "$ARTIFACTS"
-pushd "$ARTIFACTS" > /dev/nul
+pushd "$ARTIFACTS" > /dev/null
 sha256sum "$BIN" | tee "$BIN.sha256"
 
 EXTRATAG="$($ARTIFACTS/$BIN --version |

diff --git a/.github/scripts/release/build-macos-release.sh b/.github/scripts/release/build-macos-release.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# This is used to build our binaries:
+# - polkadot
+# - polkadot-parachain
+# - polkadot-omni-node
+# set -e
+
+BIN=$1
+PACKAGE=${2:-$BIN}
+
+PROFILE=${PROFILE:-production}
+# parity-macos runner needs a path where it can
+# write, so make it relative to github workspace.
+ARTIFACTS=$GITHUB_WORKSPACE/artifacts/$BIN
+VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
+
+echo "Artifacts will be copied into $ARTIFACTS"
+mkdir -p "$ARTIFACTS"
+
+git log --pretty=oneline -n 1
+time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE
+
+echo "Artifact target: $ARTIFACTS"
+
+cp ./target/$PROFILE/$BIN "$ARTIFACTS"
+pushd "$ARTIFACTS" > /dev/null
+sha256sum "$BIN" | tee "$BIN.sha256"
+
+EXTRATAG="$($ARTIFACTS/$BIN --version |
+    sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+
+EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
+
+echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+echo -n ${VERSION} > "$ARTIFACTS/VERSION"
+echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"
diff --git a/.github/scripts/release/distributions b/.github/scripts/release/distributions
@@ -0,0 +1,39 @@
+Origin: Parity
+Label: Parity
+Codename: release
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity Staging
+Codename: staging
+Architectures: amd64
+Components: main
+Description: Staging distribution for Parity Technologies Ltd. packages
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2407
+Codename: stable2407
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2409
+Codename: stable2409
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2412
+Codename: stable2412
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
diff --git a/.github/scripts/release/release_lib.sh b/.github/scripts/release/release_lib.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Set the new version by replacing the value of the constant given as patetrn
+# Set the new version by replacing the value of the constant given as pattern
 # in the file.
 #
 # input: pattern, version, file
@@ -119,21 +119,79 @@ set_polkadot_parachain_binary_version() {
 
 
 upload_s3_release() {
+    alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
+
+    product=$1
+    version=$2
+    target=$3
+
+    echo "Working on product:  $product "
+    echo "Working on version:  $version "
+    echo "Working on platform: $target "
+
+    URL_BASE=$(get_s3_url_base $product)
+
+    echo "Current content, should be empty on new uploads:"
+    aws s3 ls "s3://${URL_BASE}/${version}/${target}" --recursive --human-readable --summarize || true
+    echo "Content to be uploaded:"
+    artifacts="release-artifacts/$target/$product/"
+    ls "$artifacts"
+    aws s3 sync --acl public-read "$artifacts" "s3://${URL_BASE}/${version}/${target}"
+    echo "Uploaded files:"
+    aws s3 ls "s3://${URL_BASE}/${version}/${target}" --recursive --human-readable --summarize
+    echo "✅ The release should be at https://${URL_BASE}/${version}/${target}"
+}
+
+# Upload runtimes artifacts to s3 release bucket
+#
+# input: version (stable release tage.g. polkadot-stable2412 or polkadot-stable2412-rc1)
+# output: none
+upload_s3_runtimes_release_artifacts() {
   alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
 
-  product=$1
-  version=$2
+  version=$1
 
-  echo "Working on product: $product "
   echo "Working on version: $version "
 
   echo "Current content, should be empty on new uploads:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize || true
+  aws s3 ls "s3://releases.parity.io/polkadot/runtimes/${version}/" --recursive --human-readable --summarize || true
   echo "Content to be uploaded:"
-  artifacts="artifacts/$product/"
+  artifacts="artifacts/runtimes/"
   ls "$artifacts"
-  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/${version}/"
+  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/runtimes/${version}/"
   echo "Uploaded files:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize
-  echo "✅ The release should be at https://releases.parity.io/polkadot/${version}"
+  aws s3 ls "s3://releases.parity.io/polkadot/runtimes/${version}/" --recursive --human-readable --summarize
+  echo "✅ The release should be at https://releases.parity.io/polkadot/runtimes/${version}"
+}
+
+
+# Pass the name of the binary as input, it will
+# return the s3 base url
+function get_s3_url_base() {
+    name=$1
+    case $name in
+      polkadot | polkadot-execute-worker | polkadot-prepare-worker )
+        printf "releases.parity.io/polkadot"
+        ;;
+
+      polkadot-parachain)
+        printf "releases.parity.io/polkadot-parachain"
+        ;;
+
+      polkadot-omni-node)
+        printf "releases.parity.io/polkadot-omni-node"
+        ;;
+
+      chain-spec-builder)
+        printf "releases.parity.io/chain-spec-builder"
+        ;;
+
+      frame-omni-bencher)
+        printf "releases.parity.io/frame-omni-bencher"
+        ;;
+      *)
+        printf "UNSUPPORTED BINARY $name"
+        exit 1
+        ;;
+    esac
 }
diff --git a/.github/workflows/build-misc.yml b/.github/workflows/build-misc.yml
@@ -44,7 +44,7 @@ jobs:
           forklift cargo check -p rococo-runtime
           forklift cargo check -p polkadot-test-runtime
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
@@ -73,7 +73,7 @@ jobs:
           cd ./substrate/bin/utils/subkey
           forklift cargo build --locked --release
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}

diff --git a/.github/workflows/check-frame-omni-bencher.yml b/.github/workflows/check-frame-omni-bencher.yml
@@ -41,7 +41,7 @@ jobs:
           forklift cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
           forklift cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
@@ -99,7 +99,7 @@ jobs:
           echo "Running command: $cmd"
           eval "$cmd"
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}

diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml
@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.0 (22. Sep 2023)
 
       - name: Lychee link checker
-        uses: lycheeverse/lychee-action@7cd0af4c74a61395d455af97419279d86aafaede # for v1.9.1 (10. Jan 2024)
+        uses: lycheeverse/lychee-action@f81112d0d2814ded911bd23e3beaa9dda9093915 # for v1.9.1 (10. Jan 2024)
         with:
           args: >-
             --config .config/lychee.toml

diff --git a/.github/workflows/check-semver.yml b/.github/workflows/check-semver.yml
@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TOOLCHAIN: nightly-2024-06-01
+  TOOLCHAIN: nightly-2024-11-19
 
 jobs:
   preflight:
@@ -74,10 +74,15 @@ jobs:
 
       - name: install parity-publish
         # Set the target dir to cache the build.
-        run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.8.0 --locked -q
+        run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.10.2 --locked -q
 
       - name: check semver
         run: |
+          if [ -z "$PR" ]; then
+            echo "Skipping master/merge queue"
+            exit 0
+          fi
+
           export CARGO_TARGET_DIR=target
           export RUSTFLAGS='-A warnings -A missing_docs'
           export SKIP_WASM_BUILD=1
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
		IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-11-19-v202411281558"