bench.yml

name: Bench

on:
  pull_request:
  push:
    branches:
      - main

env:
  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
  TURBO_TEAM: hashintel
  TURBO_REMOTE_ONLY: true

  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  setup:
    runs-on: ubuntu-24.04
    outputs:
      unit: ${{ steps.packages.outputs.unit }}
      integration: ${{ steps.packages.outputs.integration }}
    steps:
      - name: Checkout source code
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: 2

      - name: Install turbo
        uses: ./.github/actions/install-turbo

      - name: Determine changed packages
        id: packages
        run: |
          UNIT_BENCH_FILTER=$(turbo run bench:unit --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]')
          UNIT_BENCH_TASKS=$(sh -c "turbo run bench:unit --dry-run=json $UNIT_BENCH_FILTER" | jq -c '.tasks[]')
          UNIT_BENCH_PACKAGES=$(echo "$UNIT_BENCH_TASKS" \
            | jq 'select(.task == "bench:unit" and .command != "<NONEXISTENT>")' \
            | jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }')

          INTEGRATION_BENCH_FILTER=$(turbo run bench:integration --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]')
          INTEGRATION_BENCH_TASKS=$(sh -c "turbo run bench:integration --dry-run=json $INTEGRATION_BENCH_FILTER" | jq -c '.tasks[]')
          INTEGRATION_BENCH_PACKAGES=$(echo "$INTEGRATION_BENCH_TASKS" \
            | jq 'select(.task == "bench:integration" and .command != "<NONEXISTENT>")' \
            | jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }')

          set -x
          echo "unit=$UNIT_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT
          echo "integration=$INTEGRATION_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT

  unit-benches:
    name: Unit
    needs: [setup]
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix: ${{ fromJSON(needs.setup.outputs.unit) }}
      fail-fast: false
    if: needs.setup.outputs.unit != '{"package":[],"include":[]}'
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: 2

      - name: Install turbo
        uses: ./.github/actions/install-turbo

      - name: Find bench steps to run
        id: benches
        run: |
          if [[ "${{ github.event_name == 'pull_request' }}" == 'true' ]]; then
            echo "create-baseline=true" | tee -a $GITHUB_OUTPUT
          else
            echo "create-baseline=false" | tee -a $GITHUB_OUTPUT
          fi

          HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
          echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
          if [[ $HAS_RUST = 'true' ]]; then
            if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then
              RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml"
            else
              RUST_TOOLCHAIN_FILE="rust-toolchain.toml"
            fi
            echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
          fi

      - name: Install Rust tools
        if: steps.benches.outputs.has-rust == 'true'
        uses: taiki-e/install-action@9c04113bd63f9659d7a53908386758275ab35630 # v2.44.68
        with:
          tool: just@1.34.0,critcmp@0.1.8

      - name: Install Protobuf
        if: steps.benches.outputs.has-rust == 'true'
        run: sudo apt install protobuf-compiler

      - name: Checkout base branch
        if: steps.benches.outputs.create-baseline == 'true'
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          clean: false
          ref: ${{ github.event.pull_request.base.sha }}

      - name: Install Rust toolchain
        if: steps.benches.outputs.has-rust == 'true' && steps.benches.outputs.create-baseline == 'true'
        uses: ./.github/actions/install-rust-toolchain
        with:
          toolchain: ${{ steps.benches.outputs.rust-toolchain }}
          working-directory: ${{ matrix.directory }}

      - name: Prune repository
        if: steps.benches.outputs.create-baseline == 'true'
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.package }}
            @rust/hash-repo-chores
            @apps/hash-external-services

      - name: Warm up repository
        if: steps.benches.outputs.create-baseline == 'true'
        uses: ./.github/actions/warm-up-repo

      - name: Cache Rust dependencies
        if: steps.benches.outputs.has-rust == 'true'
        uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
        with:
          workspaces: ${{ matrix.directory }}
          save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}

      - name: Run base benches
        if: steps.benches.outputs.create-baseline == 'true'
        run: turbo run bench:unit --filter "${{ matrix.package }}"

      - name: Checkout head branch
        if: steps.benches.outputs.create-baseline == 'true'
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Install Rust toolchain
        if: steps.benches.outputs.has-rust == 'true'
        uses: ./.github/actions/install-rust-toolchain
        with:
          toolchain: ${{ steps.benches.outputs.rust-toolchain }}
          working-directory: ${{ matrix.directory }}

      - name: Prune repository
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.package }}
            @rust/hash-repo-chores
            @apps/hash-external-services

      - name: Warm up repository
        uses: ./.github/actions/warm-up-repo

      - name: Run head benches
        run: turbo run bench:unit --filter "${{ matrix.package }}"

      - name: Analyze benchmarks
        if: steps.benches.outputs.create-baseline == 'true'
        run: |
          TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')
          echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV
          echo '<details><summary><h2>${{ matrix.package }} – Units</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph
          cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md"

      - name: Authenticate Vault
        id: secrets
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          exportToken: true
          url: ${{ env.VAULT_ADDR }}
          method: jwt
          role: dev
          secrets: |
            aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID;
            aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY;
            aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN;

      - name: Upload benchmark results
        run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph

      - name: Upload benchmark summary
        if: steps.benches.outputs.create-baseline == 'true'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }}
          path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md

  integration-benches:
    name: Integration
    needs: [setup]
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix: ${{ fromJSON(needs.setup.outputs.integration) }}
      fail-fast: false
    if: needs.setup.outputs.integration != '{"package":[],"include":[]}'
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          fetch-depth: 2

      - name: Install turbo
        uses: ./.github/actions/install-turbo

      - name: Find bench steps to run
        id: benches
        run: |
          if [[ "${{ github.event_name == 'pull_request' }}" == 'true' ]]; then
            echo "create-baseline=true" | tee -a $GITHUB_OUTPUT
          else
            echo "create-baseline=false" | tee -a $GITHUB_OUTPUT
          fi

          HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
          echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
          if [[ $HAS_RUST = 'true' ]]; then
            if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then
              RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml"
            else
              RUST_TOOLCHAIN_FILE="rust-toolchain.toml"
            fi
            echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
          fi

      - name: Install Rust tools
        if: steps.benches.outputs.has-rust == 'true'
        uses: taiki-e/install-action@9c04113bd63f9659d7a53908386758275ab35630 # v2.44.68
        with:
          tool: just@1.34.0,critcmp@0.1.8

      - name: Install Protobuf
        if: steps.benches.outputs.has-rust == 'true'
        run: sudo apt install protobuf-compiler

      - name: Checkout base branch
        if: steps.benches.outputs.create-baseline == 'true'
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          clean: false
          ref: ${{ github.event.pull_request.base.sha }}

      - name: Install Rust toolchain
        if: steps.benches.outputs.has-rust == 'true' && steps.benches.outputs.create-baseline == 'true'
        uses: ./.github/actions/install-rust-toolchain
        with:
          toolchain: ${{ steps.benches.outputs.rust-toolchain }}
          working-directory: ${{ matrix.directory }}

      - name: Prune repository
        if: steps.benches.outputs.create-baseline == 'true'
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.package }}
            @rust/hash-repo-chores
            @apps/hash-external-services

      - name: Warm up repository
        if: steps.benches.outputs.create-baseline == 'true'
        uses: ./.github/actions/warm-up-repo

      - name: Cache Rust dependencies
        if: steps.benches.outputs.has-rust == 'true'
        uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
        with:
          workspaces: ${{ matrix.directory }}
          save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}

      - name: Launch external services
        if: steps.benches.outputs.create-baseline == 'true'
        run: |
          turbo codegen --filter '@apps/hash-external-services'

          SERVICES="hydra kratos redis spicedb"

          echo "Running services: $SERVICES"
          yarn workspace @apps/hash-external-services deploy:test up $SERVICES --wait

      - name: Run base benches
        if: steps.benches.outputs.create-baseline == 'true'
        run: turbo run bench:integration --filter "${{ matrix.package }}"

      - name: Tear down external services
        if: steps.benches.outputs.create-baseline == 'true'
        run: yarn workspace @apps/hash-external-services deploy:test down

      - name: Checkout head branch
        if: steps.benches.outputs.create-baseline == 'true'
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          clean: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Install Rust toolchain
        if: steps.benches.outputs.has-rust == 'true'
        uses: ./.github/actions/install-rust-toolchain
        with:
          toolchain: ${{ steps.benches.outputs.rust-toolchain }}
          working-directory: ${{ matrix.directory }}

      - name: Prune repository
        uses: ./.github/actions/prune-repository
        with:
          scope: |
            ${{ matrix.package }}
            @rust/hash-repo-chores
            @apps/hash-external-services

      - name: Warm up repository
        uses: ./.github/actions/warm-up-repo

      - name: Launch external services
        run: |
          turbo codegen --filter '@apps/hash-external-services'

          SERVICES="hydra kratos redis spicedb"

          echo "Running services: $SERVICES"
          yarn workspace @apps/hash-external-services deploy:test up $SERVICES --wait

      - name: Run head benches
        run: turbo run bench:integration --filter "${{ matrix.package }}"

      - name: Analyze benchmarks
        if: steps.benches.outputs.create-baseline == 'true'
        run: |
          TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')
          echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV
          echo '<details><summary><h2>${{ matrix.package }} – Integrations</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph
          cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
          echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md"

      - name: Authenticate Vault
        id: secrets
        if: github.event.pull_request.head.repo.full_name == github.repository
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          exportToken: true
          url: ${{ env.VAULT_ADDR }}
          method: jwt
          role: dev
          secrets: |
            aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID;
            aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY;
            aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN;

      - name: Upload benchmark results
        if: github.event.pull_request.head.repo.full_name == github.repository
        run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph

      - name: Upload benchmark summary
        if: steps.benches.outputs.create-baseline == 'true'
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }}
          path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md

      - name: Show container logs
        if: ${{ success() || failure() }}
        run: yarn workspace @apps/hash-external-services deploy logs --timestamps

  passed:
    name: Benches passed
    needs: [setup, unit-benches, integration-benches]
    if: always()
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Check setup script
        run: |
          [[ ${{ needs.setup.result }} = success ]]
      - name: Check unit benches
        run: |
          [[ ${{ needs.unit-benches.result }} =~ success|skipped ]]
      - name: Check integration benches
        run: |
          [[ ${{ needs.integration-benches.result }} =~ success|skipped ]]

      - name: Notify Slack on failure
        uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
        if: ${{ failure() && github.event_name == 'merge_group' }}
        env:
          SLACK_LINK_NAMES: true
          SLACK_MESSAGE: "At least one bench job failed for a Pull Request in the Merge Queue failed <@U02NLJY0FGX>" # Notifies T
          SLACK_TITLE: Tests failed
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_USERNAME: GitHub
          VAULT_ADDR: ""
          VAULT_TOKEN: ""

      - name: Checkout
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Download reports' artifacts
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          path: downloads

      - name: Create comment
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
        run: |
          echo "## Benchmark results" > /tmp/summary.md
          cat downloads/*/*.md >> /tmp/summary.md
          cat /tmp/summary.md

      - name: Post comment
        uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
        if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' && github.event.pull_request.head.repo.full_name == github.repository
        with:
          message-path: /tmp/summary.md
          message-id: benchmark-results
          refresh-message-position: true