diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 2ae778f..c620c83 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,28 +1,80 @@
 steps:
-  - label: ":julia: Julia: {{matrix.julia}}"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "{{matrix.julia}}"
-      - JuliaCI/julia-test#v1:
-          test_args: "--quickfail"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-          dirs:
-            - src
-    agents:
-      queue: "juliagpu"
-      cuda: "*"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 60
-    matrix:
-      setup:
-        julia:
-          - "1"
-          - "nightly"
-      adjustments:
-        - with:
-            julia: "nightly"
-          soft_fail: true
+  - group: ":julia: CUDA GPU"
+    steps:
+      - label: ":julia: Julia: {{matrix.julia}}"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "{{matrix.julia}}"
+          - JuliaCI/julia-test#v1:
+              test_args: "--quickfail"
+          - JuliaCI/julia-coverage#v1:
+              codecov: true
+              dirs:
+                - src
+        agents:
+          queue: "juliagpu"
+          cuda: "*"
+        if: build.message !~ /\[skip tests\]/
+        timeout_in_minutes: 60
+        matrix:
+          setup:
+            julia:
+              - "1"
+              - "nightly"
+          adjustments:
+            - with:
+                julia: "nightly"
+              soft_fail: true
+
+  # Downstream CUDA Tests
+  - group: ":telescope: Downstream CUDA"
+    steps:
+      - label: ":julia: {{matrix.repo}} (Julia {{matrix.julia}} + CUDA GPU)"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "{{matrix.julia}}"
+          - JuliaCI/julia-coverage#v1:
+              codecov: true
+              dirs:
+                - src
+        command: |
+          julia --code-coverage=user --color=yes --project -e '
+            using Pkg
+
+            repo = ENV["DOWNSTREAM_TEST_REPO"]
+
+            println("--- :julia: Instantiating project")
+            withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0) do
+              Pkg.instantiate()
+
+              try
+                Pkg.develop(repo)
+                println("+++ :julia: Running tests")
+                Pkg.test("$(repo)"; coverage=true)
+              catch err
+                err isa Pkg.Resolve.ResolverError || rethrow()
+                @info "Not compatible with this release. No problem." exception=err
+                exit(0)
+              end
+            end
+
+            println("+++ :julia: Finished Downstream Test")'
+        agents:
+          queue: "juliagpu"
+          cuda: "*"
+        env:
+          GROUP: "CUDA"
+          DOWNSTREAM_TEST_REPO: "{{matrix.repo}}"
+        if: build.message !~ /\[skip tests\]/ || build.message !~ /\[skip downstream\]/
+        timeout_in_minutes: 240
+        matrix:
+          setup:
+            julia:
+              - "1"
+            repo:
+              - "Lux"
+              - "Boltz"
+              - "LuxLib"
 
 env:
   SECRET_CODECOV_TOKEN: "TTwLG9F33tgVgZHK68A3ReRNBt0sWOMAOlPv4kwqwlbWumO6dmz5Narsc889M89nkGFF18d4N/uDWlrm6yIvBX8KSv84vtDOmV5h4d1r6TDVTumibJsFUnTLUkMfbSxw/Bk/q9DKwkYzb1MsNYFJ+zvx9WHnTBd1TiCOLYIRoqxH3aiipe2Auv1sLHJXsxfOvLyrqmcZC+h9OHbVhvFKgrlXbDqONNhWEX4tkzplhIddi60GwFv9xQe7sXpNNmI3Dz/s7BI5XzOxQwKziWOhfsXHreuyby8/Jl/ncpytQkSYRwOw0u8EKNIzeGTCDhfV1EfeuyCq6BfzwSxSFoe8Dw==;U2FsdGVkX1/amMWov97QY23CDLskhDds8btz5Rh9tunCe2Ky8oocTu/5cOy13GjRfAFlQapr78KQrX67dJm/0g=="
diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml
deleted file mode 100644
index 9a215e9..0000000
--- a/.github/workflows/Downstream.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-name: Downstream
-on:
-  pull_request:
-    branches:
-      - main
-  push:
-    branches:
-      - main
-concurrency:
-  # Skip intermediate builds: always.
-  # Cancel intermediate builds: only if it is a pull request build.
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
-jobs:
-  test:
-    name: ${{ matrix.package.repo }}/${{ matrix.package.group }}
-    runs-on: ${{ matrix.os }}
-    env:
-      GROUP: ${{ matrix.package.group }}
-    strategy:
-      fail-fast: false
-      matrix:
-        julia-version: ["1"]
-        os: [ubuntu-latest]
-        package:
-          - { user: LuxDL, repo: Lux.jl, group: CUDA }
-          - { user: LuxDL, repo: LuxLib.jl, group: CUDA }
-    if: contains(github.event.pull_request.labels.*.name, 'run downstream test')
-    steps:
-      - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v1
-        with:
-          version: ${{ matrix.julia-version }}
-          arch: x64
-      - uses: julia-actions/julia-buildpkg@v1
-      - name: Clone Downstream
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ matrix.package.user }}/${{ matrix.package.repo }}
-          path: downstream
-      - name: Load this and run the downstream tests
-        shell: julia --code-coverage=user --color=yes --project=downstream {0}
-        run: |
-          using Pkg
-          try
-            # force it to use this PR's version of the package
-            Pkg.develop(PackageSpec(path="."))  # resolver may fail with main deps
-            Pkg.update()
-            Pkg.test()  # resolver may fail with test time deps
-          catch err
-            err isa Pkg.Resolve.ResolverError || rethrow()
-            # If we can't resolve that means this is incompatible by SemVer and this is fine
-            # It means we marked this as a breaking change, so we don't need to worry about
-            # Mistakenly introducing a breaking change, as we have intentionally made one
-            @info "Not compatible with this release. No problem." exception=err
-            exit(0)  # Exit immediately, as a success
-          end
-      - uses: julia-actions/julia-processcoverage@v1
-        with:
-          directories: src,ext
-      - uses: codecov/codecov-action@v3
-        with:
-          files: lcov.info
\ No newline at end of file