Switch workflows to use new requirements-iree-*.txt files. (#813)

Progress on #760. We could make
the scheduled jobs test both pinned and unpinned versions like on
#767.

Cleanup included here:

* Dropped the "Installing the PyTorch CPU wheels saves multiple minutes
and a lot of bandwidth on runner setup." comments since they are
repetitive. Could add them back if people find them useful.
* Stopped installing from the root `requirements.txt` in some workflows,
instead opting to just install from the more specific
`sharktank/requirements-tests.txt`

I did not test the changes to scheduled workflows. Could do that on
request, or just revert if we see issues.

.github/workflows/ci-llama-large-tests.yaml

@@ -55,18 +55,12 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
-
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
+          pip install -r requirements-iree-unpinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
           pip freeze
 

.github/workflows/ci-sglang-benchmark.yml

@@ -61,18 +61,19 @@ jobs:
         run: |
           source ${VENV_DIR}/bin/activate
           python -m pip install --no-compile --upgrade pip
+
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
-          pip install -f https://iree.dev/pip-release-links.html --pre iree-turbine
-          pip install --no-compile -r requirements.txt -e sharktank/ shortfin/
-
-          # Pin to known-working versions.
-          pip install -f https://iree.dev/pip-release-links.html \
-            iree-base-compiler==3.1.0rc20241220 \
-            iree-base-runtime==3.1.0rc20241220 \
-            "numpy<2.0"
+
+          # Use newest possible releases to be able to track commits that may
+          # cause errors or performance changes.
+          pip install -r requirements-iree-unpinned.txt
+
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -r shortfin/requirements-tests.txt \
+            -e sharktank/ shortfin/
 
           # Install SGLang
           pip install "git+https://github.com/nod-ai/sglang.git#subdirectory=python"

.github/workflows/ci-sglang-integration-tests.yml

@@ -52,19 +52,17 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
 
           # Use newest possible releases to be able to track commits that may
           # cause errors.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine \
-            "numpy<2.0"
+          pip install -r requirements-iree-unpinned.txt
 
-          pip install --no-compile -r requirements.txt -e sharktank/ shortfin/
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -r shortfin/requirements-tests.txt \
+            -e sharktank/ shortfin/
 
           # Install SGLang and sentence_transformers
           pip install "git+https://github.com/nod-ai/sglang.git#subdirectory=python"

.github/workflows/ci-tuner.yml

@@ -46,10 +46,7 @@ jobs:
       - name: Install tuner dependencies
         run: |
           pip install -r tuner/requirements-tuner.txt
-          python -m pip install \
-            --find-links https://iree.dev/pip-release-links.html \
-            --upgrade --pre \
-            iree-base-compiler iree-base-runtime
+          pip install -r requirements-iree-unpinned.txt
 
       - name: Run tuner tests
         run: pytest tuner/

.github/workflows/ci_eval.yaml

@@ -53,18 +53,13 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
+          pip install -r requirements-iree-unpinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
-          # Install nightly IREE packages.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-base-compiler \
-            iree-base-runtime \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
           pip freeze
 
       - name: Run perplexity test with IREE
@@ -112,16 +107,14 @@ jobs:
           python -m pip install --no-compile --upgrade pip
 
           # Note: We install in three steps in order to satisfy requirements
-          # from non default locations first. Installing the PyTorch CPU
-          # wheels saves multiple minutes and a lot of bandwidth on runner setup.
+          # from non default locations first.
           pip install --no-compile -r pytorch-cpu-requirements.txt
+          pip install -r requirements-iree-unpinned.txt
+          pip install --no-compile \
+            -r sharktank/requirements-tests.txt \
+            -e sharktank/
 
-          # Install nightly iree-turbine.
-          # We could also pin to a known working or stable version.
-          pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \
-            iree-turbine
-
-          pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/
+          pip freeze
 
       - name: Run perplexity test with Torch
         run: |