From b87554196739e4331d975374328a49ee685f6dd9 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Wed, 18 Dec 2024 13:42:41 -0500 Subject: [PATCH 1/9] switch GPU workers to d2g images --- taskcluster/config.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/taskcluster/config.yml b/taskcluster/config.yml index d72b8d1b3..15c99bca6 100644 --- a/taskcluster/config.yml +++ b/taskcluster/config.yml @@ -93,39 +93,39 @@ workers: worker-type: 'b-linux-large-gcp-1tb-64-512-std-d2g' b-linux-v100-gpu: provisioner: '{trust-domain}-{level}' - implementation: generic-worker + implementation: docker-worker os: linux - worker-type: '{alias}' + worker-type: 'b-linux-v100-gpu-d2g-4' b-linux-v100-gpu-4: provisioner: '{trust-domain}-{level}' - implementation: generic-worker + implementation: docker-worker os: linux - worker-type: '{alias}' + worker-type: 'b-linux-v100-gpu-d2g-4' b-linux-v100-gpu-4-300gb: provisioner: '{trust-domain}-{level}' - implementation: generic-worker + implementation: docker-worker os: linux - worker-type: '{alias}' + worker-type: 'b-linux-v100-gpu-d2g-4-300gb' b-linux-v100-gpu-4-300gb-standard: provisioner: '{trust-domain}-{level}' - implementation: generic-worker + implementation: docker-worker os: linux - worker-type: '{alias}' + worker-type: 'b-linux-v100-gpu-d2g-4-300gb' b-linux-v100-gpu-4-1tb: provisioner: '{trust-domain}-{level}' - implementation: generic-worker + implementation: docker-worker os: linux - worker-type: '{alias}' + worker-type: 'b-linux-v100-gpu-d2g-4-300gb' b-linux-v100-gpu-4-2tb: provisioner: '{trust-domain}-{level}' - implementation: generic-worker + implementation: docker-worker os: linux - worker-type: '{alias}' + worker-type: 'b-linux-v100-gpu-d2g-4-300gb' b-linux-v100-gpu-4-1tb-standard: provisioner: '{trust-domain}-{level}' - implementation: generic-worker + implementation: docker-worker os: linux - worker-type: '{alias}' + worker-type: 'b-linux-v100-gpu-d2g-4-300gb' images: provisioner: '{trust-domain}-{level}' implementation: docker-worker From 7a092812054b5485ed557b101c65a260cedca2f8 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Wed, 18 Dec 2024 14:31:09 -0500 Subject: [PATCH 2/9] Switch GPU tasks to run within docker The new image we're upgrading GPU workers to uses Ubuntu 24.04, which makes it incompatible with various parts of the pipeline (mostly due to Python package pinning). As it turns out, the easiest way to fix this is to dockerize the GPU tasks. We need slight updates to GPU task payloads to accommodate this. This will fix #391. --- taskcluster/kinds/bicleaner/kind.yml | 3 ++- taskcluster/kinds/evaluate-quantized/kind.yml | 3 ++- taskcluster/kinds/evaluate-teacher-ensemble/kind.yml | 3 ++- taskcluster/kinds/evaluate/kind.yml | 3 ++- taskcluster/kinds/finetune-student/kind.yml | 3 ++- taskcluster/kinds/score/kind.yml | 3 ++- taskcluster/kinds/train-backwards/kind.yml | 3 ++- taskcluster/kinds/train-student/kind.yml | 3 ++- taskcluster/kinds/train-teacher/kind.yml | 3 ++- taskcluster/kinds/translate-corpus/kind.yml | 3 ++- taskcluster/kinds/translate-mono-src/kind.yml | 3 ++- taskcluster/kinds/translate-mono-trg/kind.yml | 3 ++- 12 files changed, 24 insertions(+), 12 deletions(-) diff --git a/taskcluster/kinds/bicleaner/kind.yml b/taskcluster/kinds/bicleaner/kind.yml index 5a864865c..4fd3c7c7d 100644 --- a/taskcluster/kinds/bicleaner/kind.yml +++ b/taskcluster/kinds/bicleaner/kind.yml @@ -64,9 +64,10 @@ tasks: worker-type: b-largegpu-largedisk worker: + docker-image: {"in-tree": "train"} artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory # 7 days. yes, it can take a while to clean a huge dataset max-run-time: 604800 diff --git a/taskcluster/kinds/evaluate-quantized/kind.yml b/taskcluster/kinds/evaluate-quantized/kind.yml index bc9fdccce..66c3e17f3 100644 --- a/taskcluster/kinds/evaluate-quantized/kind.yml +++ b/taskcluster/kinds/evaluate-quantized/kind.yml @@ -54,9 +54,10 @@ tasks: worker-type: b-gpu worker: + docker-image: {"in-tree": "train"} artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory max-run-time: 2592000 env: diff --git a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml index 10cddf1c5..ff7a2976e 100644 --- a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml +++ b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml @@ -54,9 +54,10 @@ tasks: worker-type: b-gpu worker: + docker-image: {"in-tree": "train"} artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory max-run-time: 2592000 env: diff --git a/taskcluster/kinds/evaluate/kind.yml b/taskcluster/kinds/evaluate/kind.yml index f9bb43550..be8d11bb5 100644 --- a/taskcluster/kinds/evaluate/kind.yml +++ b/taskcluster/kinds/evaluate/kind.yml @@ -52,9 +52,10 @@ task-defaults: owner: owner worker-type: b-gpu worker: + docker-image: {"in-tree": "train"} artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory max-run-time: 2592000 env: diff --git a/taskcluster/kinds/finetune-student/kind.yml b/taskcluster/kinds/finetune-student/kind.yml index 7ba921dab..cde67584b 100644 --- a/taskcluster/kinds/finetune-student/kind.yml +++ b/taskcluster/kinds/finetune-student/kind.yml @@ -62,6 +62,7 @@ tasks: worker-type: b-largegpu-xxlargedisk worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 # train_taskcluster.py exits with 17 if a request to Taskcluster fails # 128 happens when cloning this repository fails @@ -75,7 +76,7 @@ tasks: TASKCLUSTER_SECRET: project/translations/level-1/weights-and-biases artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory # Taskcluster proxy is required to read secrets diff --git a/taskcluster/kinds/score/kind.yml b/taskcluster/kinds/score/kind.yml index b391de044..554ed8bd5 100644 --- a/taskcluster/kinds/score/kind.yml +++ b/taskcluster/kinds/score/kind.yml @@ -46,6 +46,7 @@ tasks: worker-type: b-largegpu-largedisk worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 env: # TODO: what should we _actually_ use for the workspace value? @@ -59,7 +60,7 @@ tasks: TRG: "{trg_locale}" artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory # 128 happens when cloning this repository fails retry-exit-status: [128] diff --git a/taskcluster/kinds/train-backwards/kind.yml b/taskcluster/kinds/train-backwards/kind.yml index fa141f77b..d0ba47c1c 100644 --- a/taskcluster/kinds/train-backwards/kind.yml +++ b/taskcluster/kinds/train-backwards/kind.yml @@ -62,6 +62,7 @@ tasks: github-pull-request: b-largegpu default: b-largegpu-largedisk worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 # train_taskcluster.py exits with 17 if a request to Taskcluster fails retry-exit-status: [17] @@ -74,7 +75,7 @@ tasks: TASKCLUSTER_SECRET: project/translations/level-1/weights-and-biases artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory # Taskcluster proxy is required to read secrets diff --git a/taskcluster/kinds/train-student/kind.yml b/taskcluster/kinds/train-student/kind.yml index e3bbb1f35..97f1fdb00 100644 --- a/taskcluster/kinds/train-student/kind.yml +++ b/taskcluster/kinds/train-student/kind.yml @@ -60,6 +60,7 @@ tasks: github-pull-request: b-largegpu-largedisk default: b-largegpu-xxlargedisk worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 # train_taskcluster.py exits with 17 if a request to Taskcluster fails # 128 happens when cloning this repository fails @@ -73,7 +74,7 @@ tasks: TASKCLUSTER_SECRET: project/translations/level-1/weights-and-biases artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory # Taskcluster proxy is required to read secrets diff --git a/taskcluster/kinds/train-teacher/kind.yml b/taskcluster/kinds/train-teacher/kind.yml index a14d64771..824dc8ba3 100644 --- a/taskcluster/kinds/train-teacher/kind.yml +++ b/taskcluster/kinds/train-teacher/kind.yml @@ -82,6 +82,7 @@ tasks: github-pull-request: b-largegpu default: b-largegpu-xlargedisk worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 # train_taskcluster.py exits with 17 if a request to Taskcluster fails # 128 happens when cloning this repository fails @@ -95,7 +96,7 @@ tasks: TASKCLUSTER_SECRET: project/translations/level-1/weights-and-biases artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory # Taskcluster proxy is required to read secrets diff --git a/taskcluster/kinds/translate-corpus/kind.yml b/taskcluster/kinds/translate-corpus/kind.yml index 452264a75..71209053a 100644 --- a/taskcluster/kinds/translate-corpus/kind.yml +++ b/taskcluster/kinds/translate-corpus/kind.yml @@ -93,10 +93,11 @@ tasks: worker-type: b-largegpu-xlargedisk worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory env: CUDA_DIR: fetches/cuda-toolkit diff --git a/taskcluster/kinds/translate-mono-src/kind.yml b/taskcluster/kinds/translate-mono-src/kind.yml index 7480e4986..939502c7f 100644 --- a/taskcluster/kinds/translate-mono-src/kind.yml +++ b/taskcluster/kinds/translate-mono-src/kind.yml @@ -93,10 +93,11 @@ tasks: worker-type: b-largegpu worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory env: CUDA_DIR: fetches/cuda-toolkit diff --git a/taskcluster/kinds/translate-mono-trg/kind.yml b/taskcluster/kinds/translate-mono-trg/kind.yml index ca6e7c9d3..4758688df 100644 --- a/taskcluster/kinds/translate-mono-trg/kind.yml +++ b/taskcluster/kinds/translate-mono-trg/kind.yml @@ -94,10 +94,11 @@ tasks: from-parameters: training_config.marian-args.decoding-backward worker-type: b-largegpu worker: + docker-image: {"in-tree": "train"} max-run-time: 2592000 artifacts: - name: public/build - path: artifacts + path: /builds/worker/artifacts type: directory env: CUDA_DIR: fetches/cuda-toolkit From 441edefc898bd2c5e3438ddad7a2993d8b4a4ae7 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Wed, 18 Dec 2024 20:48:53 -0500 Subject: [PATCH 3/9] enable cache for ~/.cache Without this we end up with these files being inaccessible in subsequent tasks. --- taskcluster/docker/train/Dockerfile | 1 + taskcluster/kinds/bicleaner/kind.yml | 1 + taskcluster/kinds/evaluate-quantized/kind.yml | 1 + taskcluster/kinds/evaluate-teacher-ensemble/kind.yml | 1 + taskcluster/kinds/evaluate/kind.yml | 1 + taskcluster/kinds/finetune-student/kind.yml | 1 + taskcluster/kinds/score/kind.yml | 1 + taskcluster/kinds/train-backwards/kind.yml | 1 + taskcluster/kinds/train-student/kind.yml | 1 + taskcluster/kinds/train-teacher/kind.yml | 1 + taskcluster/kinds/train-vocab/kind.yml | 1 + taskcluster/kinds/translate-corpus/kind.yml | 1 + taskcluster/kinds/translate-mono-src/kind.yml | 1 + taskcluster/kinds/translate-mono-trg/kind.yml | 1 + 14 files changed, 14 insertions(+) diff --git a/taskcluster/docker/train/Dockerfile b/taskcluster/docker/train/Dockerfile index f7d3864b5..719322066 100644 --- a/taskcluster/docker/train/Dockerfile +++ b/taskcluster/docker/train/Dockerfile @@ -20,3 +20,4 @@ RUN apt-get update -qq \ VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache diff --git a/taskcluster/kinds/bicleaner/kind.yml b/taskcluster/kinds/bicleaner/kind.yml index 4fd3c7c7d..282fdcb1f 100644 --- a/taskcluster/kinds/bicleaner/kind.yml +++ b/taskcluster/kinds/bicleaner/kind.yml @@ -89,6 +89,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -c diff --git a/taskcluster/kinds/evaluate-quantized/kind.yml b/taskcluster/kinds/evaluate-quantized/kind.yml index 66c3e17f3..54dc5fdc8 100644 --- a/taskcluster/kinds/evaluate-quantized/kind.yml +++ b/taskcluster/kinds/evaluate-quantized/kind.yml @@ -85,6 +85,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -c diff --git a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml index ff7a2976e..3f50496e1 100644 --- a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml +++ b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml @@ -85,6 +85,7 @@ tasks: run: using: run-task + cache-dotcache: true # The two sed commands here are the unfortunate result of us consuming # a marian config that was produced by an earlier step. These configs # have hardcoded absolute paths to the models they were trained on, diff --git a/taskcluster/kinds/evaluate/kind.yml b/taskcluster/kinds/evaluate/kind.yml index be8d11bb5..f78484ea7 100644 --- a/taskcluster/kinds/evaluate/kind.yml +++ b/taskcluster/kinds/evaluate/kind.yml @@ -83,6 +83,7 @@ task-defaults: run: using: run-task + cache-dotcache: true # The two sed commands here are the unfortunate result of us consuming # a marian config that was produced by an earlier step. These configs # have hardcoded absolute paths to the models they were trained on, diff --git a/taskcluster/kinds/finetune-student/kind.yml b/taskcluster/kinds/finetune-student/kind.yml index cde67584b..fe28d3a62 100644 --- a/taskcluster/kinds/finetune-student/kind.yml +++ b/taskcluster/kinds/finetune-student/kind.yml @@ -90,6 +90,7 @@ tasks: from-parameters: training_config.marian-args.training-student-finetuned run: using: run-task + cache-dotcache: true command: - bash - -cx diff --git a/taskcluster/kinds/score/kind.yml b/taskcluster/kinds/score/kind.yml index 554ed8bd5..a74edcea2 100644 --- a/taskcluster/kinds/score/kind.yml +++ b/taskcluster/kinds/score/kind.yml @@ -70,6 +70,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -c diff --git a/taskcluster/kinds/train-backwards/kind.yml b/taskcluster/kinds/train-backwards/kind.yml index d0ba47c1c..350184c53 100644 --- a/taskcluster/kinds/train-backwards/kind.yml +++ b/taskcluster/kinds/train-backwards/kind.yml @@ -95,6 +95,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -cx diff --git a/taskcluster/kinds/train-student/kind.yml b/taskcluster/kinds/train-student/kind.yml index 97f1fdb00..1f9db2a23 100644 --- a/taskcluster/kinds/train-student/kind.yml +++ b/taskcluster/kinds/train-student/kind.yml @@ -91,6 +91,7 @@ tasks: from-parameters: training_config.marian-args.training-student run: using: run-task + cache-dotcache: true command: - bash - -cx diff --git a/taskcluster/kinds/train-teacher/kind.yml b/taskcluster/kinds/train-teacher/kind.yml index 824dc8ba3..21bde13e9 100644 --- a/taskcluster/kinds/train-teacher/kind.yml +++ b/taskcluster/kinds/train-teacher/kind.yml @@ -113,6 +113,7 @@ tasks: from-parameters: training_config.marian-args.training-teacher run: using: run-task + cache-dotcache: true # order of comma separated datasets is important command: - bash diff --git a/taskcluster/kinds/train-vocab/kind.yml b/taskcluster/kinds/train-vocab/kind.yml index 739538d90..398b90939 100644 --- a/taskcluster/kinds/train-vocab/kind.yml +++ b/taskcluster/kinds/train-vocab/kind.yml @@ -61,6 +61,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -c diff --git a/taskcluster/kinds/translate-corpus/kind.yml b/taskcluster/kinds/translate-corpus/kind.yml index 71209053a..6fe13e969 100644 --- a/taskcluster/kinds/translate-corpus/kind.yml +++ b/taskcluster/kinds/translate-corpus/kind.yml @@ -108,6 +108,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -xc diff --git a/taskcluster/kinds/translate-mono-src/kind.yml b/taskcluster/kinds/translate-mono-src/kind.yml index 939502c7f..c0d80c052 100644 --- a/taskcluster/kinds/translate-mono-src/kind.yml +++ b/taskcluster/kinds/translate-mono-src/kind.yml @@ -114,6 +114,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -xc diff --git a/taskcluster/kinds/translate-mono-trg/kind.yml b/taskcluster/kinds/translate-mono-trg/kind.yml index 4758688df..fa7dbbfbd 100644 --- a/taskcluster/kinds/translate-mono-trg/kind.yml +++ b/taskcluster/kinds/translate-mono-trg/kind.yml @@ -112,6 +112,7 @@ tasks: run: using: run-task + cache-dotcache: true command: - bash - -xc From f56fbc441ac0cbb298942eaaa358662930d27ba4 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 19 Dec 2024 11:23:58 -0500 Subject: [PATCH 4/9] install cuda and friends in train Dockerfile --- taskcluster/docker/train/Dockerfile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/taskcluster/docker/train/Dockerfile b/taskcluster/docker/train/Dockerfile index 719322066..fb1ea58bb 100644 --- a/taskcluster/docker/train/Dockerfile +++ b/taskcluster/docker/train/Dockerfile @@ -1,6 +1,10 @@ FROM $DOCKER_IMAGE_PARENT LABEL maintainer="Mozilla Release Engineering " +RUN curl -L https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb > /tmp/cuda-keyring.deb \ + && dpkg -i /tmp/cuda-keyring.deb \ + && rm /tmp/cuda-keyring.deb + RUN apt-get update -qq \ && apt-get install -y python3-numpy \ python3-fasttext \ @@ -16,6 +20,17 @@ RUN apt-get update -qq \ wget \ pkg-config \ libicu-dev \ + cuda-toolkit \ + software-properties-common \ + && apt-get clean + +RUN curl -L https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin > /tmp/cuda.pin \ + && mv /tmp/cuda.pin /etc/apt/preferences.d/cuda-repository-pin-600 \ + && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub \ + && add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" \ + && apt-get update -qq \ + && apt-get install -y cudnn9-cuda-12 \ + libcudnn9-dev-cuda-12 \ && apt-get clean From b8fd92edbc7c5b96d0828d3da87f159c5aaa48b4 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Wed, 8 Jan 2025 19:17:18 -0500 Subject: [PATCH 5/9] fix: add 'requests' to ctranslate2 requirements This has always been needed, but it was found on the host system on the previous image. --- .../requirements/translate-ctranslate2.in | 1 + .../requirements/translate-ctranslate2.txt | 362 ++++++++++++------ 2 files changed, 243 insertions(+), 120 deletions(-) diff --git a/pipeline/translate/requirements/translate-ctranslate2.in b/pipeline/translate/requirements/translate-ctranslate2.in index 0d266b389..d884af784 100644 --- a/pipeline/translate/requirements/translate-ctranslate2.in +++ b/pipeline/translate/requirements/translate-ctranslate2.in @@ -1,3 +1,4 @@ ctranslate2==4.3.1 sentencepiece==0.2.0 gpustat==1.1.1 +requests==2.32.3 diff --git a/pipeline/translate/requirements/translate-ctranslate2.txt b/pipeline/translate/requirements/translate-ctranslate2.txt index 49d6cb81c..e41936e29 100644 --- a/pipeline/translate/requirements/translate-ctranslate2.txt +++ b/pipeline/translate/requirements/translate-ctranslate2.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --allow-unsafe --generate-hashes pipeline/translate/requirements/translate-ctranslate2.in @@ -8,6 +8,104 @@ blessed==1.20.0 \ --hash=sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058 \ --hash=sha256:2cdd67f8746e048f00df47a2880f4d6acbcdb399031b604e34ba8f71d5787680 # via gpustat +certifi==2024.12.14 \ + --hash=sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56 \ + --hash=sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db + # via requests +charset-normalizer==3.4.1 \ + --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ + --hash=sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa \ + --hash=sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a \ + --hash=sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294 \ + --hash=sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b \ + --hash=sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd \ + --hash=sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601 \ + --hash=sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd \ + --hash=sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4 \ + --hash=sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d \ + --hash=sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2 \ + --hash=sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313 \ + --hash=sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd \ + --hash=sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa \ + --hash=sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8 \ + --hash=sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1 \ + --hash=sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2 \ + --hash=sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496 \ + --hash=sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d \ + --hash=sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b \ + --hash=sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e \ + --hash=sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a \ + --hash=sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4 \ + --hash=sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca \ + --hash=sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78 \ + --hash=sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408 \ + --hash=sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5 \ + --hash=sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3 \ + --hash=sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f \ + --hash=sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a \ + --hash=sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765 \ + --hash=sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6 \ + --hash=sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146 \ + --hash=sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6 \ + --hash=sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9 \ + --hash=sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd \ + --hash=sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c \ + --hash=sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f \ + --hash=sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545 \ + --hash=sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176 \ + --hash=sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770 \ + --hash=sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824 \ + --hash=sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f \ + --hash=sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf \ + --hash=sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487 \ + --hash=sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d \ + --hash=sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd \ + --hash=sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b \ + --hash=sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534 \ + --hash=sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f \ + --hash=sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b \ + --hash=sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9 \ + --hash=sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd \ + --hash=sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125 \ + --hash=sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9 \ + --hash=sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de \ + --hash=sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11 \ + --hash=sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d \ + --hash=sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35 \ + --hash=sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f \ + --hash=sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda \ + --hash=sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7 \ + --hash=sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a \ + --hash=sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971 \ + --hash=sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8 \ + --hash=sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41 \ + --hash=sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d \ + --hash=sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f \ + --hash=sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757 \ + --hash=sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a \ + --hash=sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886 \ + --hash=sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77 \ + --hash=sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76 \ + --hash=sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247 \ + --hash=sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85 \ + --hash=sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb \ + --hash=sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7 \ + --hash=sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e \ + --hash=sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6 \ + --hash=sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037 \ + --hash=sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1 \ + --hash=sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e \ + --hash=sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807 \ + --hash=sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407 \ + --hash=sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c \ + --hash=sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12 \ + --hash=sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3 \ + --hash=sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089 \ + --hash=sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd \ + --hash=sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e \ + --hash=sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00 \ + --hash=sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616 + # via requests ctranslate2==4.3.1 \ --hash=sha256:08626f115d5a39c56a666680735d6eebfc4d8a215288896d4d8afc14cfcdcffe \ --hash=sha256:30c02fcd5a7be93bf42a8adf81a9ac4f394e23bd639192907b2e11feae589971 \ @@ -38,129 +136,149 @@ ctranslate2==4.3.1 \ gpustat==1.1.1 \ --hash=sha256:c18d3ed5518fc16300c42d694debc70aebb3be55cae91f1db64d63b5fa8af9d8 # via -r pipeline/translate/requirements/translate-ctranslate2.in -numpy==2.0.1 \ - --hash=sha256:08458fbf403bff5e2b45f08eda195d4b0c9b35682311da5a5a0a0925b11b9bd8 \ - --hash=sha256:0fbb536eac80e27a2793ffd787895242b7f18ef792563d742c2d673bfcb75134 \ - --hash=sha256:12f5d865d60fb9734e60a60f1d5afa6d962d8d4467c120a1c0cda6eb2964437d \ - --hash=sha256:15eb4eca47d36ec3f78cde0a3a2ee24cf05ca7396ef808dda2c0ddad7c2bde67 \ - --hash=sha256:173a00b9995f73b79eb0191129f2455f1e34c203f559dd118636858cc452a1bf \ - --hash=sha256:1b902ce0e0a5bb7704556a217c4f63a7974f8f43e090aff03fcf262e0b135e02 \ - --hash=sha256:1f682ea61a88479d9498bf2091fdcd722b090724b08b31d63e022adc063bad59 \ - --hash=sha256:1f87fec1f9bc1efd23f4227becff04bd0e979e23ca50cc92ec88b38489db3b55 \ - --hash=sha256:24a0e1befbfa14615b49ba9659d3d8818a0f4d8a1c5822af8696706fbda7310c \ - --hash=sha256:2c3a346ae20cfd80b6cfd3e60dc179963ef2ea58da5ec074fd3d9e7a1e7ba97f \ - --hash=sha256:36d3a9405fd7c511804dc56fc32974fa5533bdeb3cd1604d6b8ff1d292b819c4 \ - --hash=sha256:3fdabe3e2a52bc4eff8dc7a5044342f8bd9f11ef0934fcd3289a788c0eb10018 \ - --hash=sha256:4127d4303b9ac9f94ca0441138acead39928938660ca58329fe156f84b9f3015 \ - --hash=sha256:4658c398d65d1b25e1760de3157011a80375da861709abd7cef3bad65d6543f9 \ - --hash=sha256:485b87235796410c3519a699cfe1faab097e509e90ebb05dcd098db2ae87e7b3 \ - --hash=sha256:529af13c5f4b7a932fb0e1911d3a75da204eff023ee5e0e79c1751564221a5c8 \ - --hash=sha256:5a3d94942c331dd4e0e1147f7a8699a4aa47dffc11bf8a1523c12af8b2e91bbe \ - --hash=sha256:5daab361be6ddeb299a918a7c0864fa8618af66019138263247af405018b04e1 \ - --hash=sha256:61728fba1e464f789b11deb78a57805c70b2ed02343560456190d0501ba37b0f \ - --hash=sha256:6790654cb13eab303d8402354fabd47472b24635700f631f041bd0b65e37298a \ - --hash=sha256:69ff563d43c69b1baba77af455dd0a839df8d25e8590e79c90fcbe1499ebde42 \ - --hash=sha256:6bf4e6f4a2a2e26655717a1983ef6324f2664d7011f6ef7482e8c0b3d51e82ac \ - --hash=sha256:6e4eeb6eb2fced786e32e6d8df9e755ce5be920d17f7ce00bc38fcde8ccdbf9e \ - --hash=sha256:72dc22e9ec8f6eaa206deb1b1355eb2e253899d7347f5e2fae5f0af613741d06 \ - --hash=sha256:75b4e316c5902d8163ef9d423b1c3f2f6252226d1aa5cd8a0a03a7d01ffc6268 \ - --hash=sha256:7b9853803278db3bdcc6cd5beca37815b133e9e77ff3d4733c247414e78eb8d1 \ - --hash=sha256:7d6fddc5fe258d3328cd8e3d7d3e02234c5d70e01ebe377a6ab92adb14039cb4 \ - --hash=sha256:81b0893a39bc5b865b8bf89e9ad7807e16717f19868e9d234bdaf9b1f1393868 \ - --hash=sha256:8efc84f01c1cd7e34b3fb310183e72fcdf55293ee736d679b6d35b35d80bba26 \ - --hash=sha256:8fae4ebbf95a179c1156fab0b142b74e4ba4204c87bde8d3d8b6f9c34c5825ef \ - --hash=sha256:99d0d92a5e3613c33a5f01db206a33f8fdf3d71f2912b0de1739894668b7a93b \ - --hash=sha256:9adbd9bb520c866e1bfd7e10e1880a1f7749f1f6e5017686a5fbb9b72cf69f82 \ - --hash=sha256:a1e01dcaab205fbece13c1410253a9eea1b1c9b61d237b6fa59bcc46e8e89343 \ - --hash=sha256:a8fc2de81ad835d999113ddf87d1ea2b0f4704cbd947c948d2f5513deafe5a7b \ - --hash=sha256:b83e16a5511d1b1f8a88cbabb1a6f6a499f82c062a4251892d9ad5d609863fb7 \ - --hash=sha256:bb2124fdc6e62baae159ebcfa368708867eb56806804d005860b6007388df171 \ - --hash=sha256:bfc085b28d62ff4009364e7ca34b80a9a080cbd97c2c0630bb5f7f770dae9414 \ - --hash=sha256:cbab9fc9c391700e3e1287666dfd82d8666d10e69a6c4a09ab97574c0b7ee0a7 \ - --hash=sha256:e5eeca8067ad04bc8a2a8731183d51d7cbaac66d86085d5f4766ee6bf19c7f87 \ - --hash=sha256:e9e81fa9017eaa416c056e5d9e71be93d05e2c3c2ab308d23307a8bc4443c368 \ - --hash=sha256:ea2326a4dca88e4a274ba3a4405eb6c6467d3ffbd8c7d38632502eaae3820587 \ - --hash=sha256:eacf3291e263d5a67d8c1a581a8ebbcfd6447204ef58828caf69a5e3e8c75990 \ - --hash=sha256:ec87f5f8aca726117a1c9b7083e7656a9d0d606eec7299cc067bb83d26f16e0c \ - --hash=sha256:f1659887361a7151f89e79b276ed8dff3d75877df906328f14d8bb40bb4f5101 \ - --hash=sha256:f9cf5ea551aec449206954b075db819f52adc1638d46a6738253a712d553c7b4 +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 + # via requests +numpy==2.2.1 \ + --hash=sha256:059e6a747ae84fce488c3ee397cee7e5f905fd1bda5fb18c66bc41807ff119b2 \ + --hash=sha256:08ef779aed40dbc52729d6ffe7dd51df85796a702afbf68a4f4e41fafdc8bda5 \ + --hash=sha256:164a829b6aacf79ca47ba4814b130c4020b202522a93d7bff2202bfb33b61c60 \ + --hash=sha256:26c9c4382b19fcfbbed3238a14abf7ff223890ea1936b8890f058e7ba35e8d71 \ + --hash=sha256:27f5cdf9f493b35f7e41e8368e7d7b4bbafaf9660cba53fb21d2cd174ec09631 \ + --hash=sha256:31b89fa67a8042e96715c68e071a1200c4e172f93b0fbe01a14c0ff3ff820fc8 \ + --hash=sha256:32cb94448be47c500d2c7a95f93e2f21a01f1fd05dd2beea1ccd049bb6001cd2 \ + --hash=sha256:360137f8fb1b753c5cde3ac388597ad680eccbbbb3865ab65efea062c4a1fd16 \ + --hash=sha256:3683a8d166f2692664262fd4900f207791d005fb088d7fdb973cc8d663626faa \ + --hash=sha256:38efc1e56b73cc9b182fe55e56e63b044dd26a72128fd2fbd502f75555d92591 \ + --hash=sha256:3d03883435a19794e41f147612a77a8f56d4e52822337844fff3d4040a142964 \ + --hash=sha256:3ecc47cd7f6ea0336042be87d9e7da378e5c7e9b3c8ad0f7c966f714fc10d821 \ + --hash=sha256:40f9e544c1c56ba8f1cf7686a8c9b5bb249e665d40d626a23899ba6d5d9e1484 \ + --hash=sha256:4250888bcb96617e00bfa28ac24850a83c9f3a16db471eca2ee1f1714df0f957 \ + --hash=sha256:4511d9e6071452b944207c8ce46ad2f897307910b402ea5fa975da32e0102800 \ + --hash=sha256:45681fd7128c8ad1c379f0ca0776a8b0c6583d2f69889ddac01559dfe4390918 \ + --hash=sha256:48fd472630715e1c1c89bf1feab55c29098cb403cc184b4859f9c86d4fcb6a95 \ + --hash=sha256:4c86e2a209199ead7ee0af65e1d9992d1dce7e1f63c4b9a616500f93820658d0 \ + --hash=sha256:4dfda918a13cc4f81e9118dea249e192ab167a0bb1966272d5503e39234d694e \ + --hash=sha256:5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d \ + --hash=sha256:51faf345324db860b515d3f364eaa93d0e0551a88d6218a7d61286554d190d73 \ + --hash=sha256:526fc406ab991a340744aad7e25251dd47a6720a685fa3331e5c59fef5282a59 \ + --hash=sha256:53c09385ff0b72ba79d8715683c1168c12e0b6e84fb0372e97553d1ea91efe51 \ + --hash=sha256:55ba24ebe208344aa7a00e4482f65742969a039c2acfcb910bc6fcd776eb4355 \ + --hash=sha256:5b6c390bfaef8c45a260554888966618328d30e72173697e5cabe6b285fb2348 \ + --hash=sha256:5c5cc0cbabe9452038ed984d05ac87910f89370b9242371bd9079cb4af61811e \ + --hash=sha256:5edb4e4caf751c1518e6a26a83501fda79bff41cc59dac48d70e6d65d4ec4440 \ + --hash=sha256:61048b4a49b1c93fe13426e04e04fdf5a03f456616f6e98c7576144677598675 \ + --hash=sha256:676f4eebf6b2d430300f1f4f4c2461685f8269f94c89698d832cdf9277f30b84 \ + --hash=sha256:67d4cda6fa6ffa073b08c8372aa5fa767ceb10c9a0587c707505a6d426f4e046 \ + --hash=sha256:694f9e921a0c8f252980e85bce61ebbd07ed2b7d4fa72d0e4246f2f8aa6642ab \ + --hash=sha256:733585f9f4b62e9b3528dd1070ec4f52b8acf64215b60a845fa13ebd73cd0712 \ + --hash=sha256:7671dc19c7019103ca44e8d94917eba8534c76133523ca8406822efdd19c9308 \ + --hash=sha256:780077d95eafc2ccc3ced969db22377b3864e5b9a0ea5eb347cc93b3ea900315 \ + --hash=sha256:7ba9cc93a91d86365a5d270dee221fdc04fb68d7478e6bf6af650de78a8339e3 \ + --hash=sha256:89b16a18e7bba224ce5114db863e7029803c179979e1af6ad6a6b11f70545008 \ + --hash=sha256:9036d6365d13b6cbe8f27a0eaf73ddcc070cae584e5ff94bb45e3e9d729feab5 \ + --hash=sha256:93cf4e045bae74c90ca833cba583c14b62cb4ba2cba0abd2b141ab52548247e2 \ + --hash=sha256:9ad014faa93dbb52c80d8f4d3dcf855865c876c9660cb9bd7553843dd03a4b1e \ + --hash=sha256:9b1d07b53b78bf84a96898c1bc139ad7f10fda7423f5fd158fd0f47ec5e01ac7 \ + --hash=sha256:a7746f235c47abc72b102d3bce9977714c2444bdfaea7888d241b4c4bb6a78bf \ + --hash=sha256:aa3017c40d513ccac9621a2364f939d39e550c542eb2a894b4c8da92b38896ab \ + --hash=sha256:b34d87e8a3090ea626003f87f9392b3929a7bbf4104a05b6667348b6bd4bf1cd \ + --hash=sha256:b541032178a718c165a49638d28272b771053f628382d5e9d1c93df23ff58dbf \ + --hash=sha256:ba5511d8f31c033a5fcbda22dd5c813630af98c70b2661f2d2c654ae3cdfcfc8 \ + --hash=sha256:bc8a37ad5b22c08e2dbd27df2b3ef7e5c0864235805b1e718a235bcb200cf1cb \ + --hash=sha256:bff7d8ec20f5f42607599f9994770fa65d76edca264a87b5e4ea5629bce12268 \ + --hash=sha256:c1ad395cf254c4fbb5b2132fee391f361a6e8c1adbd28f2cd8e79308a615fe9d \ + --hash=sha256:f1d09e520217618e76396377c81fba6f290d5f926f50c35f3a5f72b01a0da780 \ + --hash=sha256:f3eac17d9ec51be534685ba877b6ab5edc3ab7ec95c8f163e5d7b39859524716 \ + --hash=sha256:f419290bc8968a46c4933158c91a0012b7a99bb2e465d5ef5293879742f8797e \ + --hash=sha256:f62aa6ee4eb43b024b0e5a01cf65a0bb078ef8c395e8713c6e8a12a697144528 \ + --hash=sha256:f74e6fdeb9a265624ec3a3918430205dff1df7e95a230779746a6af78bc615af \ + --hash=sha256:f9b57eaa3b0cd8db52049ed0330747b0364e899e8a606a624813452b8203d5f7 \ + --hash=sha256:fce4f615f8ca31b2e61aa0eb5865a21e14f5629515c9151850aa936c02a1ee51 # via ctranslate2 nvidia-ml-py==12.560.30 \ --hash=sha256:f0254dc7400647680a072ee02509bfd46102b60bdfeca321576d4d4817e7fe97 \ --hash=sha256:fea371c94d63e38a611c17bbb85fe400e9c8ddb9e8684a9cd0e47786a4bc3c73 # via gpustat -psutil==6.1.0 \ - --hash=sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047 \ - --hash=sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc \ - --hash=sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e \ - --hash=sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747 \ - --hash=sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e \ - --hash=sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a \ - --hash=sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b \ - --hash=sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76 \ - --hash=sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca \ - --hash=sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688 \ - --hash=sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e \ - --hash=sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38 \ - --hash=sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85 \ - --hash=sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be \ - --hash=sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942 \ - --hash=sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a \ - --hash=sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0 +psutil==6.1.1 \ + --hash=sha256:018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca \ + --hash=sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377 \ + --hash=sha256:1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468 \ + --hash=sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3 \ + --hash=sha256:384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603 \ + --hash=sha256:6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac \ + --hash=sha256:8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303 \ + --hash=sha256:8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4 \ + --hash=sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160 \ + --hash=sha256:9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8 \ + --hash=sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003 \ + --hash=sha256:c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030 \ + --hash=sha256:ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777 \ + --hash=sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5 \ + --hash=sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53 \ + --hash=sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649 \ + --hash=sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8 # via gpustat -pyyaml==6.0.1 \ - --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ - --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ - --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ - --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ - --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ - --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ - --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ - --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ - --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ - --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ - --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ - --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ - --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ - --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ - --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ - --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ - --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ - --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ - --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ - --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ - --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ - --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ - --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ - --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ - --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ - --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ - --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ - --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ - --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ - --hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \ - --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ - --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ - --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ - --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ - --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ - --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ - --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ - --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ - --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ - --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ - --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ - --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ - --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ - --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ - --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ - --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ - --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ - --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ - --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ - --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ - --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f +pyyaml==6.0.2 \ + --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ + --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ + --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ + --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ + --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ + --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ + --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ + --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ + --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ + --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ + --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ + --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ + --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ + --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ + --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ + --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ + --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ + --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ + --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ + --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ + --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ + --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ + --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ + --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ + --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ + --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ + --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ + --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ + --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ + --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ + --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ + --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ + --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ + --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ + --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ + --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ + --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ + --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ + --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ + --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ + --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ + --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ + --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ + --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ + --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ + --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ + --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ + --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ + --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ + --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ + --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ + --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ + --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 # via ctranslate2 +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via -r pipeline/translate/requirements/translate-ctranslate2.in sentencepiece==0.2.0 \ --hash=sha256:0461324897735512a32d222e3d886e24ad6a499761952b6bda2a9ee6e4313ea5 \ --hash=sha256:0993dbc665f4113017892f1b87c3904a44d0640eda510abcacdfb07f74286d36 \ @@ -220,13 +338,17 @@ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via blessed +urllib3==2.3.0 \ + --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \ + --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d + # via requests wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via blessed # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 \ - --hash=sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1 \ - --hash=sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec +setuptools==75.8.0 \ + --hash=sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6 \ + --hash=sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3 # via ctranslate2 From e0755bad00694163eb70269c57660d4143159fc8 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 9 Jan 2025 12:06:21 -0500 Subject: [PATCH 6/9] bump cuda version to match the version on new worker image --- taskcluster/kinds/fetch/toolchains.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/taskcluster/kinds/fetch/toolchains.yml b/taskcluster/kinds/fetch/toolchains.yml index dc01e4415..2914a726a 100644 --- a/taskcluster/kinds/fetch/toolchains.yml +++ b/taskcluster/kinds/fetch/toolchains.yml @@ -43,12 +43,12 @@ extract-lex: revision: 42fa605b53f32eaf6c6e0b5677255c21c91b3d49 cuda: - description: CUDA 12.1.0 Source + description: CUDA 12.6.3 Source fetch: type: static-url - url: https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda_12.1.0_530.30.02_linux.run # yamllint disable-line rule:line-length - sha256: 68699036c12d71adb9ad2799dce2ff070270fab4488b90920b9756ab3f52c41c - size: 4245586997 + url: https://developer.download.nvidia.com/compute/cuda/12.6.3/local_installers/cuda_12.6.3_560.35.05_linux.run # yamllint disable-line rule:line-length + sha256: 81d60e48044796d7883aa8a049afe6501b843f2c45639b3703b2378de30d55d3 + size: 4446722669 artifact-name: cuda-source.run From 8492a887e615adcc83172656d1ec916429629739 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Sun, 12 Jan 2025 20:22:57 -0500 Subject: [PATCH 7/9] install the same version of cuda as new worker image --- taskcluster/docker/train/Dockerfile | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/taskcluster/docker/train/Dockerfile b/taskcluster/docker/train/Dockerfile index fb1ea58bb..a80126760 100644 --- a/taskcluster/docker/train/Dockerfile +++ b/taskcluster/docker/train/Dockerfile @@ -1,10 +1,6 @@ FROM $DOCKER_IMAGE_PARENT LABEL maintainer="Mozilla Release Engineering " -RUN curl -L https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb > /tmp/cuda-keyring.deb \ - && dpkg -i /tmp/cuda-keyring.deb \ - && rm /tmp/cuda-keyring.deb - RUN apt-get update -qq \ && apt-get install -y python3-numpy \ python3-fasttext \ @@ -20,19 +16,16 @@ RUN apt-get update -qq \ wget \ pkg-config \ libicu-dev \ - cuda-toolkit \ software-properties-common \ && apt-get clean -RUN curl -L https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin > /tmp/cuda.pin \ - && mv /tmp/cuda.pin /etc/apt/preferences.d/cuda-repository-pin-600 \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub \ - && add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" \ +# Install the same cuda version that the worker host uses +# See https://github.com/mozilla-platform-ops/worker-images/blob/main/scripts/linux/ubuntu-2404-amd64-headless/fxci/05-nvidia-gcp-driver-cudnn.sh +RUN curl -sSO https://developer.download.nvidia.com/compute/cudnn/9.6.0/local_installers/cudnn-local-repo-ubuntu2204-9.6.0_1.0-1_amd64.deb \ + && dpkg -i cudnn-local-repo-ubuntu2204-9.6.0_1.0-1_amd64.deb \ + && cp /var/cudnn-local-repo-ubuntu2204-9.6.0/cudnn-*-keyring.gpg /usr/share/keyrings/ \ && apt-get update -qq \ - && apt-get install -y cudnn9-cuda-12 \ - libcudnn9-dev-cuda-12 \ - && apt-get clean - + && apt-get install -y cudnn VOLUME /builds/worker/checkouts VOLUME /builds/worker/.cache From c2a268834948aed99a6d1b0ec54e5eee1eb9fa41 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Fri, 17 Jan 2025 11:29:01 -0500 Subject: [PATCH 8/9] don't install cuda/cudnn on train image; use cuda-toolkit toolchain instead --- taskcluster/docker/train/Dockerfile | 8 -------- taskcluster/kinds/evaluate-quantized/kind.yml | 2 ++ taskcluster/kinds/evaluate-teacher-ensemble/kind.yml | 2 ++ taskcluster/kinds/evaluate/kind.yml | 5 +++++ taskcluster/kinds/finetune-student/kind.yml | 2 ++ taskcluster/kinds/score/kind.yml | 2 ++ taskcluster/kinds/train-backwards/kind.yml | 2 ++ taskcluster/kinds/train-student/kind.yml | 2 ++ taskcluster/kinds/train-teacher/kind.yml | 2 ++ 9 files changed, 19 insertions(+), 8 deletions(-) diff --git a/taskcluster/docker/train/Dockerfile b/taskcluster/docker/train/Dockerfile index a80126760..27c3e0109 100644 --- a/taskcluster/docker/train/Dockerfile +++ b/taskcluster/docker/train/Dockerfile @@ -19,13 +19,5 @@ RUN apt-get update -qq \ software-properties-common \ && apt-get clean -# Install the same cuda version that the worker host uses -# See https://github.com/mozilla-platform-ops/worker-images/blob/main/scripts/linux/ubuntu-2404-amd64-headless/fxci/05-nvidia-gcp-driver-cudnn.sh -RUN curl -sSO https://developer.download.nvidia.com/compute/cudnn/9.6.0/local_installers/cudnn-local-repo-ubuntu2204-9.6.0_1.0-1_amd64.deb \ - && dpkg -i cudnn-local-repo-ubuntu2204-9.6.0_1.0-1_amd64.deb \ - && cp /var/cudnn-local-repo-ubuntu2204-9.6.0/cudnn-*-keyring.gpg /usr/share/keyrings/ \ - && apt-get update -qq \ - && apt-get install -y cudnn - VOLUME /builds/worker/checkouts VOLUME /builds/worker/.cache diff --git a/taskcluster/kinds/evaluate-quantized/kind.yml b/taskcluster/kinds/evaluate-quantized/kind.yml index 54dc5fdc8..cfcc25c51 100644 --- a/taskcluster/kinds/evaluate-quantized/kind.yml +++ b/taskcluster/kinds/evaluate-quantized/kind.yml @@ -92,6 +92,7 @@ tasks: - >- export PATH=$PATH:~/.local/bin && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && pip install --upgrade pip && pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt && pip install $VCS_PATH/tracking && @@ -131,3 +132,4 @@ tasks: # Quantized models are only supported via the browsermt fork of Marian. # https://github.com/browsermt/marian-dev - browsermt-marian + - cuda-toolkit diff --git a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml index 3f50496e1..61ed36893 100644 --- a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml +++ b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml @@ -98,6 +98,7 @@ tasks: - >- export PATH=$PATH:~/.local/bin && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && pip install --upgrade pip && pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt && pip install $VCS_PATH/tracking && @@ -142,3 +143,4 @@ tasks: extract: false toolchain: - marian + - cuda-toolkit diff --git a/taskcluster/kinds/evaluate/kind.yml b/taskcluster/kinds/evaluate/kind.yml index f78484ea7..0c8c99835 100644 --- a/taskcluster/kinds/evaluate/kind.yml +++ b/taskcluster/kinds/evaluate/kind.yml @@ -96,6 +96,7 @@ task-defaults: - >- export PATH=$PATH:~/.local/bin && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && pip install --upgrade pip && pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt && pip install $VCS_PATH/tracking && @@ -147,6 +148,7 @@ tasks: extract: false toolchain: - marian + - cuda-toolkit teacher-{provider}-{dataset_sanitized}-{src_locale}-{trg_locale}-{this_chunk}: description: teacher evaluation for {dataset} {src_locale}-{trg_locale} {this_chunk} @@ -192,6 +194,7 @@ tasks: extract: false toolchain: - marian + - cuda-toolkit student-{provider}-{dataset_sanitized}-{src_locale}-{trg_locale}: description: student evaluation for {dataset_sanitized} {src_locale}-{trg_locale} @@ -226,6 +229,7 @@ tasks: extract: false toolchain: - marian + - cuda-toolkit finetuned-student-{provider}-{dataset_sanitized}-{src_locale}-{trg_locale}: description: finetuned-student evaluation for {dataset_sanitized} {src_locale}-{trg_locale} @@ -260,3 +264,4 @@ tasks: extract: false toolchain: - marian + - cuda-toolkit diff --git a/taskcluster/kinds/finetune-student/kind.yml b/taskcluster/kinds/finetune-student/kind.yml index fe28d3a62..f8a65a9a1 100644 --- a/taskcluster/kinds/finetune-student/kind.yml +++ b/taskcluster/kinds/finetune-student/kind.yml @@ -101,6 +101,7 @@ tasks: export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && $VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py student finetune @@ -129,6 +130,7 @@ tasks: fetches: toolchain: - marian + - cuda-toolkit train-vocab: - artifact: vocab.spm extract: false diff --git a/taskcluster/kinds/score/kind.yml b/taskcluster/kinds/score/kind.yml index a74edcea2..0b43b9750 100644 --- a/taskcluster/kinds/score/kind.yml +++ b/taskcluster/kinds/score/kind.yml @@ -76,6 +76,7 @@ tasks: - -c - >- export MARIAN=$MOZ_FETCHES_DIR && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && find fetches && $VCS_PATH/pipeline/cefilter/score.sh $TASK_WORKDIR/fetches/final.model.npz.best-{best_model}.npz @@ -90,6 +91,7 @@ tasks: fetches: toolchain: - marian + - cuda-toolkit train-backwards: - artifact: final.model.npz.best-{best_model}.npz extract: false diff --git a/taskcluster/kinds/train-backwards/kind.yml b/taskcluster/kinds/train-backwards/kind.yml index 350184c53..7e2256404 100644 --- a/taskcluster/kinds/train-backwards/kind.yml +++ b/taskcluster/kinds/train-backwards/kind.yml @@ -106,6 +106,7 @@ tasks: export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && $VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py backward train @@ -131,6 +132,7 @@ tasks: fetches: toolchain: - marian + - cuda-toolkit merge-corpus: - artifact: corpus.{src_locale}.zst extract: false diff --git a/taskcluster/kinds/train-student/kind.yml b/taskcluster/kinds/train-student/kind.yml index 1f9db2a23..bb8812c62 100644 --- a/taskcluster/kinds/train-student/kind.yml +++ b/taskcluster/kinds/train-student/kind.yml @@ -102,6 +102,7 @@ tasks: export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && $VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py student train @@ -127,6 +128,7 @@ tasks: fetches: toolchain: - marian + - cuda-toolkit train-vocab: - artifact: vocab.spm extract: false diff --git a/taskcluster/kinds/train-teacher/kind.yml b/taskcluster/kinds/train-teacher/kind.yml index 21bde13e9..a8090eb0b 100644 --- a/taskcluster/kinds/train-teacher/kind.yml +++ b/taskcluster/kinds/train-teacher/kind.yml @@ -125,6 +125,7 @@ tasks: export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && $VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py teacher train @@ -151,6 +152,7 @@ tasks: fetches: toolchain: - marian + - cuda-toolkit train-vocab: - artifact: vocab.spm extract: false From c755fc022253a519a7909dce9788b177f00cb150 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Fri, 17 Jan 2025 16:19:27 -0500 Subject: [PATCH 9/9] fix reference to decoder.yml in evaluate-quantize --- taskcluster/kinds/evaluate-quantized/kind.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/kinds/evaluate-quantized/kind.yml b/taskcluster/kinds/evaluate-quantized/kind.yml index cfcc25c51..feeb87057 100644 --- a/taskcluster/kinds/evaluate-quantized/kind.yml +++ b/taskcluster/kinds/evaluate-quantized/kind.yml @@ -105,7 +105,7 @@ tasks: --vocab "$MOZ_FETCHES_DIR/vocab.spm" --shortlist "$MOZ_FETCHES_DIR/lex.s2t.pruned" --artifacts_prefix "$TASK_WORKDIR/artifacts/{dataset_sanitized}" - --marian_config "$TASK_WORKDIR/$VCS_PATH/pipeline/quantize/decoder.yml" + --marian_config "$VCS_PATH/pipeline/quantize/decoder.yml" --marian "$BMT_MARIAN" --gpus "$GPUS" --model_variant quantized