From 07369e534b7cf3edbb0da5023a0735d4cf2ebeae Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 15 Aug 2024 12:26:28 +0000 Subject: [PATCH 1/2] refactor: consume nixpkgs' release-cuda.nix --- effects/git-push/effects-fun.nix | 130 -------------- flake.lock | 23 +-- flake.nix | 54 ++---- nix/ci/aalto.nix | 40 ----- nix/ci/cuda-jetson.nix | 57 ------ nix/ci/cuda-updates.nix | 33 ---- nix/ci/cuda-x86_64-linux.nix | 68 -------- nix/ci/default.nix | 127 -------------- nix/ci/pytorch.nix | 9 - nix/ci/whisper.nix | 22 --- nix/gen-jobnames-list.nix | 13 -- nix/jobs.nix | 290 ------------------------------- nix/list-jobs-fancy.nix | 13 -- nix/overlays.nix | 121 ------------- nix/utils.nix | 86 --------- 15 files changed, 20 insertions(+), 1066 deletions(-) delete mode 100644 effects/git-push/effects-fun.nix delete mode 100644 nix/ci/aalto.nix delete mode 100644 nix/ci/cuda-jetson.nix delete mode 100644 nix/ci/cuda-updates.nix delete mode 100644 nix/ci/cuda-x86_64-linux.nix delete mode 100644 nix/ci/default.nix delete mode 100644 nix/ci/pytorch.nix delete mode 100644 nix/ci/whisper.nix delete mode 100644 nix/gen-jobnames-list.nix delete mode 100644 nix/jobs.nix delete mode 100644 nix/list-jobs-fancy.nix delete mode 100644 nix/overlays.nix delete mode 100644 nix/utils.nix diff --git a/effects/git-push/effects-fun.nix b/effects/git-push/effects-fun.nix deleted file mode 100644 index fe40ae31..00000000 --- a/effects/git-push/effects-fun.nix +++ /dev/null @@ -1,130 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: - -let - inherit (lib) mkOption optionalString types; - - cfg = config.git.push; - - # Copied from - # https://github.com/hercules-ci/hercules-ci-effects/blob/15ff4f63e5f28070391a5b09a82f6d5c6cc5c9d0/effects/modules/git-auth.nix#L10-L16 - parseURL = - gitRemote: - let - m = builtins.match "([a-z]*)://([^/]*)(/?.*)" gitRemote; - in - if m == null then - throw "Could not parse the git remote as a url. Value: ${gitRemote}" - else - { - scheme = lib.elemAt m 0; - host = lib.elemAt m 1; - path = lib.elemAt m 2; - }; - - source = parseURL cfg.source.url; - destination = parseURL cfg.destination.url; -in -{ - options = { - git.push.user = mkOption { - description = '' - GitHub repository to pull from - ''; - default = "x-access-token"; - type = types.str; - }; - git.push.source.url = mkOption { - description = '' - GitHub repository to pull from - ''; - type = types.str; - }; - git.push.source.ref = mkOption { - description = '' - The source ref to pull - ''; - type = types.str; - }; - git.push.source.tokenSecret = mkOption { - description = '' - Name of the secret that contains the git token for the source repo. - ''; - default = "token"; - type = types.str; - }; - git.push.destination.url = mkOption { - description = '' - GitHub repository to push to - ''; - type = types.str; - }; - git.push.destination.ref = mkOption { - description = '' - The new git ref to push as - ''; - type = types.str; - }; - git.push.destination.tokenSecret = mkOption { - description = '' - Name of the secret that contains the git token for the destination repo. - ''; - type = types.str; - }; - git.push.force = mkOption { - description = '' - Whether to --force push - ''; - type = types.bool; - default = false; - }; - git.push.beforePush = mkOption { - description = '' - Extra commands to run prior to `git push` - ''; - default = ""; - type = types.lines; - }; - }; - config = { - env.HCI_GIT_SOURCE_URL = cfg.source.url; - env.HCI_GIT_SOURCE_REF = cfg.source.ref; - env.HCI_GIT_DESTINATION_URL = cfg.destination.url; - env.HCI_GIT_DESTINATION_REF = cfg.destination.ref; - env.HCI_GIT_PUSH_FORCE = optionalString cfg.force "--force"; - - # Copy-pasted from https://github.com/hercules-ci/hercules-ci-effects/blob/15ff4f63e5f28070391a5b09a82f6d5c6cc5c9d0/effects/modules/git.nix - inputs = [ pkgs.git ]; - env = { - EMAIL = "hercules-ci[bot]@users.noreply.github.com"; - GIT_AUTHOR_NAME = "Hercules CI Effects"; - GIT_COMMITTER_NAME = "Hercules CI Effects"; - PAGER = "cat"; - }; - - effectScript = '' - # Based on https://github.com/hercules-ci/hercules-ci-effects/blob/15ff4f63e5f28070391a5b09a82f6d5c6cc5c9d0/effects/modules/git-auth.nix#L73-L74 - echo "${source.scheme}://${cfg.user}:$(readSecretString ${cfg.source.tokenSecret} .token)@${source.host}${source.path}" >>~/.git-credentials - echo "${destination.scheme}://${cfg.user}:$(readSecretString ${cfg.destination.tokenSecret} .token)@${destination.host}${destination.path}" >>~/.git-credentials - echo "${destination.scheme}://${cfg.user}:...@${destination.host}${destination.path}" - git config --global credential.helper store - git config --global init.defaultBranch "doNotPesterMeAboutMainsAndMasters" - - mkdir "repo" - cd "repo" - git init - git remote add "source" "$HCI_GIT_SOURCE_URL" - git fetch "source" "$HCI_GIT_SOURCE_REF" - git checkout -b source_branch "$HCI_GIT_SOURCE_REF" - git remote add "destination" "$HCI_GIT_DESTINATION_URL" - - ${cfg.beforePush} - - git push $HCI_GIT_PUSH_FORCE "destination" HEAD:"$HCI_GIT_DESTINATION_REF" - ''; - }; -} diff --git a/flake.lock b/flake.lock index b4281dd1..ab1305fe 100644 --- a/flake.lock +++ b/flake.lock @@ -43,29 +43,13 @@ "type": "github" } }, - "nixpkgs-cuda-updates": { - "locked": { - "lastModified": 1712133680, - "narHash": "sha256-tBe7Ivc6GSLHoVVs4nyWV3r3DmCZnPocwKXnQK9nwYM=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "a2954dea377e749d9cff5f71f3815330a5907ee9", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "cuda-updates", - "repo": "nixpkgs", - "type": "github" - } - }, "nixpkgs-master": { "locked": { - "lastModified": 1723634395, - "narHash": "sha256-K1ohl/M/HWXqieqNtsF39FbWqtaNX/mUgulrqcOO1KU=", + "lastModified": 1723724544, + "narHash": "sha256-Imp9b3YXmVMN17A2RQCdwATd6nE07rkXaNosFY9U+mA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "57d0d4a8f3025e2b902d2b4403bcece26ad1ea74", + "rev": "40dc65f56d8586a6bfb40f04ecf13dfff952a4b6", "type": "github" }, "original": { @@ -146,7 +130,6 @@ "nixpkgs": [ "nixpkgs-master" ], - "nixpkgs-cuda-updates": "nixpkgs-cuda-updates", "nixpkgs-master": "nixpkgs-master", "nixpkgs-nixos-unstable": "nixpkgs-nixos-unstable", "nixpkgs-nixpkgs-unstable": "nixpkgs-nixpkgs-unstable", diff --git a/flake.nix b/flake.nix index f0e5ff17..16f1689f 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,6 @@ nixpkgs-nixos-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; nixpkgs-release.url = "github:NixOS/nixpkgs/nixos-24.05"; nixpkgs-release-staging.url = "github:NixOS/nixpkgs/staging-24.05"; - nixpkgs-cuda-updates.url = "github:NixOS/nixpkgs/cuda-updates"; hercules-ci-effects = { url = "github:hercules-ci/hercules-ci-effects"; @@ -32,48 +31,29 @@ outputs = inputs@{ self, flake-parts, ... }: - flake-parts.lib.mkFlake { inherit inputs; } { - imports = [ - inputs.hercules-ci-effects.flakeModule - ./nix/ci - ]; + let systems = [ "x86_64-linux" ]; + inherit (inputs) nixpkgs; + inherit (nixpkgs) lib; + in + flake-parts.lib.mkFlake { inherit inputs; } { + inherit systems; flake = let - - inherit (inputs) nixpkgs; - inherit (nixpkgs) lib; - - systems = [ "x86_64-linux" ]; - eachSystem = lib.genAttrs systems; - - x = eachSystem ( - system: - import ./nix/jobs.nix { - inherit system nixpkgs lib; - extraConfig.cudaCapabilities = [ - "7.0" - "8.0" - "8.6" - ]; - extraConfig.cudaEnableForwardCompat = false; - } - ); + eachBranch = lib.genAttrs [ "master" ]; in { - # Inherit from upstream - inherit (nixpkgs) lib; # nixosModules htmlDocs; - - # But replace legacyPackages with the unfree version - legacyPackages = eachSystem (system: x.${system}.legacyPackages); - - # And load all the unfree+redistributable packages as checks - checks = eachSystem (system: x.${system}.neverBreak); - - # Expose our own unfree overrides - overlays = import ./nix/overlays.nix; - overlay = self.overlays.basic; + herculesCI.onPush.default.outputs = eachSystem ( + system: + (eachBranch ( + branch: + import (inputs.nixpkgs + "/pkgs/top-level/release-cuda.nix") { + inherit system; + packageSet = import inputs."nixpkgs-${branch}"; + } + )) + ); }; }; } diff --git a/nix/ci/aalto.nix b/nix/ci/aalto.nix deleted file mode 100644 index 41587894..00000000 --- a/nix/ci/aalto.nix +++ /dev/null @@ -1,40 +0,0 @@ -let - default = { - cuda = { - capabilities = [ - "7.0" - "8.0" - "8.6" - ]; - forwardCompat = false; - }; - reason = "Aalto Triton + RTX3090"; - system = "x86_64-linux"; - }; -in -{ - # Aalto Triton + RTX3090 - hci.jobSets = { - aalto-master-daily-neverBreak = default // { - branch = "master"; - jobsAttr = "neverBreak"; - when.hour = [ - 1 - 17 - ]; - }; - aalto-nixos-unstable-daily-neverBreak = default // { - branch = "nixos-unstable"; - jobsAttr = "neverBreak"; - when.hour = [ - 3 - 21 - ]; - }; - aalto-nixos-unstable-weekly-checks = default // { - branch = "nixos-unstable"; - jobsAttr = "checks"; - when.dayOfWeek = [ "Sat" ]; - }; - }; -} diff --git a/nix/ci/cuda-jetson.nix b/nix/ci/cuda-jetson.nix deleted file mode 100644 index afc4a934..00000000 --- a/nix/ci/cuda-jetson.nix +++ /dev/null @@ -1,57 +0,0 @@ -let - default = { - branch = "master"; - cuda.forwardCompat = false; - jobsAttr = "neverBreak"; - system = "aarch64-linux"; - }; - defaultUnstable = default // { - branch = "nixos-unstable"; - }; -in -{ - hci.jobSets = { - cuda-jetson-orin-master-weekly-neverBreak = default // { - cuda.capabilities = [ "8.7" ]; - reason = "Jetson Orin"; - when.dayOfWeek = [ "Fri" ]; - }; - cuda-jetson-xavier-master-weekly-neverBreak = default // { - cuda.capabilities = [ "7.2" ]; - reason = "Jetson Xavier"; - when.dayOfWeek = [ "Fri" ]; - }; - cuda-jetson-tx2-master-weekly-neverBreak = default // { - cuda.capabilities = [ "6.2" ]; - reason = "Jetson TX2"; - when.dayOfWeek = [ "Fri" ]; - }; - cuda-jetson-nano-master-weekly-neverBreak = default // { - cuda.capabilities = [ "5.3" ]; - reason = "Jetson Nano"; - when.dayOfWeek = [ "Fri" ]; - }; - - # Testing out, for llama.cpp's CI - cuda-jetson-orin-unstable-weekly-neverBreak = defaultUnstable // { - cuda.capabilities = [ "8.7" ]; - reason = "Jetson Orin"; - when.dayOfWeek = [ "Fri" ]; - }; - cuda-jetson-xavier-unstable-weekly-neverBreak = defaultUnstable // { - cuda.capabilities = [ "7.2" ]; - reason = "Jetson Xavier"; - when.dayOfWeek = [ "Fri" ]; - }; - cuda-jetson-tx2-unstable-weekly-neverBreak = defaultUnstable // { - cuda.capabilities = [ "6.2" ]; - reason = "Jetson TX2"; - when.dayOfWeek = [ "Fri" ]; - }; - cuda-jetson-nano-unstable-weekly-neverBreak = defaultUnstable // { - cuda.capabilities = [ "5.3" ]; - reason = "Jetson Nano"; - when.dayOfWeek = [ "Fri" ]; - }; - }; -} diff --git a/nix/ci/cuda-updates.nix b/nix/ci/cuda-updates.nix deleted file mode 100644 index f25a6a74..00000000 --- a/nix/ci/cuda-updates.nix +++ /dev/null @@ -1,33 +0,0 @@ -let - allWeek = [ - "Mon" - "Tue" - "Wed" - "Thu" - "Fri" - "Sat" - "Sun" - ]; -in -{ - hci.jobSets = { - sm_86-cuda-updates-daily = { - branch = "master"; - jobsAttr = "neverBreak"; - cuda = { - capabilities = [ "8.6" ]; - forwardCompat = false; - }; - reason = "reused for nixpkgs-review"; - system = "x86_64-linux"; - when.dayOfWeek = allWeek; - }; - default-cuda-updates-daily = { - jobsAttr = "neverBreak"; - reason = "staging mass rebuilds"; - system = "x86_64-linux"; - branch = "master"; - when.dayOfWeek = allWeek; - }; - }; -} diff --git a/nix/ci/cuda-x86_64-linux.nix b/nix/ci/cuda-x86_64-linux.nix deleted file mode 100644 index 6eddc227..00000000 --- a/nix/ci/cuda-x86_64-linux.nix +++ /dev/null @@ -1,68 +0,0 @@ -let - sm_86-checks = { - cuda = { - capabilities = [ "8.6" ]; - forwardCompat = false; - }; - jobsAttr = "checks"; - reason = "reused for nixpkgs-review"; - system = "x86_64-linux"; - }; - default-checks = { - jobsAttr = "checks"; - reason = "check"; - system = "x86_64-linux"; - }; -in -{ - hci.jobSets = { - cuda-sm_86-master-daily-neverBreak = sm_86-checks // { - branch = "master"; - jobsAttr = "neverBreak"; - when.hour = [ - 0 - 2 - 20 - 22 - ]; - }; - cuda-sm_86-master-daily-checks = sm_86-checks // { - branch = "master"; - reason = "daily check"; - when.hour = [ 21 ]; - }; - cuda-sm_86-nixos-unstable-weekly-checks = sm_86-checks // { - branch = "nixos-unstable"; - when.dayOfWeek = [ "Fri" ]; - }; - cuda-sm_86-nixpkgs-unstable-weekly-checks = sm_86-checks // { - branch = "nixpkgs-unstable"; - when.dayOfWeek = [ "Sat" ]; - }; - cuda-default-release-staging-weekly-checks = default-checks // { - branch = "release-staging"; - when.dayOfWeek = [ "Wed" ]; - }; - cuda-default-release-daily-checks = default-checks // { - branch = "release"; - when = { - hour = [ 21 ]; - }; - }; - cuda-default-master-weekly-checks = default-checks // { - branch = "master"; - when = { - dayOfWeek = [ "Fri" ]; - hour = [ 21 ]; - }; - }; - cuda-default-nixpkgs-unstable-weekly-checks = default-checks // { - branch = "nixpkgs-unstable"; - when.dayOfWeek = [ "Sat" ]; - }; - cuda-default-nixos-unstable-weekly-checks = default-checks // { - branch = "nixos-unstable"; - when.dayOfWeek = [ "Sat" ]; - }; - }; -} diff --git a/nix/ci/default.nix b/nix/ci/default.nix deleted file mode 100644 index 76485b00..00000000 --- a/nix/ci/default.nix +++ /dev/null @@ -1,127 +0,0 @@ -{ - config, - inputs, - lib, - withSystem, - ... -}: -let - inherit (lib) attrsets types; - inherit (lib.options) mkOption; -in -{ - imports = [ - ./aalto.nix - ./cuda-jetson.nix - ./cuda-x86_64-linux.nix - ./pytorch.nix - ./whisper.nix - ./cuda-updates.nix - ]; - options.hci.jobSets = mkOption { - description = "Job sets to build"; - type = types.attrsOf ( - types.submodule { - options = { - branch = mkOption { - description = "The branch to build"; - type = types.enum [ - "master" - "nixos-unstable" - "nixpkgs-unstable" - "release" - "release-staging" - ]; - }; - cuda = { - capabilities = mkOption { - description = '' - List of CUDA capabilities to build for. - If empty, the default capabilities are used. - ''; - default = [ ]; - type = types.listOf types.nonEmptyStr; - }; - forwardCompat = mkOption { - description = "Whether to enable forward compatibility for CUDA"; - default = true; - type = types.bool; - }; - }; - jobsAttr = mkOption { - description = "The attribute set to build"; - type = types.enum [ - "neverBreak" - "checks" - ]; - }; - reason = mkOption { - description = "The reason for building"; - type = types.nonEmptyStr; - }; - system = mkOption { - description = "The system to build for"; - type = types.enum [ - "x86_64-linux" - "aarch64-linux" - ]; - }; - when = - (import "${inputs.hercules-ci-effects}/flake-modules/types/when.nix" { inherit lib; }).option; - }; - } - ); - }; - config.herculesCI = { - onSchedule = - let - mkOnScheduleEntry = - { - branch, - cuda, - jobsAttr, - reason, - system, - when, - }: - let - jobs = import ../jobs.nix { - inherit system; - nixpkgs = inputs."nixpkgs-${branch}"; - extraConfig = { - allowInsecurePredicate = x: true; - allowUnfree = true; - cudaSupport = true; - cudaEnableForwardCompat = cuda.forwardCompat; - } // attrsets.optionalAttrs (cuda.capabilities != [ ]) { cudaCapabilities = cuda.capabilities; }; - }; - in - { - inherit when; - outputs = jobs.${jobsAttr}; - }; - in - attrsets.mapAttrs (_: mkOnScheduleEntry) config.hci.jobSets; - onPush.default.outputs = { - effects = withSystem "x86_64-linux" ( - { hci-effects, pkgs, ... }: - { - releaseBranch = hci-effects.modularEffect { - imports = [ ../../effects/git-push/effects-fun.nix ]; - git.push = { - source = { - url = "https://github.com/NixOS/nixpkgs.git"; - ref = inputs.nixpkgs-master.rev; - }; - destination = { - url = "https://github.com/9d80dba85131ab22/nixpkgs.git"; - ref = "buildNixosUnstable80"; - tokenSecret = "nixpkgCuda"; - }; - }; - }; - } - ); - }; - }; -} diff --git a/nix/ci/pytorch.nix b/nix/ci/pytorch.nix deleted file mode 100644 index 8ec8c619..00000000 --- a/nix/ci/pytorch.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - hci.jobSets.pytorch-default-master-daily-neverBreak = { - branch = "master"; - jobsAttr = "neverBreak"; - reason = "daily pytorch&c build"; - system = "x86_64-linux"; - when.hour = [ 3 ]; - }; -} diff --git a/nix/ci/whisper.nix b/nix/ci/whisper.nix deleted file mode 100644 index 877540ea..00000000 --- a/nix/ci/whisper.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ inputs, ... }: -let - pkgs = import inputs."nixpkgs-nixos-unstable" { - system = "x86_64-linux"; - config = { - allowUnfree = true; - cudaSupport = true; - cudaCapabilities = [ "5.2" ]; - }; - }; -in -{ - # Build faster-whisper for 5.2+PTX, weekly - config.herculesCI.onSchedule.fasterWhisper-5_2-nixos-unstable-weekly = { - when.dayOfWeek = [ "Wed" ]; - outputs = { - torch = pkgs.python3Packages.torch; - faster-whisper = pkgs.python3Packages.faster-whisper; - faster-whisper-server = pkgs.wyoming-faster-whisper; - }; - }; -} diff --git a/nix/gen-jobnames-list.nix b/nix/gen-jobnames-list.nix deleted file mode 100644 index eec7f90f..00000000 --- a/nix/gen-jobnames-list.nix +++ /dev/null @@ -1,13 +0,0 @@ -let - f = builtins.getFlake (builtins.toString ./..); - ci = f.herculesCI { }; - pkgs = f.legacyPackages.${builtins.currentSystem}; - inherit (pkgs) lib; - utils = import ./utils.nix; - - checks = ci.onSchedule.buildMasterAmpereMatrix.outputs; - - paths = lib.mapAttrs (name: drv: builtins.unsafeDiscardStringContext drv.outPath) checks; - pathsTxt = builtins.attrValues (lib.mapAttrs (name: path: "${name} -> ${path}") paths); -in -paths diff --git a/nix/jobs.nix b/nix/jobs.nix deleted file mode 100644 index 7072ea66..00000000 --- a/nix/jobs.nix +++ /dev/null @@ -1,290 +0,0 @@ -# FIXME: make filter-jobs.nix instead, and import nixpkgs in the flake - -{ - system ? builtins.currentSystem, - nixpkgs ? (builtins.getFlake (builtins.toString ./.)).inputs.nixpkgs, - extraConfig ? { }, - lib ? nixpkgs.lib, - debug ? false, -}: -let - utils = import ./utils.nix { inherit debug; }; - - inherit (utils) maybeBuildable isSupportedCuPackage; - - overlays = import ./overlays.nix; - nixpkgsInstances = lib.mapAttrs ( - configName: overlay: - import nixpkgs ({ - inherit system; - config = { - allowInsecurePredicate = x: true; - allowUnfree = true; - cudaSupport = true; - } // extraConfig; - overlays = [ overlay ]; - }) - ) overlays; - - neverBreakExtra = [ - # Messy, but vital to keep cached: - [ "blender" ] - [ "colmapWithCuda" ] - [ "faiss" ] - [ "tts" ] - ]; - - extraPackages = neverBreakExtra ++ [ - [ "blas" ] - [ "cudnn" ] - [ "lapack" ] - [ "mpich" ] - [ "nccl" ] - [ "opencv" ] - [ "openmpi" ] - [ "ucx" ] - - [ "cctag" ] # Failed in https://github.com/NixOS/nixpkgs/pull/233581 - [ "cholmod-extra" ] - [ "ctranslate2" ] - [ "gpu-screen-recorder" ] - [ "lightgbm" ] - [ "llama-cpp" ] - [ "monado" ] # Failed in https://github.com/NixOS/nixpkgs/pull/233581 - [ - "obs-studio-plugins" - "obs-backgroundremoval" - ] - [ "ollama" ] - [ "onnxruntime" ] - [ "openmvg" ] - [ "openmvs" ] - [ "opentrack" ] - [ "openvino" ] - [ "pixinsight" ] # Failed in https://github.com/NixOS/nixpkgs/pull/233581 - [ "rtabmap" ] - [ "suitesparse" ] - [ "truecrack-cuda" ] - [ "wyoming-faster-whisper" ] - [ "xgboost" ] - - # GUI and similar mess, but desirable to have in cache: - [ - "deepin" - "image-editor" - ] # Failed in https://github.com/NixOS/nixpkgs/pull/233581 - [ "ffmpeg-full" ] - [ "gimp" ] - [ - "gst_all_1" - "gst-plugins-bad" - ] - [ "meshlab" ] - [ "qgis" ] - [ "saga" ] - [ "ueberzugpp" ] # Failed in https://github.com/NixOS/nixpkgs/pull/233581 - - [ - "linuxPackages" - "nvidia_x11" - ] - [ - "linuxPackages" - "nvidia_x11_beta" - ] - [ - "linuxPackages" - "nvidia_x11_beta_open" - ] - [ - "linuxPackages" - "nvidia_x11_vulkan_beta_open" - ] - ]; - - pythonAttrs = - let - matrix = lib.cartesianProductOfSets { - pkg = - [ - "Keras" - "TheanoWithCuda" - "boxx" - "bpycv" - "caffe" - "catboost" - "chainer" - "cupy" - "faiss" - "faster-whisper" - "ffmpeg-full" - "flax" - "gpt-2-simple" - "grad-cam" - "jax" - "jaxlib" - "jaxlib" - "kornia" - "libgpuarray" - "mmcv" - "mxnet" - "noisetorch" - "onnx" - "openai-triton" - "openai-whisper" - "opencv4" - "pycuda" - "pyrealsense2WithCuda" - "pytorch" - "pytorch-lightning" - "scikit-learn" # makes sure it's cached with MKL - "scikitimage" - "spacy-transformers" - "tensorflow-probability" - "tensorflowWithCuda" - "tesserocr" - "tiny-cuda-nn" - "torch" - "torchaudio" - "torchvision" - "transformers" - "tts" - "vidstab" - ] - ++ [ - # These need to be rebuilt because of MKL - "numpy" - "scipy" - ]; - ps = [ - "python3Packages" - # "python310Packages" - # "python311Packages" - ]; - }; - - mkPath = - { pkg, ps }: - [ - ps - pkg - ]; - in - builtins.map mkPath matrix; - - hasFridhPR = nixpkgs: nixpkgs.cudaPackages ? "overrideScope'"; - - cudaPackages = lib.concatMap ( - cfg: - let - nixpkgs = nixpkgsInstances.${cfg}; - jobs = builtins.map (pkg: { - inherit cfg; - path = [ - "cudaPackages" - pkg - ]; - }) (builtins.attrNames (lib.filterAttrs isSupportedCuPackage nixpkgs.cudaPackages)); - in - if hasFridhPR nixpkgs then jobs else [ ] - ) (builtins.attrNames overlays); - - checks = - let - matrix = - lib.cartesianProductOfSets { - cfg = builtins.attrNames overlays; - path = extraPackages ++ pythonAttrs; - } - ++ cudaPackages; - supported = builtins.concatMap ( - { cfg, path }: - let - jobName = lib.concatStringsSep "." ([ cfg ] ++ path); - package = lib.attrByPath path [ ] nixpkgsInstances.${cfg}; - mbSupported = maybeBuildable package; - in - if mbSupported == [ ] then - [ ] - else - [ - { - inherit jobName; - package = (builtins.head mbSupported); - } - ] - ) matrix; - kvPairs = builtins.map ({ jobName, package }: lib.nameValuePair jobName package) supported; - - inherit (utils) dedupOutpaths; - in - lib.listToAttrs (dedupOutpaths kvPairs); - - # List packages that we never want to be even marked as "broken" - # These will be checked just for x86_64-linux and for one release of python - neverBreak = - let - pkgs = nixpkgsInstances.basic; - cuPackages = lib.attrNames ( - lib.filterAttrs ( - name: value: - (isSupportedCuPackage name value) - || builtins.elem name [ - "saxpy" - "cuda-samples" - "cuda_nvcc" - "cuda_cuobjdump" - "cuda_cudart" - ] - ) pkgs.cudaPackages - ); - latestPython = "python3Packages"; - pyPackages = [ - "imageio-ffmpeg" - "jax" - "jaxlib" - "kornia" - "opencv4" - "opensfm" - "tensorflowWithCuda" - "torch" - "torchaudio" - "torchvision" - ]; - matrixPy = lib.cartesianProductOfSets { - pkg = pyPackages; - ps = [ latestPython ]; - }; - matrixCu = lib.cartesianProductOfSets { - pkg = cuPackages; - ps = [ "cudaPackages" ]; - }; - mkPath = - { pkg, ps }: - [ - ps - pkg - ]; - matrix = neverBreakExtra ++ builtins.map mkPath (matrixPy ++ matrixCu); - jobs = builtins.concatMap ( - path: - let - jobName = lib.concatStringsSep "." path; - package = lib.attrByPath path [ ] pkgs; - in - assert builtins.isList path; - assert builtins.isString (builtins.head path); - [ { inherit jobName package; } ] - ) matrix; - kvPairs = builtins.map ({ jobName, package }: lib.nameValuePair jobName package) jobs; - in - assert builtins.isList matrix; - (lib.listToAttrs kvPairs); -in -{ - # Export the whole tree - legacyPackages = nixpkgsInstances.basic; - - # Returns the recursive set of unfree but redistributable packages as checks - inherit checks neverBreak; -} diff --git a/nix/list-jobs-fancy.nix b/nix/list-jobs-fancy.nix deleted file mode 100644 index 13ab3f67..00000000 --- a/nix/list-jobs-fancy.nix +++ /dev/null @@ -1,13 +0,0 @@ -let - f = builtins.getFlake (builtins.toString ./..); - ci = f.herculesCI { }; - pkgs = f.legacyPackages.${builtins.currentSystem}; - inherit (pkgs) lib; - utils = import ./utils.nix; - - checks = ci.onSchedule.buildMasterAmpereMatrix.outputs; - paths = lib.mapAttrs ( - name: drv: "${name} -> ${builtins.unsafeDiscardStringContext drv.outPath}" - ) checks; -in -paths diff --git a/nix/overlays.nix b/nix/overlays.nix deleted file mode 100644 index fa752954..00000000 --- a/nix/overlays.nix +++ /dev/null @@ -1,121 +0,0 @@ -let - prepareOverlay = - { - isIntel ? false, - # cud(a|nn)Version - # - # : Maybe String - # Semver ( or ) naming - # the cuda or cudnn revision to set as the default - cudaVersion ? null, - cudnnVersion ? null, - # *mpiProvider* - # - # : Maybe String - # If null: don't touch mpi. - # If str: names the attribute providing mpi implementation - # and adds a few MPISupport=true overrides in selected packages - mpiProvider ? null, - }: - final: prev: - let - inherit (prev) lib; - inherit (lib) optionalAttrs versionOlder replaceChars; - overrideMpi = !(builtins.isNull mpiProvider); - in - ( - lib.optionalAttrs (versionOlder lib.version "24.05pre-git") { - # These ignore config.cudaSupport in some releases - - openmpi = prev.openmpi.override { cudaSupport = true; }; - - # https://github.com/NixOS/nixpkgs/issues/239182 - cudaPackages = prev.cudaPackages.overrideScope' ( - fin: pre: { - cudatoolkit = pre.cudatoolkit.override { ucx = final.ucx.override { enableCuda = false; }; }; - } - ); - - ucx = prev.ucx.override { enableCuda = true; }; - - suitesparse = prev.suitesparse.override { enableCuda = true; }; - } - // optionalAttrs overrideMpi { - mpi = final.${mpiProvider}; - - # Instead of libfabric - mpich = prev.mpich.override { ch4backend = final.ucx; }; - - # TODO: pythonPackageOverrides - pytorchMpi = prev.python3Packages.pytorch.override { MPISupport = true; }; - } - // optionalAttrs isIntel ( - let - # Not using optionalAttrs so as to avoid infinite recursion - mklBlas = prev.blas.override { blasProvider = final.mkl; }; - mklLapack = prev.lapack.override { lapackProvider = final.mkl; }; - cond = prev.hostPlatform.is64bit; - in - { - blas = if cond then mklBlas else prev.blas; - lapack = if cond then mklLapack else prev.lapack; - opencvWithTbb = prev.opencv.override { enableTbb = true; }; - } - ) - // ( - let - dontOverride = builtins.isNull cudaVersion && builtins.isNull cudnnVersion; - - versionToAttr = v: if builtins.isNull v then "" else "_${replaceChars [ "." ] [ "_" ] v}"; - cudaAttr = versionToAttr cudaVersion; - cudnnAttr = versionToAttr cudnnVersion; - - overlays."21.11" = - # using prev in assert to avoid infinite recursion - assert (builtins.isNull cudnnVersion || cudnnVersion == prev."cudnn_cudatoolkit${cudaAttr}"); - { - cudatoolkit_11 = final."cudatoolkit${cudaAttr}"; - cudatoolkit = final."cudatoolkit${cudaAttr}"; - cudnn = final."cudnn_cudatoolkit${cudaAttr}"; - cutensor = final."cutensor_cudatoolkit${cudaAttr}"; - }; - overlays."22.05" = { - # Assuming Fridh's PR has been merged - cudaPackages = prev."cudaPackages${cudaAttr}".overrideScope' ( - final: prev: { cudnn = final."cudnn${cudnnAttr}"; } - ); - }; - - release = prev.lib.version; - overlay = - if versionOlder release "21.12" then - "21.11" - else if versionOlder release "22.06" then - "22.05" - else if versionOlder release "24.05" then - "24.05" - else - throw "Unsuported nixpkgs release: ${release}"; - in - optionalAttrs (!dontOverride) overlays.${overlay} - ) - ); -in -{ - # Lexicographic ordering of the names matters for the groupBy in default.nix - # Overrides that don't change the derivation (compared to basic) - # won't show up as attributes in the jobset - # (as long as config's name is lexicographically bigger than "basic") - - basic = prepareOverlay { }; - - intel = prepareOverlay { isIntel = true; }; - - # mpich = prepareOverlay { - # mpiProvider = "mpich"; - # }; - - # openmpi = prepareOverlay { - # mpiProvider = "openmpi"; - # }; -} diff --git a/nix/utils.nix b/nix/utils.nix deleted file mode 100644 index 654cd5a7..00000000 --- a/nix/utils.nix +++ /dev/null @@ -1,86 +0,0 @@ -{ - debug ? false, -}: -rec { - optionals = - cond: lst: - assert builtins.isBool cond; - assert builtins.isList lst; - if cond then lst else [ ]; - optionalString = cond: str: if cond then str else ""; - - # nixpkgs' lib.lists.toList - ensureList = x: if builtins.isList x then x else [ x ]; - - # Copy-paste from nixpkgs - hasPrefix = - # Prefix to check for - pref: - # Input string - str: - builtins.substring 0 (builtins.stringLength pref) str == pref; - - trace = if debug then builtins.trace else (msg: value: value); - - # This used to be a tryEval-based routine similar to nixpkgs/maintainers/scripts/check-hydra-by-maintainer.nix - # Now we only respect `platforms`/`badPlatforms`, not `broken`. - maybeBuildable = v: optionals (isDerivation v && !(v.meta.unsupported or false)) [ v ]; - - # removed packages (like cudatoolkit_6) are just aliases that `throw`: - notRemoved = pkg: (builtins.tryEval (builtins.seq pkg true)).success; - - isUnfreeRedistributable = - licenses: builtins.any (l: (!l.free or true) && (l.redistributable or false)) licenses; - - hasLicense = pkg: pkg ? meta.license; - - hasUnfreeRedistributableLicense = - pkg: hasLicense pkg && isUnfreeRedistributable (ensureList pkg.meta.license); - - isDerivation = a: a ? type && a.type == "derivation"; - - # Picking out the redist parts of cuda - # and specifically ignoring the runfile-based cudatoolkit - cuPrefixae = [ - "cudnn" - "cutensor" - "cuda_" - "cuda-" - "libcu" - "libnv" - "libnpp" - "nccl" - "nsight_systems" - "nsight_compute" - "nvidia_" - # Requires manually adding trt to /nix/store: - # "tensorrt" - ]; - unsupportedCuPackages = [ - "cuda-samples" - "nvidia_driver" - "tensorrt" - "tensorrt_8_4_0" - ]; - isCuPackage = - name: drv: - (notRemoved drv) && (isDerivation drv) && (builtins.any (p: hasPrefix p name) cuPrefixae); - isSupportedCuPackage = - name: drv: (isCuPackage name drv) && !builtins.elem name unsupportedCuPackages; - - resultToString = { success, value }: optionalString success value; - - dedupOutpaths = - nameDrvPairs: - let - outPathToPair = builtins.groupBy ( - pair: - (builtins.unsafeDiscardStringContext ( - resultToString (builtins.tryEval (builtins.seq pair.value.outPath pair.value.outPath)) - )) - ) nameDrvPairs; - groupedPairs = builtins.attrValues outPathToPair; - uniquePairs = builtins.map builtins.head groupedPairs; - in - builtins.filter ({ name, value }: name != "") uniquePairs; -} From d6f1f3f019d7b18101bedfa7dbb65a2249a263cb Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 15 Aug 2024 12:31:11 +0000 Subject: [PATCH 2/2] re: release-cuda.nix; notice in README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c492d627..60fc0977 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ With the above in mind, let's proceed. ## Latest warnings +- 2024-08-15: As of https://github.com/SomeoneSerge/nixpkgs-cuda-ci/pull/33, + this repo no longer maintains any jobsets and consumes instead the new + `release-cuda.nix` maintained in Nixpkgs - 2024-06-09: I had been having issues with the Hercules CI's "effects" queue getting jammed, effectively shutting down the "flake-update" effect. The effects were disabled and replaced with a flake-update github action, which