diff --git a/lib/modules.nix b/lib/modules.nix index f5b8af0e9858b..f9441684cf15a 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -844,6 +844,8 @@ let in warnDeprecation opt // { value = addErrorContext "while evaluating the option `${showOption loc}':" value; + # raw value before "apply" above + rawValue = addErrorContext "while evaluating the option `${showOption loc}':" res.mergedValue; inherit (res.defsFinal') highestPrio; definitions = map (def: def.value) res.defsFinal; files = map (def: def.file) res.defsFinal; diff --git a/nixos/modules/misc/nixpkgs.nix b/nixos/modules/misc/nixpkgs.nix index e1c705eef3b5a..f99f75af7659b 100644 --- a/nixos/modules/misc/nixpkgs.nix +++ b/nixos/modules/misc/nixpkgs.nix @@ -70,19 +70,24 @@ let ++ lib.optional (opt.localSystem.highestPrio < (lib.mkOptionDefault { }).priority) opt.localSystem ++ lib.optional (opt.crossSystem.highestPrio < (lib.mkOptionDefault { }).priority) opt.crossSystem; + # pkgs/top-level/default.nix takes great strides to pass the *original* localSystem/crossSystem args + # on to nixpkgsFun to create package sets like pkgsStatic, pkgsMusl. This is to be able to infer default + # values again. Since cfg.xxxPlatform and cfg.xxxSystem are elaborated via apply, those can't be passed + # directly. Instead we use the rawValue before the apply/elaboration step, via opt.xxx.rawValue. defaultPkgs = if opt.hostPlatform.isDefined then let + # This compares elaborated systems on purpose, **not** using rawValue. isCross = cfg.buildPlatform != cfg.hostPlatform; systemArgs = if isCross then { - localSystem = cfg.buildPlatform; - crossSystem = cfg.hostPlatform; + localSystem = opt.buildPlatform.rawValue; + crossSystem = opt.hostPlatform.rawValue; } else { - localSystem = cfg.hostPlatform; + localSystem = opt.hostPlatform.rawValue; }; in import ../../.. ( @@ -96,9 +101,9 @@ let inherit (cfg) config overlays - localSystem - crossSystem ; + localSystem = opt.localSystem.rawValue; + crossSystem = opt.crossSystem.rawValue; }; finalPkgs = if opt.pkgs.isDefined then cfg.pkgs.appendOverlays cfg.overlays else defaultPkgs; diff --git a/pkgs/test/top-level/stage.nix b/pkgs/test/top-level/stage.nix new file mode 100644 index 0000000000000..26e167fe3d031 --- /dev/null +++ b/pkgs/test/top-level/stage.nix @@ -0,0 +1,104 @@ +# run like this: +# nix-build pkgs/test/top-level/stage.nix +{ + localSystem ? { + system = builtins.currentSystem; + }, +}: + +with import ../../top-level { inherit localSystem; }; + +let + # To silence platform specific evaluation errors + discardEvaluationErrors = e: (builtins.tryEval e).success -> e; + + # Basic test for idempotency of the package set, i.e: + # Applying the same package set twice should work and + # not change anything. + isIdempotent = set: discardEvaluationErrors (pkgs.${set}.stdenv == pkgs.${set}.${set}.stdenv); + + # Some package sets should be noops in certain circumstances. + # This is very similar to the idempotency test, but not going + # via the super' overlay. + isNoop = + parent: child: + discardEvaluationErrors ( + (lib.getAttrFromPath parent pkgs).stdenv == (lib.getAttrFromPath parent pkgs).${child}.stdenv + ); + + allMuslExamples = builtins.attrNames ( + lib.filterAttrs (_: system: lib.hasSuffix "-musl" system.config) lib.systems.examples + ); + + allLLVMExamples = builtins.attrNames ( + lib.filterAttrs (_: system: system.useLLVM or false) lib.systems.examples + ); + + # A package set should only change specific configuration, but needs + # to keep all other configuration from previous layers in place. + # Each package set has one or more key characteristics for which we + # test here. Those should be kept, even when applying the "set" package + # set. + isComposable = + set: + discardEvaluationErrors ( + pkgsCross.mingwW64.${set}.stdenv.hostPlatform.config == "x86_64-w64-mingw32" + ) + && discardEvaluationErrors (pkgsCross.mingwW64.${set}.stdenv.hostPlatform.libc == "msvcrt") + && discardEvaluationErrors (pkgsCross.ppc64-musl.${set}.stdenv.hostPlatform.gcc.abi == "elfv2") + && discardEvaluationErrors ( + builtins.elem "trivialautovarinit" pkgs.pkgsExtraHardening.${set}.stdenv.cc.defaultHardeningFlags + ) + && discardEvaluationErrors (pkgs.pkgsLLVM.${set}.stdenv.hostPlatform.useLLVM) + && discardEvaluationErrors (pkgs.pkgsArocc.${set}.stdenv.hostPlatform.useArocc) + && discardEvaluationErrors (pkgs.pkgsZig.${set}.stdenv.hostPlatform.useZig) + && discardEvaluationErrors (pkgs.pkgsLinux.${set}.stdenv.buildPlatform.isLinux) + && discardEvaluationErrors (pkgs.pkgsMusl.${set}.stdenv.hostPlatform.isMusl) + && discardEvaluationErrors (pkgs.pkgsStatic.${set}.stdenv.hostPlatform.isStatic) + && discardEvaluationErrors (pkgs.pkgsi686Linux.${set}.stdenv.hostPlatform.isx86_32) + && discardEvaluationErrors (pkgs.pkgsx86_64Darwin.${set}.stdenv.hostPlatform.isx86_64); +in + +# Appends same defaultHardeningFlags again on each .pkgsExtraHardening - thus not idempotent. +# assert isIdempotent "pkgsExtraHardening"; +# TODO: Remove the isDarwin condition, which currently results in infinite recursion. +# Also see https://github.com/NixOS/nixpkgs/pull/330567#discussion_r1894653309 +assert (stdenv.hostPlatform.isDarwin || isIdempotent "pkgsLLVM"); +assert isIdempotent "pkgsArocc"; +assert isIdempotent "pkgsZig"; +assert isIdempotent "pkgsLinux"; +assert isIdempotent "pkgsMusl"; +assert isIdempotent "pkgsStatic"; +assert isIdempotent "pkgsi686Linux"; +assert isIdempotent "pkgsx86_64Darwin"; + +assert isNoop [ "pkgsStatic" ] "pkgsMusl"; +assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples; +assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples; + +assert isComposable "pkgsExtraHardening"; +assert isComposable "pkgsLLVM"; +assert isComposable "pkgsArocc"; +# TODO: unexpected argument 'bintools' - uncomment once https://github.com/NixOS/nixpkgs/pull/331011 is done +# assert isComposable "pkgsZig"; +assert isComposable "pkgsMusl"; +assert isComposable "pkgsStatic"; +assert isComposable "pkgsi686Linux"; + +# Special cases regarding buildPlatform vs hostPlatform +assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl); +assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32); +assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); +assert discardEvaluationErrors ( + pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64 +); + +# pkgsCross should keep upper cross settings +assert discardEvaluationErrors ( + with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic +); +assert discardEvaluationErrors ( + with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM +); + +emptyFile diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index fd443ac773ffb..6787793dbdfc7 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -122,22 +122,18 @@ in let config = lib.showWarnings configEval.config.warnings configEval.config; # A few packages make a new package set to draw their dependencies from. - # (Currently to get a cross tool chain, or forced-i686 package.) Rather than - # give `all-packages.nix` all the arguments to this function, even ones that - # don't concern it, we give it this function to "re-call" nixpkgs, inheriting - # whatever arguments it doesn't explicitly provide. This way, - # `all-packages.nix` doesn't know more than it needs too. + # Rather than give `all-packages.nix` all the arguments to this function, + # even ones that don't concern it, we give it this function to "re-call" + # nixpkgs, inheriting whatever arguments it doesn't explicitly provide. This + # way, `all-packages.nix` doesn't know more than it needs to. # # It's OK that `args` doesn't include default arguments from this file: # they'll be deterministically inferred. In fact we must *not* include them, # because it's important that if some parameter which affects the default is # substituted with a different argument, the default is re-inferred. # - # To put this in concrete terms, this function is basically just used today to - # use package for a different platform for the current platform (namely cross - # compiling toolchains and 32-bit packages on x86_64). In both those cases we - # want the provided non-native `localSystem` argument to affect the stdenv - # chosen. + # To put this in concrete terms, we want the provided non-native `localSystem` + # and `crossSystem` arguments to affect the stdenv chosen. # # NB!!! This thing gets its `config` argument from `args`, i.e. it's actually # `config0`. It is important to keep it to `config0` format (as opposed to the @@ -146,7 +142,7 @@ in let # via `evalModules` is not idempotent. In other words, if you add `config` to # `newArgs`, expect strange very hard to debug errors! (Yes, I'm speaking from # experience here.) - nixpkgsFun = newArgs: import ./. (args // newArgs); + nixpkgsFun = f0: import ./. (args // f0 args); # Partially apply some arguments for building bootstraping stage pkgs # sets. Only apply arguments which no stdenv would want to override. diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 1cedd8dd18458..3fd0bf7e6b9ae 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -176,155 +176,104 @@ let ((config.packageOverrides or (super: {})) super); # Convenience attributes for instantitating package sets. Each of - # these will instantiate a new version of allPackages. Currently the - # following package sets are provided: - # - # - pkgsCross. where system is a member of lib.systems.examples - # - pkgsMusl - # - pkgsi686Linux - otherPackageSets = self: super: { + # these will instantiate a new version of allPackages. + otherPackageSets = let + mkPkgs = name: fn: nixpkgsFun (prevArgs: let nixpkgsArgs = fn prevArgs; in nixpkgsArgs // { + overlays = [ + (self': super': { + "${name}" = super'; + }) + ] ++ nixpkgsArgs.overlays or [] ++ prevArgs.overlays or []; + }); + # This is always cross. + mkCrossPkgs = name: crossAttrs: mkPkgs name (prevArgs: { + crossSystem = + (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // crossAttrs; + }); + # This is only cross when we are already cross, otherwise local. + # For the case of "native cross", i.e. pkgsCross.gnu64 on a x86_64-linux system, we need to adjust **both** + # localSystem **and** crossSystem, otherwise they're out of sync. + mkHybridPkgs = name: hybridAttrs: mkPkgs name (prevArgs: let + newSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // hybridAttrs; + in { crossSystem = newSystem; } + // lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform) { localSystem = newSystem; } + ); + in self: super: { # This maps each entry in lib.systems.examples to its own package # set. Each of these will contain all packages cross compiled for # that target system. For instance, pkgsCross.raspberryPi.hello, # will refer to the "hello" package built for the ARM6-based # Raspberry Pi. pkgsCross = lib.mapAttrs (n: crossSystem: - nixpkgsFun { inherit crossSystem; }) + nixpkgsFun (prevArgs: { crossSystem = (lib.systems.systemToAttrs (lib.defaultTo { } prevArgs.crossSystem or null)) // crossSystem; })) lib.systems.examples; - pkgsLLVM = nixpkgsFun { - overlays = [ - (self': super': { - pkgsLLVM = super'; - }) - ] ++ overlays; - # Bootstrap a cross stdenv using the LLVM toolchain. - # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { - useLLVM = true; - linker = "lld"; - }; + # Bootstrap a cross stdenv using the LLVM toolchain. + # This is currently not possible when compiling natively. + pkgsLLVM = mkCrossPkgs "pkgsLLVM" { + useLLVM = true; + linker = "lld"; }; - pkgsArocc = nixpkgsFun { - overlays = [ - (self': super': { - pkgsArocc = super'; - }) - ] ++ overlays; - # Bootstrap a cross stdenv using the Aro C compiler. - # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { - useArocc = true; - linker = "lld"; - }; + # Bootstrap a cross stdenv using the Aro C compiler. + # This is currently not possible when compiling natively. + pkgsArocc = mkCrossPkgs "pkgsArocc" { + useArocc = true; + linker = "lld"; }; - pkgsZig = nixpkgsFun { - overlays = [ - (self': super': { - pkgsZig = super'; - }) - ] ++ overlays; - # Bootstrap a cross stdenv using the Zig toolchain. - # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { - useZig = true; - linker = "lld"; - }; + # Bootstrap a cross stdenv using the Zig toolchain. + # This is currently not possible when compiling natively. + pkgsZig = mkCrossPkgs "pkgsZig" { + useZig = true; + linker = "lld"; }; # All packages built with the Musl libc. This will override the # default GNU libc on Linux systems. Non-Linux systems are not # supported. 32-bit is also not supported. - pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then nixpkgsFun { - overlays = [ (self': super': { - pkgsMusl = super'; - })] ++ overlays; - ${if stdenv.hostPlatform == stdenv.buildPlatform - then "localSystem" else "crossSystem"} = { - config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); - }; + pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkHybridPkgs "pkgsMusl" { + config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); } else throw "Musl libc only supports 64-bit Linux systems."; # All packages built for i686 Linux. # Used by wine, firefox with debugging version of Flash, ... - pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then nixpkgsFun { - overlays = [ (self': super': { - pkgsi686Linux = super'; - })] ++ overlays; - ${if stdenv.hostPlatform == stdenv.buildPlatform - then "localSystem" else "crossSystem"} = { - config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { - cpu = lib.systems.parse.cpuTypes.i686; - }); - }; + pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkHybridPkgs "pkgsi686Linux" { + config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { + cpu = lib.systems.parse.cpuTypes.i686; + }); } else throw "i686 Linux package set can only be used with the x86 family."; # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages - pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then nixpkgsFun { - overlays = [ (self': super': { - pkgsx86_64Darwin = super'; - })] ++ overlays; - localSystem = { - config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { - cpu = lib.systems.parse.cpuTypes.x86_64; - }); - }; + pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkHybridPkgs "pkgsx86_64Darwin" { + config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { + cpu = lib.systems.parse.cpuTypes.x86_64; + }); } else throw "x86_64 Darwin package set can only be used on Darwin systems."; # If already linux: the same package set unaltered - # Otherwise, return a natively built linux package set for the current cpu architecture string. + # Otherwise, return a linux package set for the current cpu architecture string. # (ABI and other details will be set to the default for the cpu/os pair) pkgsLinux = if stdenv.hostPlatform.isLinux then self - else nixpkgsFun { - localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; + else mkHybridPkgs "pkgsLinux" { + config = lib.systems.parse.tripleFromSystem (lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux").parsed; }; - # Extend the package set with zero or more overlays. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - appendOverlays = extraOverlays: - if extraOverlays == [] - then self - else nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; - - # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB - # of allocations. DO NOT USE THIS IN NIXPKGS. - # - # Extend the package set with a single overlay. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - # Prefer appendOverlays if used repeatedly. - extend = f: self.appendOverlays [f]; - # Fully static packages. # Currently uses Musl on Linux (couldn’t get static glibc to work). - pkgsStatic = nixpkgsFun ({ - overlays = [ (self': super': { - pkgsStatic = super'; - })] ++ overlays; - crossSystem = { - isStatic = true; - config = lib.systems.parse.tripleFromSystem ( - if stdenv.hostPlatform.isLinux - then makeMuslParsedPlatform stdenv.hostPlatform.parsed - else stdenv.hostPlatform.parsed - ); - gcc = lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } // - stdenv.hostPlatform.gcc or {}; - }; + pkgsStatic = mkCrossPkgs "pkgsStatic" ({ + isStatic = true; + } // lib.optionalAttrs stdenv.hostPlatform.isLinux { + config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); + } // lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { + gcc = { abi = "elfv2"; } // stdenv.hostPlatform.gcc or {}; }); - pkgsExtraHardening = nixpkgsFun { + pkgsExtraHardening = mkPkgs "pkgsExtraHardening" (_: { overlays = [ (self': super': { - pkgsExtraHardening = super'; stdenv = super'.withDefaultHardeningFlags ( super'.stdenv.cc.defaultHardeningFlags ++ [ "shadowstack" @@ -343,8 +292,25 @@ let pcre-cpp = super'.pcre-cpp.override { enableJit = false; }; pcre16 = super'.pcre16.override { enableJit = false; }; }) - ] ++ overlays; - }; + ]; + }); + + # Extend the package set with zero or more overlays. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + appendOverlays = extraOverlays: + if extraOverlays == [] + then self + else nixpkgsFun (prevArgs: { overlays = prevArgs.overlays ++ extraOverlays; }); + + # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB + # of allocations. DO NOT USE THIS IN NIXPKGS. + # + # Extend the package set with a single overlay. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + # Prefer appendOverlays if used repeatedly. + extend = f: self.appendOverlays [f]; }; # The complete chain of package set builders, applied from top to bottom.