WIP: implement a secret vars store in nixpkgs

This allows to create secrets (and public files) outside or inside the
nix store in a more delclarative way. This is shipped with an example
(working) implementation of an on-machine storage. The vars options can
easily be used to implement custom backends and extend the behaviour to
integrate with already existing solutions, like sops-nix or agenix.

nixos/modules/module-list.nix

@@ -1709,6 +1709,8 @@
   ./system/boot/timesyncd.nix
   ./system/boot/tmp.nix
   ./system/boot/uvesafb.nix
+  ./system/vars/options.nix
+  ./system/vars/on-machine-backend.nix
   ./system/etc/etc-activation.nix
   ./tasks/auto-upgrade.nix
   ./tasks/bcache.nix

nixos/modules/system/vars/on-machine-backend.nix

@@ -0,0 +1,84 @@
+# we use this vars backend as an example backend.
+# this generates a script which creates the values at the expected path.
+# this script has to be run manually (I guess after updating the system) to generate the required vars.
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.vars.settings.on-machine;
+  sortedGenerators =
+    (lib.toposort (a: b: builtins.elem a.name b.dependencies) (lib.attrValues config.vars.generators))
+    .result;
+  generate-vars = pkgs.writeShellApplication {
+    name = "generate-vars";
+    text = ''
+      OUT_DIR=''${OUT_DIR:-${cfg.fileLocation}}
+      ${lib.concatMapStringsSep "\n" (gen: ''
+        all_files_missing=true
+        all_files_present=true
+        ${lib.concatMapStringsSep "\n" (file: ''
+          if test -e ${file.path} ; then
+            all_files_missing=false
+          else
+            all_files_present=false
+          fi
+        '') (lib.attrValues gen.files)}
+        if [ $all_files_missing = false ] && [ $all_files_present = false ] ; then
+          echo "Inconsistent state for generator ${gen.name}"
+          exit 1
+        fi
+        if [ $all_files_present = true ] ; then
+          echo "All secrets for ${gen.name} are present"
+        elif [ $all_files_missing = true ] ; then
+          echo "Generating vars for ${gen.name}"
+          # TODO add inputs
+          # TODO add dependencies
+          out=$(mktemp -d)
+          trap 'rm -rf $out' EXIT
+          export out
+          mkdir -p "$out"
+          PATH=${lib.makeBinPath gen.runtimeInputs}
+          export PATH
+          ${gen.script}
+          ${lib.concatMapStringsSep "\n" (file: ''
+            if ! test -e "$out"/${file.name} ; then
+              echo 'generator ${gen.name} failed to generate ${file.name}'
+              exit 1
+            fi
+          '') (lib.attrValues gen.files)}
+          ${lib.concatMapStringsSep "\n" (file: ''
+            OUT_FILE="$OUT_DIR"/${if file.secret then "secret" else "public"}/${file.generator}/${file.name}
+            mkdir -p "$(dirname "$OUT_FILE")"
+            mv "$out"/'${file.name}' "$OUT_FILE"
+          '') (lib.attrValues gen.files)}
+          rm -rf "$out"
+        fi
+      '') sortedGenerators}
+    '';
+  };
+in
+{
+  options.vars.settings.on-machine = {
+    enable = lib.mkEnableOption "Enable on-machine vars backend";
+    fileLocation = lib.mkOption {
+      type = lib.types.str;
+      default = "/etc/vars";
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    vars.settings.fileModule = file: {
+      path =
+        if file.config.secret then
+          "${cfg.fileLocation}/secret/${file.config.generator}/${file.config.name}"
+        else
+          "${cfg.fileLocation}/public/${file.config.generator}/${file.config.name}";
+    };
+    environment.systemPackages = [
+      generate-vars
+    ];
+    system.build.generate-vars = generate-vars;
+  };
+}

nixos/modules/system/vars/options.nix

@@ -0,0 +1,189 @@
+{ lib, config, ... }:
+{
+  options.vars = {
+    settings = {
+      fileModule = lib.mkOption {
+        type = lib.types.deferredModule;
+        internal = true;
+        description = ''
+          A module to be imported in every vars.files.<name> submodule.
+          Used by backends to define the `path` attribute.
+
+          Takes the file as an arument and returns maybe an attrset with should at least contain the `path` attribute.
+          Can be used to set other file attributes as well, like `value`.
+        '';
+        default = { };
+      };
+    };
+    generators = lib.mkOption {
+      description = ''
+        A set of generators that can be used to generate files.
+        Generators are scripts that produce files based on the values of other generators and user input.
+        Each generator is expected to produce a set of files under a directory.
+      '';
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (generator: {
+          options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The name of the generator.
+                This name will be used to refer to the generator in other generators.
+              '';
+              readOnly = true;
+              default = generator.config._module.args.name;
+              defaultText = "Name of the generator";
+            };
+
+            dependencies = lib.mkOption {
+              description = ''
+                A list of other generators that this generator depends on.
+                The output values of these generators will be available to the generator script as files.
+                For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1.
+              '';
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+            };
+            files = lib.mkOption {
+              description = ''
+                A set of files to generate.
+                The generator 'script' is expected to produce exactly these files under $out.
+              '';
+              defaultText = "attrs of files";
+              type = lib.types.attrsOf (
+                lib.types.submodule (file: {
+                  imports = [
+                    config.vars.settings.fileModule
+                  ];
+                  options = {
+                    name = lib.mkOption {
+                      type = lib.types.str;
+                      description = ''
+                        name of the public fact
+                      '';
+                      readOnly = true;
+                      default = file.config._module.args.name;
+                      defaultText = "Name of the file";
+                    };
+                    generator = lib.mkOption {
+                      description = ''
+                        The generator that produces the file.
+                        This is the name of another generator.
+                      '';
+                      type = lib.types.str;
+                      default = generator.config.name;
+                    };
+                    deploy = lib.mkOption {
+                      description = ''
+                        Whether the file should be deployed to the target machine.
+
+                        Enable this if the generated file is only used as an input to other generators.
+                      '';
+                      type = lib.types.bool;
+                      default = true;
+                    };
+                    secret = lib.mkOption {
+                      description = ''
+                        Whether the file should be treated as a secret.
+                      '';
+                      type = lib.types.bool;
+                      default = true;
+                    };
+                    path = lib.mkOption {
+                      description = ''
+                        The path to the file containing the content of the generated value.
+                        This will be set automatically
+                      '';
+                      type = lib.types.str;
+                    };
+                    owner = lib.mkOption {
+                      description = "The user name or id that will own the file.";
+                      default = "root";
+                    };
+                    group = lib.mkOption {
+                      description = "The group name or id that will own the file.";
+                      default = "root";
+                    };
+                    mode = lib.mkOption {
+                      type = lib.types.strMatching "^[0-7]{3}$";
+                      description = "The unix file mode of the file. Must be a 3-digit octal number.";
+                      default = "400";
+                    };
+                  };
+                })
+              );
+            };
+            prompts = lib.mkOption {
+              description = ''
+                A set of prompts to ask the user for values.
+                Prompts are available to the generator script as files.
+                For example, a prompt named 'prompt1' will be available via $prompts/prompt1
+              '';
+              default = { };
+              type = lib.types.attrsOf (
+                lib.types.submodule (prompt: {
+                  options = {
+                    name = lib.mkOption {
+                      description = ''
+                        The name of the prompt.
+                        This name will be used to refer to the prompt in the generator script.
+                      '';
+                      type = lib.types.str;
+                      default = prompt.config._module.args.name;
+                      defaultText = "Name of the prompt";
+                    };
+                    description = lib.mkOption {
+                      description = ''
+                        The description of the prompted value
+                      '';
+                      type = lib.types.str;
+                      example = "SSH private key";
+                      default = prompt.config._module.args.name;
+                      defaultText = "Name of the prompt";
+                    };
+                    type = lib.mkOption {
+                      description = ''
+                        The input type of the prompt.
+                        The following types are available:
+                          - hidden: A hidden text (e.g. password)
+                          - line: A single line of text
+                          - multiline: A multiline text
+                      '';
+                      type = lib.types.enum [
+                        "hidden"
+                        "line"
+                        "multiline"
+                      ];
+                      default = "line";
+                    };
+                  };
+                })
+              );
+            };
+            runtimeInputs = lib.mkOption {
+              description = ''
+                A list of packages that the generator script requires.
+                These packages will be available in the PATH when the script is run.
+              '';
+              type = lib.types.listOf lib.types.package;
+              default = [ ];
+            };
+            script = lib.mkOption {
+              description = ''
+                The script to run to generate the files.
+                The script will be run with the following environment variables:
+                  - $in: The directory containing the output values of all declared dependencies
+                  - $out: The output directory to put the generated files
+                  - $prompts: The directory containing the prompted values as files
+                The script should produce the files specified in the 'files' attribute under $out.
+              '';
+              type = lib.types.either lib.types.str lib.types.path;
+              default = "";
+            };
+          };
+        })
+      );
+    };
+  };
+}

nixos/tests/all-tests.nix

@@ -1132,6 +1132,7 @@ in {
   user-home-mode = handleTest ./user-home-mode.nix {};
   ustreamer = handleTest ./ustreamer.nix {};
   uwsgi = handleTest ./uwsgi.nix {};
+  vars = handleTest ./vars.nix {};
   v2ray = handleTest ./v2ray.nix {};
   varnish60 = handleTest ./varnish.nix { package = pkgs.varnish60; };
   varnish75 = handleTest ./varnish.nix { package = pkgs.varnish75; };

nixos/tests/vars.nix

@@ -0,0 +1,28 @@
+import ./make-test-python.nix (
+  { lib, pkgs, ... }:
+
+  {
+    name = "vars";
+    meta.maintainers = with lib.maintainers; [ lassulus ];
+
+    nodes.machine =
+      { ... }:
+      {
+        vars.settings.on-machine.enable = true;
+        services.syncthing.enable = true;
+      };
+
+    testScript =
+      { nodes, ... }:
+      ''
+        import subprocess
+        subprocess.run(
+          "${nodes.machine.config.system.build.generate-vars}/bin/generate-vars",
+          shell=True,
+          env={
+            "OUT_DIR": "./vars",
+          },
+        )
+      '';
+  }
+)