From 642dea6d3b7496d53a93f0e2b8917ed8e1adce53 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 3 Jan 2025 02:44:38 +0100 Subject: [PATCH] WIP: implement a secret vars store in nixpkgs This allows to create secrets (and public files) outside or inside the nix store in a more delclarative way. This is shipped with an example (working) implementation of an on-machine storage. The vars options can easily be used to implement custom backends and extend the behaviour to integrate with already existing solutions, like sops-nix or agenix. --- nixos/modules/module-list.nix | 2 + .../system/vars/on-machine-backend.nix | 84 ++++++++ nixos/modules/system/vars/options.nix | 189 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/vars.nix | 28 +++ 5 files changed, 304 insertions(+) create mode 100644 nixos/modules/system/vars/on-machine-backend.nix create mode 100644 nixos/modules/system/vars/options.nix create mode 100644 nixos/tests/vars.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 06979a4df508f9..f475cb92e2fc9e 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1709,6 +1709,8 @@ ./system/boot/timesyncd.nix ./system/boot/tmp.nix ./system/boot/uvesafb.nix + ./system/vars/options.nix + ./system/vars/on-machine-backend.nix ./system/etc/etc-activation.nix ./tasks/auto-upgrade.nix ./tasks/bcache.nix diff --git a/nixos/modules/system/vars/on-machine-backend.nix b/nixos/modules/system/vars/on-machine-backend.nix new file mode 100644 index 00000000000000..a7421584c2dc67 --- /dev/null +++ b/nixos/modules/system/vars/on-machine-backend.nix @@ -0,0 +1,84 @@ +# we use this vars backend as an example backend. +# this generates a script which creates the values at the expected path. +# this script has to be run manually (I guess after updating the system) to generate the required vars. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.vars.settings.on-machine; + sortedGenerators = + (lib.toposort (a: b: builtins.elem a.name b.dependencies) (lib.attrValues config.vars.generators)) + .result; + generate-vars = pkgs.writeShellApplication { + name = "generate-vars"; + text = '' + OUT_DIR=''${OUT_DIR:-${cfg.fileLocation}} + ${lib.concatMapStringsSep "\n" (gen: '' + all_files_missing=true + all_files_present=true + ${lib.concatMapStringsSep "\n" (file: '' + if test -e ${file.path} ; then + all_files_missing=false + else + all_files_present=false + fi + '') (lib.attrValues gen.files)} + if [ $all_files_missing = false ] && [ $all_files_present = false ] ; then + echo "Inconsistent state for generator ${gen.name}" + exit 1 + fi + if [ $all_files_present = true ] ; then + echo "All secrets for ${gen.name} are present" + elif [ $all_files_missing = true ] ; then + echo "Generating vars for ${gen.name}" + # TODO add inputs + # TODO add dependencies + out=$(mktemp -d) + trap 'rm -rf $out' EXIT + export out + mkdir -p "$out" + PATH=${lib.makeBinPath gen.runtimeInputs} + export PATH + ${gen.script} + ${lib.concatMapStringsSep "\n" (file: '' + if ! test -e "$out"/${file.name} ; then + echo 'generator ${gen.name} failed to generate ${file.name}' + exit 1 + fi + '') (lib.attrValues gen.files)} + ${lib.concatMapStringsSep "\n" (file: '' + OUT_FILE="$OUT_DIR"/${if file.secret then "secret" else "public"}/${file.generator}/${file.name} + mkdir -p "$(dirname "$OUT_FILE")" + mv "$out"/'${file.name}' "$OUT_FILE" + '') (lib.attrValues gen.files)} + rm -rf "$out" + fi + '') sortedGenerators} + ''; + }; +in +{ + options.vars.settings.on-machine = { + enable = lib.mkEnableOption "Enable on-machine vars backend"; + fileLocation = lib.mkOption { + type = lib.types.str; + default = "/etc/vars"; + }; + }; + config = lib.mkIf cfg.enable { + vars.settings.fileModule = file: { + path = + if file.config.secret then + "${cfg.fileLocation}/secret/${file.config.generator}/${file.config.name}" + else + "${cfg.fileLocation}/public/${file.config.generator}/${file.config.name}"; + }; + environment.systemPackages = [ + generate-vars + ]; + system.build.generate-vars = generate-vars; + }; +} diff --git a/nixos/modules/system/vars/options.nix b/nixos/modules/system/vars/options.nix new file mode 100644 index 00000000000000..d5142e3a4b4854 --- /dev/null +++ b/nixos/modules/system/vars/options.nix @@ -0,0 +1,189 @@ +{ lib, config, ... }: +{ + options.vars = { + settings = { + fileModule = lib.mkOption { + type = lib.types.deferredModule; + internal = true; + description = '' + A module to be imported in every vars.files. submodule. + Used by backends to define the `path` attribute. + + Takes the file as an arument and returns maybe an attrset with should at least contain the `path` attribute. + Can be used to set other file attributes as well, like `value`. + ''; + default = { }; + }; + }; + generators = lib.mkOption { + description = '' + A set of generators that can be used to generate files. + Generators are scripts that produce files based on the values of other generators and user input. + Each generator is expected to produce a set of files under a directory. + ''; + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule (generator: { + options = { + name = lib.mkOption { + type = lib.types.str; + description = '' + The name of the generator. + This name will be used to refer to the generator in other generators. + ''; + readOnly = true; + default = generator.config._module.args.name; + defaultText = "Name of the generator"; + }; + + dependencies = lib.mkOption { + description = '' + A list of other generators that this generator depends on. + The output values of these generators will be available to the generator script as files. + For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1. + ''; + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + files = lib.mkOption { + description = '' + A set of files to generate. + The generator 'script' is expected to produce exactly these files under $out. + ''; + defaultText = "attrs of files"; + type = lib.types.attrsOf ( + lib.types.submodule (file: { + imports = [ + config.vars.settings.fileModule + ]; + options = { + name = lib.mkOption { + type = lib.types.str; + description = '' + name of the public fact + ''; + readOnly = true; + default = file.config._module.args.name; + defaultText = "Name of the file"; + }; + generator = lib.mkOption { + description = '' + The generator that produces the file. + This is the name of another generator. + ''; + type = lib.types.str; + default = generator.config.name; + }; + deploy = lib.mkOption { + description = '' + Whether the file should be deployed to the target machine. + + Enable this if the generated file is only used as an input to other generators. + ''; + type = lib.types.bool; + default = true; + }; + secret = lib.mkOption { + description = '' + Whether the file should be treated as a secret. + ''; + type = lib.types.bool; + default = true; + }; + path = lib.mkOption { + description = '' + The path to the file containing the content of the generated value. + This will be set automatically + ''; + type = lib.types.str; + }; + owner = lib.mkOption { + description = "The user name or id that will own the file."; + default = "root"; + }; + group = lib.mkOption { + description = "The group name or id that will own the file."; + default = "root"; + }; + mode = lib.mkOption { + type = lib.types.strMatching "^[0-7]{3}$"; + description = "The unix file mode of the file. Must be a 3-digit octal number."; + default = "400"; + }; + }; + }) + ); + }; + prompts = lib.mkOption { + description = '' + A set of prompts to ask the user for values. + Prompts are available to the generator script as files. + For example, a prompt named 'prompt1' will be available via $prompts/prompt1 + ''; + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule (prompt: { + options = { + name = lib.mkOption { + description = '' + The name of the prompt. + This name will be used to refer to the prompt in the generator script. + ''; + type = lib.types.str; + default = prompt.config._module.args.name; + defaultText = "Name of the prompt"; + }; + description = lib.mkOption { + description = '' + The description of the prompted value + ''; + type = lib.types.str; + example = "SSH private key"; + default = prompt.config._module.args.name; + defaultText = "Name of the prompt"; + }; + type = lib.mkOption { + description = '' + The input type of the prompt. + The following types are available: + - hidden: A hidden text (e.g. password) + - line: A single line of text + - multiline: A multiline text + ''; + type = lib.types.enum [ + "hidden" + "line" + "multiline" + ]; + default = "line"; + }; + }; + }) + ); + }; + runtimeInputs = lib.mkOption { + description = '' + A list of packages that the generator script requires. + These packages will be available in the PATH when the script is run. + ''; + type = lib.types.listOf lib.types.package; + default = [ ]; + }; + script = lib.mkOption { + description = '' + The script to run to generate the files. + The script will be run with the following environment variables: + - $in: The directory containing the output values of all declared dependencies + - $out: The output directory to put the generated files + - $prompts: The directory containing the prompted values as files + The script should produce the files specified in the 'files' attribute under $out. + ''; + type = lib.types.either lib.types.str lib.types.path; + default = ""; + }; + }; + }) + ); + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index cf06168863017a..60a0afb73ca95f 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1132,6 +1132,7 @@ in { user-home-mode = handleTest ./user-home-mode.nix {}; ustreamer = handleTest ./ustreamer.nix {}; uwsgi = handleTest ./uwsgi.nix {}; + vars = handleTest ./vars.nix {}; v2ray = handleTest ./v2ray.nix {}; varnish60 = handleTest ./varnish.nix { package = pkgs.varnish60; }; varnish75 = handleTest ./varnish.nix { package = pkgs.varnish75; }; diff --git a/nixos/tests/vars.nix b/nixos/tests/vars.nix new file mode 100644 index 00000000000000..9d848841732158 --- /dev/null +++ b/nixos/tests/vars.nix @@ -0,0 +1,28 @@ +import ./make-test-python.nix ( + { lib, pkgs, ... }: + + { + name = "vars"; + meta.maintainers = with lib.maintainers; [ lassulus ]; + + nodes.machine = + { ... }: + { + vars.settings.on-machine.enable = true; + services.syncthing.enable = true; + }; + + testScript = + { nodes, ... }: + '' + import subprocess + subprocess.run( + "${nodes.machine.config.system.build.generate-vars}/bin/generate-vars", + shell=True, + env={ + "OUT_DIR": "./vars", + }, + ) + ''; + } +)