fluent-bit: link against Nix dependencies, fix Darwin builds, and add NixOS module #365493
+263
−59
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,108 @@ | ||||||
| { | ||||||
| config, | ||||||
| lib, | ||||||
| pkgs, | ||||||
| utils, | ||||||
| ... | ||||||
| }: | ||||||
| let | ||||||
| cfg = config.services.fluent-bit; | ||||||
|
|
||||||
| yamlFormat = pkgs.formats.yaml { }; | ||||||
|
|
||||||
| configurationFile = | ||||||
| if (cfg.configurationFile == null) then | ||||||
| (yamlFormat.generate "fluent-bit.yaml" cfg.configuration) | ||||||
| else | ||||||
| cfg.configurationFile; | ||||||
| in | ||||||
| { | ||||||
| options.services.fluent-bit = { | ||||||
| enable = lib.mkEnableOption "Fluent Bit"; | ||||||
| package = lib.mkPackageOption pkgs "fluent-bit" { }; | ||||||
| configurationFile = lib.mkOption { | ||||||
| type = lib.types.nullOr lib.types.path; | ||||||
| default = null; | ||||||
| description = '' | ||||||
| Fluent Bit configuration. See | ||||||
| <https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml> | ||||||
| for supported values. | ||||||
|
|
||||||
| {option}`configurationFile` takes precedence over {option}`configuration`. | ||||||
|
|
||||||
| Note: Restricted evaluation blocks access to paths outside the Nix store. | ||||||
| This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done. | ||||||
| As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change. | ||||||
| `systemctl restart fluent-bit.service` must be used instead. | ||||||
| ''; | ||||||
| example = "/etc/fluent-bit/fluent-bit.yaml"; | ||||||
| }; | ||||||
| configuration = lib.mkOption { | ||||||
| type = yamlFormat.type; | ||||||
| default = { }; | ||||||
| description = '' | ||||||
| See {option}`configurationFile`. | ||||||
|
|
||||||
| {option}`configurationFile` takes precedence over {option}`configuration`. | ||||||
|
There was a problem hiding this comment.
Suggested change
|
||||||
| ''; | ||||||
| example = { | ||||||
| service = { | ||||||
| grace = 30; | ||||||
| }; | ||||||
| pipeline = { | ||||||
| inputs = [ | ||||||
| { | ||||||
| name = "systemd"; | ||||||
| systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; | ||||||
| } | ||||||
| ]; | ||||||
| outputs = [ | ||||||
| { | ||||||
| name = "file"; | ||||||
| path = "/var/log/fluent-bit"; | ||||||
| file = "fluent-bit.out"; | ||||||
| } | ||||||
| ]; | ||||||
| }; | ||||||
| }; | ||||||
| }; | ||||||
| # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section. | ||||||
| graceLimit = lib.mkOption { | ||||||
| type = lib.types.nullOr ( | ||||||
| lib.types.oneOf [ | ||||||
| lib.types.ints.positive | ||||||
| lib.types.str | ||||||
| ] | ||||||
| ); | ||||||
| default = null; | ||||||
| description = '' | ||||||
| The grace time limit. Sets the systemd unit's `TimeoutStopSec`. | ||||||
|
|
||||||
| The `service.grace` option in the Fluent Bit configuration should be ≤ this option. | ||||||
| ''; | ||||||
| example = 30; | ||||||
| }; | ||||||
fpletz marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
| }; | ||||||
|
|
||||||
| config = lib.mkIf cfg.enable { | ||||||
| # See https://github.com/fluent/fluent-bit/blob/v3.2.3/init/systemd.in. | ||||||
| systemd.services.fluent-bit = { | ||||||
| description = "Fluent Bit"; | ||||||
| after = [ "network.target" ]; | ||||||
| requires = [ "network.target" ]; | ||||||
| wantedBy = [ "multi-user.target" ]; | ||||||
| serviceConfig = { | ||||||
| DynamicUser = true; | ||||||
| # See https://nixos.org/manual/nixos/stable#sec-logging. | ||||||
| SupplementaryGroups = "systemd-journal"; | ||||||
| ExecStart = utils.escapeSystemdExecArgs [ | ||||||
| (lib.getExe cfg.package) | ||||||
| "--config" | ||||||
| configurationFile | ||||||
| ]; | ||||||
| Restart = "always"; | ||||||
| TimeoutStopSec = lib.mkIf (cfg.graceLimit != null) cfg.graceLimit; | ||||||
| }; | ||||||
| }; | ||||||
| }; | ||||||
| } | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| import ./make-test-python.nix ( | ||
| { lib, pkgs, ... }: | ||
| { | ||
| name = "fluent-bit"; | ||
|
|
||
| nodes.machine = | ||
| { config, pkgs, ... }: | ||
| { | ||
| services.fluent-bit = { | ||
| enable = true; | ||
| configuration = { | ||
| pipeline = { | ||
| inputs = [ | ||
| { | ||
| name = "systemd"; | ||
| systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; | ||
| } | ||
| ]; | ||
| outputs = [ | ||
| { | ||
| name = "file"; | ||
| path = "/var/log/fluent-bit"; | ||
| file = "fluent-bit.out"; | ||
| } | ||
| ]; | ||
| }; | ||
| }; | ||
| }; | ||
|
|
||
| systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit"; | ||
| }; | ||
|
|
||
| testScript = '' | ||
| start_all() | ||
|
|
||
| machine.wait_for_unit("fluent-bit.service") | ||
| machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out") | ||
| ''; | ||
| } | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,81 +1,147 @@ | ||
| { | ||
| lib, | ||
| arrow-glib, | ||
| bison, | ||
| c-ares, | ||
| cmake, | ||
| curl, | ||
| fetchFromGitHub, | ||
| flex, | ||
| jemalloc, | ||
| libbacktrace, | ||
| libbpf, | ||
| libnghttp2, | ||
| libyaml, | ||
| luajit, | ||
| nix-update-script, | ||
| nixosTests, | ||
| openssl, | ||
| pkg-config, | ||
| postgresql, | ||
| rdkafka, | ||
| stdenv, | ||
| systemd, | ||
| versionCheckHook, | ||
| zlib, | ||
| zstd, | ||
| }: | ||
|
|
||
| stdenv.mkDerivation rec { | ||
| pname = "fluent-bit"; | ||
| version = "3.2.4"; | ||
|
|
||
| src = fetchFromGitHub { | ||
| owner = "fluent"; | ||
| repo = "fluent-bit"; | ||
| tag = "v${version}"; | ||
| hash = "sha256-oTCGjDmGVovsfj+4fjIKy/xpiuYc0Q44LYwYPI4dSF8="; | ||
| }; | ||
|
|
||
| # `src/CMakeLists.txt` installs fluent-bit's systemd unit files at the path in the `SYSTEMD_UNITDIR` CMake variable. | ||
| # | ||
| # The initial value of `SYSTEMD_UNITDIR` is set in `cmake/FindJournald` which uses pkg-config to find the systemd | ||
| # unit directory. `src/CMakeLists.txt` only sets `SYSTEMD_UNITDIR` to `/lib/systemd/system` if it's unset. | ||
| # | ||
| # Unfortunately, this resolves to systemd's Nix store path which is immutable. Consequently, CMake fails when trying | ||
| # to install fluent-bit's systemd unit files to the systemd Nix store path. | ||
| # | ||
| # We fix this by replacing `${SYSTEMD_UNITDIR}` instances in `src/CMakeLists.txt`. | ||
| postPatch = '' | ||
| substituteInPlace src/CMakeLists.txt \ | ||
| --replace-fail \''${SYSTEMD_UNITDIR} $out/lib/systemd/system | ||
| ''; | ||
|
|
||
| # The source build documentation covers some dependencies and CMake options. | ||
| # | ||
| # - Linux: https://docs.fluentbit.io/manual/installation/sources/build-and-install | ||
| # - Darwin: https://docs.fluentbit.io/manual/installation/macos#compile-from-source | ||
| # | ||
| # Unfortunately, fluent-bit vends many dependencies (e.g. luajit) as source files and tries to compile them by | ||
| # default, with none of their dependencies and CMake options documented. | ||
| # | ||
| # Fortunately, there's the undocumented `FLB_PREFER_SYSTEM_LIBS` CMake option to link against system libraries for | ||
| # some dependencies. | ||
| # | ||
| # See https://github.com/fluent/fluent-bit/blob/v3.2.3/CMakeLists.txt#L211-L218. | ||
| # | ||
| # Like `FLB_PREFER_SYSTEM_LIBS`, several CMake options aren't documented. | ||
| # | ||
| # See https://github.com/fluent/fluent-bit/blob/v3.2.3/CMakeLists.txt#L111-L157. | ||
| # | ||
| # The CMake options may differ across target platforms. We'll stick to the minimum. | ||
| # | ||
| # See https://github.com/fluent/fluent-bit/tree/v3.2.3/packaging/distros. | ||
|
|
||
| strictDeps = true; | ||
|
|
||
| nativeBuildInputs = [ | ||
| bison | ||
| cmake | ||
| flex | ||
| pkg-config | ||
| ]; | ||
|
|
||
| buildInputs = | ||
| [ | ||
| arrow-glib | ||
| c-ares | ||
| # Needed by rdkafka. | ||
| curl | ||
| jemalloc | ||
| libbacktrace | ||
| libnghttp2 | ||
| libyaml | ||
| luajit | ||
| openssl | ||
| postgresql | ||
| rdkafka | ||
| # Needed by rdkafka. | ||
| zlib | ||
| # Needed by rdkafka. | ||
| zstd | ||
| ] | ||
| ++ lib.optionals stdenv.hostPlatform.isLinux [ | ||
| # libbpf doesn't build for Darwin yet. | ||
| libbpf | ||
| systemd | ||
| ]; | ||
|
|
||
| cmakeFlags = | ||
| [ | ||
| (lib.cmakeBool "FLB_RELEASE" true) | ||
| (lib.cmakeBool "FLB_PREFER_SYSTEM_LIBS" true) | ||
| ] | ||
| ++ lib.optionals stdenv.cc.isClang [ | ||
| # `FLB_SECURITY` causes bad linker options for Clang to be set. | ||
| (lib.cmakeBool "FLB_SECURITY" false) | ||
| ]; | ||
|
|
||
| outputs = [ | ||
| "out" | ||
| "dev" | ||
| ]; | ||
|
|
||
| doInstallCheck = true; | ||
|
|
||
| nativeInstallCheckInputs = [ versionCheckHook ]; | ||
|
|
||
| versionCheckProgram = "${builtins.placeholder "out"}/bin/fluent-bit"; | ||
|
|
||
| versionCheckProgramArg = "--version"; | ||
|
|
||
| passthru = { | ||
| tests = lib.optionalAttrs stdenv.isLinux { | ||
| inherit (nixosTests) fluent-bit; | ||
| }; | ||
|
|
||
| updateScript = nix-update-script { }; | ||
| }; | ||
|
|
||
| meta = { | ||
| description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows"; | ||
| homepage = "https://fluentbit.io"; | ||
| license = lib.licenses.asl20; | ||
| mainProgram = "fluent-bit"; | ||
| maintainers = with lib.maintainers; [ samrose ]; | ||
| }; | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let's follow RFC 42 here
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does the RFC mandate the option be named
settings? Outside of the option name, it seems like this is adhering to the RFC.Generally I've been trying to expose options with names based on the program's command line options (in this case
fluent-bituses--configso I've expanded that toconfiguration) or their relevant config file field name (e.g.graceLimitbased ongrace).