diff --git a/CHANGELOG.md b/CHANGELOG.md index 0013a7b..96ed7c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ This project adheres to [Semantic Versioning](http://semver.org/). ... +## [3.2.0] - 2019-03-22 + +### Added +- The daemon will accept directory paths as command-line arguments and expand + them to load all contained `*.yml` and `*.yaml` files. + [#11](//github.com/greglook/solanum/issues/11) +- New `shell` source allows for arbitrary command execution to produce metrics + in a flexible manner. + [#13](//github.com/greglook/solanum/issues/13) + ## [3.1.2] - 2018-12-11 ### Fixed @@ -45,7 +55,8 @@ Clojure rewrite. Final cut of Ruby version. -[Unreleased]: https://github.com/greglook/solanum/compare/3.1.2...HEAD +[Unreleased]: https://github.com/greglook/solanum/compare/3.2.0...HEAD +[3.2.0]: https://github.com/greglook/solanum/compare/3.1.2...3.2.0 [3.1.2]: https://github.com/greglook/solanum/compare/3.1.1...3.1.2 [3.1.1]: https://github.com/greglook/solanum/compare/3.1.0...3.1.1 [3.1.0]: https://github.com/greglook/solanum/compare/3.0.0...3.1.0 diff --git a/Makefile b/Makefile index 0c24864..4928a72 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,6 @@ ifndef GRAAL_PATH $(error GRAAL_PATH is not set) endif -# TODO: fetch graal? setup: lein deps @@ -26,18 +25,22 @@ lint: test: lein test -$(uberjar_path): src/**/* resources/**/* svm/java/**/* +$(uberjar_path): src/**/* resources/* svm/java/**/* lein with-profile +svm uberjar uberjar: $(uberjar_path) -# TODO: --static ? +# TODO: further options +# --static # --enable-url-protocols=http,https solanum: reflection-config := svm/reflection-config.json solanum: $(uberjar_path) $(reflection-config) $(GRAAL_PATH)/bin/native-image \ + --allow-incomplete-classpath \ --report-unsupported-elements-at-runtime \ + --delay-class-initialization-to-runtime=io.netty.handler.ssl.ConscryptAlpnSslEngine \ --delay-class-initialization-to-runtime=io.netty.handler.ssl.ReferenceCountedOpenSslEngine \ + --delay-class-initialization-to-runtime=io.netty.util.internal.logging.Log4JLogger \ -H:ReflectionConfigurationFiles=$(reflection-config) \ -J-Xms3G -J-Xmx3G \ --no-server \ diff --git a/doc/sources.md b/doc/sources.md index 3e28c07..75c70dd 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -235,6 +235,33 @@ allocated, some of which may be unused or paged out to disk. to ensure that _no more than_ a certain number of processes are running. +## shell + +This source provides an escape hatch if some metrics collection is not supported +directly by the daemon. It periodically executes a shell command and interprets +the response as metrics events. + +- `command` (required) + + A command to execute with the shell in order to collect the metrics data. + +- `shell` (default: `$SHELL`) + + The shell to use to execute the command. + +The command is expected to return some output which conforms to the following +line protocol: + +``` +\t[\t=][\t...] +``` + +The first entry on the line should be the measurement's service name, followed +by a tab `\t` character, then the numeric metric value. The value may be an +integer or a floating-point number. Following that may be zero or more +attribute-value pairs, similarly separated by tabs. + + ## tcp This source tests that a local port is open and accepting TCP connections. It diff --git a/project.clj b/project.clj index 3ba7c7c..49be0db 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject mvxcvi/solanum "3.1.2" +(defproject mvxcvi/solanum "3.2.0" :description "Local host monitoring daemon." :url "https://github.com/greglook/solanum" :license {:name "Public Domain" @@ -31,7 +31,8 @@ :profiles {:repl - {:source-paths ["dev"] + {:pedantic? false + :source-paths ["dev"] :dependencies [[clj-stacktrace "0.2.8"] [org.clojure/tools.namespace "0.2.11"]] @@ -40,7 +41,7 @@ :svm {:java-source-paths ["svm/java"] :dependencies - [[com.oracle.substratevm/svm "1.0.0-rc8" :scope "provided"]]} + [[com.oracle.substratevm/svm "1.0.0-rc14" :scope "provided"]]} :uberjar {:target-path "target/uberjar" diff --git a/src/solanum/config.clj b/src/solanum/config.clj index 3c7aa05..5ae9663 100644 --- a/src/solanum/config.clj +++ b/src/solanum/config.clj @@ -18,6 +18,7 @@ [solanum.source.memory] [solanum.source.network] [solanum.source.process] + [solanum.source.shell] [solanum.source.tcp] [solanum.source.test] [solanum.source.uptime] @@ -63,16 +64,14 @@ (defn- read-file "Load some configuration from a file." - [path] - (let [file (io/file path)] - (if (.exists file) - (try - (let [parser (Yaml.) - data (.load parser (slurp file))] - (walk/prewalk yaml->clj data)) - (catch Exception ex - (log/error ex "Failed to load configuration from" path))) - (log/warn "Can't load configuration from nonexistent file" path)))) + [file] + (log/debug "Reading configuration file" (str file)) + (try + (let [parser (Yaml.) + data (.load parser (slurp file))] + (walk/prewalk yaml->clj data)) + (catch Exception ex + (log/error ex "Failed to load configuration from" (str file))))) (defn- merge-config @@ -135,10 +134,38 @@ :outputs outputs))) +(defn- yaml-file? + "True if the file or path appears to be a YAML file." + [file] + (or (str/ends-with? (str file) ".yml") + (str/ends-with? (str file) ".yaml"))) + + +(defn- select-configs + "Select a sequence of configuration files located using the given arguments. + If the argument is a regular file, it is returned as a single-element vector. + If it is a directory, all `*.yml` and `*.yaml` files in the directory are + selected. Otherwise, the result is nil." + [path] + (let [file (io/file path)] + (cond + (not (.exists file)) + (log/warn "Can't load configuration from nonexistent file" path) + + (.isDirectory file) + (sort (filter yaml-file? (.listFiles file))) + + :else + [file]))) + + (defn load-files "Load multiple files, merge them together, and initialize the plugins." [config-paths] ; TODO: warn if defaults include :host - (->> (map read-file config-paths) - (reduce merge-config) - (initialize-plugins))) + (->> + config-paths + (mapcat select-configs) + (map read-file) + (reduce merge-config) + (initialize-plugins))) diff --git a/src/solanum/main.clj b/src/solanum/main.clj index 6ed96be..7595d7e 100644 --- a/src/solanum/main.clj +++ b/src/solanum/main.clj @@ -137,10 +137,10 @@ (defn -main "Main entry point." [& args] - (let [parse (cli/parse-opts args cli-options) - config-paths (parse :arguments) - options (parse :options)] - (when-let [errors (parse :errors)] + (let [parsed (cli/parse-opts args cli-options) + config-paths (parsed :arguments) + options (parsed :options)] + (when-let [errors (parsed :errors)] (binding [*out* *err*] (run! println errors) (System/exit 1))) @@ -151,7 +151,7 @@ (when (or (:help options) (empty? config-paths)) (println "Usage: solanum [options] [config2.yml ...]") (newline) - (println (parse :summary)) + (println (parsed :summary)) (flush) (System/exit (if (:help options) 0 1))) (let [config (cfg/load-files config-paths)] diff --git a/src/solanum/source/shell.clj b/src/solanum/source/shell.clj new file mode 100644 index 0000000..bd4d538 --- /dev/null +++ b/src/solanum/source/shell.clj @@ -0,0 +1,81 @@ +(ns solanum.source.shell + "Metrics source that executes a shell command." + (:require + [clojure.java.shell :as shell] + [clojure.string :as str] + [clojure.tools.logging :as log] + [solanum.source.core :as source])) + + +;; ## Measurements + +(defn- parse-metric + "Parse a metric string as an integer or a float. Returns the number, or throws + an exception if the metric is not a valid numeric literal." + [metric] + (if (str/includes? metric ".") + (Double/parseDouble metric) + (Long/parseLong metric))) + + +(defn- parse-attribute + "Parse an attribute pair. Returns a tuple with the attribute and value, or + nil if parsing failed." + [attr] + (if (str/includes? attr "=") + (let [[k v] (str/split attr #"=" 2)] + [(keyword k) v]) + (log/warn "Dropping invalid attribute pair:" (pr-str attr)))) + + +(defn- parse-line + "Parse a single line according to the line protocol, returning an event + constructed from the parsed data. Returns nil if the line is blank or + invalid." + [line] + (when-not (str/blank? line) + (let [[service metric & attrs] (str/split line #"\t+")] + (if-not (or (str/blank? service) (str/blank? metric)) + (try + (let [metric (parse-metric metric) + attrs (into {} (keep parse-attribute) attrs)] + (assoc attrs + :service service + :metric metric)) + (catch Exception ex + (log/warn "Failed to parse metrics line:" + (pr-str line) + (.getName (class ex)) + (.getMessage ex)))) + (log/warn "Dropped invalid metrics line - missing service or metric:" + (pr-str line)))))) + + + +;; ## TCP Source + +(defrecord ShellSource + [shell command] + + source/Source + + (collect-events + [this] + (let [result (shell/sh shell "-s" :in command)] + (if (zero? (:exit result)) + (->> (:out result) + (str/split-lines) + (into [] (keep parse-line))) + (log/warn "Failed to execute shell command:" + (pr-str command) + (pr-str (:err result))))))) + + +(defmethod source/initialize :shell + [config] + (when-not (:command config) + (throw (IllegalArgumentException. + "Cannot initialize shell source without a command"))) + (map->ShellSource + {:shell (:shell config (System/getenv "SHELL")) + :command (:command config)})) diff --git a/test/solanum/config_test.clj b/test/solanum/config_test.clj index 6492433..fded4e0 100644 --- a/test/solanum/config_test.clj +++ b/test/solanum/config_test.clj @@ -53,7 +53,6 @@ outputs: (deftest config-errors (write-configs!) (testing "file reading" - (is (nil? (#'config/read-file "target/test/not-a-file.yml"))) (with-redefs [config/yaml->clj boom!] (is (nil? (#'config/read-file path-a))))) (testing "source configuration"