Merge branch 'release-3.2.0'

CHANGELOG.md

@@ -9,6 +9,16 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ...
 
+## [3.2.0] - 2019-03-22
+
+### Added
+- The daemon will accept directory paths as command-line arguments and expand
+  them to load all contained `*.yml` and `*.yaml` files.
+  [#11](//github.com/greglook/solanum/issues/11)
+- New `shell` source allows for arbitrary command execution to produce metrics
+  in a flexible manner.
+  [#13](//github.com/greglook/solanum/issues/13)
+
 ## [3.1.2] - 2018-12-11
 
 ### Fixed
@@ -45,7 +55,8 @@ Clojure rewrite.
 
 Final cut of Ruby version.
 
-[Unreleased]: https://github.com/greglook/solanum/compare/3.1.2...HEAD
+[Unreleased]: https://github.com/greglook/solanum/compare/3.2.0...HEAD
+[3.2.0]: https://github.com/greglook/solanum/compare/3.1.2...3.2.0
 [3.1.2]: https://github.com/greglook/solanum/compare/3.1.1...3.1.2
 [3.1.1]: https://github.com/greglook/solanum/compare/3.1.0...3.1.1
 [3.1.0]: https://github.com/greglook/solanum/compare/3.0.0...3.1.0

Makefile

@@ -13,7 +13,6 @@ ifndef GRAAL_PATH
 $(error GRAAL_PATH is not set)
 endif
 
-# TODO: fetch graal?
 setup:
 	lein deps
 
@@ -26,18 +25,22 @@ lint:
 test:
 	lein test
 
-$(uberjar_path): src/**/* resources/**/* svm/java/**/*
+$(uberjar_path): src/**/* resources/* svm/java/**/*
 	lein with-profile +svm uberjar
 
 uberjar: $(uberjar_path)
 
-# TODO: --static ?
+# TODO: further options
+# --static
 # --enable-url-protocols=http,https
 solanum: reflection-config := svm/reflection-config.json
 solanum: $(uberjar_path) $(reflection-config)
 	$(GRAAL_PATH)/bin/native-image \
+	    --allow-incomplete-classpath \
 	    --report-unsupported-elements-at-runtime \
+	    --delay-class-initialization-to-runtime=io.netty.handler.ssl.ConscryptAlpnSslEngine \
 	    --delay-class-initialization-to-runtime=io.netty.handler.ssl.ReferenceCountedOpenSslEngine \
+	    --delay-class-initialization-to-runtime=io.netty.util.internal.logging.Log4JLogger \
 	    -H:ReflectionConfigurationFiles=$(reflection-config) \
 	    -J-Xms3G -J-Xmx3G \
 	    --no-server \

doc/sources.md

@@ -235,6 +235,33 @@ allocated, some of which may be unused or paged out to disk.
   to ensure that _no more than_ a certain number of processes are running.
 
 
+## shell
+
+This source provides an escape hatch if some metrics collection is not supported
+directly by the daemon. It periodically executes a shell command and interprets
+the response as metrics events.
+
+- `command` (required)
+
+  A command to execute with the shell in order to collect the metrics data.
+
+- `shell` (default: `$SHELL`)
+
+  The shell to use to execute the command.
+
+The command is expected to return some output which conforms to the following
+line protocol:
+
+```
+<service>\t<metric>[\t<attribute>=<value>][\t...]
+```
+
+The first entry on the line should be the measurement's service name, followed
+by a tab `\t` character, then the numeric metric value. The value may be an
+integer or a floating-point number. Following that may be zero or more
+attribute-value pairs, similarly separated by tabs.
+
+
 ## tcp
 
 This source tests that a local port is open and accepting TCP connections. It

project.clj

@@ -1,4 +1,4 @@
-(defproject mvxcvi/solanum "3.1.2"
+(defproject mvxcvi/solanum "3.2.0"
   :description "Local host monitoring daemon."
   :url "https://github.com/greglook/solanum"
   :license {:name "Public Domain"
@@ -31,7 +31,8 @@
 
   :profiles
   {:repl
-   {:source-paths ["dev"]
+   {:pedantic? false
+    :source-paths ["dev"]
     :dependencies
     [[clj-stacktrace "0.2.8"]
      [org.clojure/tools.namespace "0.2.11"]]
@@ -40,7 +41,7 @@
    :svm
    {:java-source-paths ["svm/java"]
     :dependencies
-    [[com.oracle.substratevm/svm "1.0.0-rc8" :scope "provided"]]}
+    [[com.oracle.substratevm/svm "1.0.0-rc14" :scope "provided"]]}
 
    :uberjar
    {:target-path "target/uberjar"

src/solanum/config.clj

@@ -18,6 +18,7 @@
     [solanum.source.memory]
     [solanum.source.network]
     [solanum.source.process]
+    [solanum.source.shell]
     [solanum.source.tcp]
     [solanum.source.test]
     [solanum.source.uptime]
@@ -63,16 +64,14 @@
 
 (defn- read-file
   "Load some configuration from a file."
-  [path]
-  (let [file (io/file path)]
-    (if (.exists file)
-      (try
-        (let [parser (Yaml.)
-              data (.load parser (slurp file))]
-          (walk/prewalk yaml->clj data))
-        (catch Exception ex
-          (log/error ex "Failed to load configuration from" path)))
-      (log/warn "Can't load configuration from nonexistent file" path))))
+  [file]
+  (log/debug "Reading configuration file" (str file))
+  (try
+    (let [parser (Yaml.)
+          data (.load parser (slurp file))]
+      (walk/prewalk yaml->clj data))
+    (catch Exception ex
+      (log/error ex "Failed to load configuration from" (str file)))))
 
 
 (defn- merge-config
@@ -135,10 +134,38 @@
            :outputs outputs)))
 
 
+(defn- yaml-file?
+  "True if the file or path appears to be a YAML file."
+  [file]
+  (or (str/ends-with? (str file) ".yml")
+      (str/ends-with? (str file) ".yaml")))
+
+
+(defn- select-configs
+  "Select a sequence of configuration files located using the given arguments.
+  If the argument is a regular file, it is returned as a single-element vector.
+  If it is a directory, all `*.yml` and `*.yaml` files in the directory are
+  selected. Otherwise, the result is nil."
+  [path]
+  (let [file (io/file path)]
+    (cond
+      (not (.exists file))
+      (log/warn "Can't load configuration from nonexistent file" path)
+
+      (.isDirectory file)
+      (sort (filter yaml-file? (.listFiles file)))
+
+      :else
+      [file])))
+
+
 (defn load-files
   "Load multiple files, merge them together, and initialize the plugins."
   [config-paths]
   ; TODO: warn if defaults include :host
-  (->> (map read-file config-paths)
-       (reduce merge-config)
-       (initialize-plugins)))
+  (->>
+    config-paths
+    (mapcat select-configs)
+    (map read-file)
+    (reduce merge-config)
+    (initialize-plugins)))

src/solanum/main.clj

@@ -137,10 +137,10 @@
 (defn -main
   "Main entry point."
   [& args]
-  (let [parse (cli/parse-opts args cli-options)
-        config-paths (parse :arguments)
-        options (parse :options)]
-    (when-let [errors (parse :errors)]
+  (let [parsed (cli/parse-opts args cli-options)
+        config-paths (parsed :arguments)
+        options (parsed :options)]
+    (when-let [errors (parsed :errors)]
       (binding [*out* *err*]
         (run! println errors)
         (System/exit 1)))
@@ -151,7 +151,7 @@
     (when (or (:help options) (empty? config-paths))
       (println "Usage: solanum [options] <config.yml> [config2.yml ...]")
       (newline)
-      (println (parse :summary))
+      (println (parsed :summary))
       (flush)
       (System/exit (if (:help options) 0 1)))
     (let [config (cfg/load-files config-paths)]

src/solanum/source/shell.clj

@@ -0,0 +1,81 @@
+(ns solanum.source.shell
+  "Metrics source that executes a shell command."
+  (:require
+    [clojure.java.shell :as shell]
+    [clojure.string :as str]
+    [clojure.tools.logging :as log]
+    [solanum.source.core :as source]))
+
+
+;; ## Measurements
+
+(defn- parse-metric
+  "Parse a metric string as an integer or a float. Returns the number, or throws
+  an exception if the metric is not a valid numeric literal."
+  [metric]
+  (if (str/includes? metric ".")
+    (Double/parseDouble metric)
+    (Long/parseLong metric)))
+
+
+(defn- parse-attribute
+  "Parse an attribute pair. Returns a tuple with the attribute and value, or
+  nil if parsing failed."
+  [attr]
+  (if (str/includes? attr "=")
+    (let [[k v] (str/split attr #"=" 2)]
+      [(keyword k) v])
+    (log/warn "Dropping invalid attribute pair:" (pr-str attr))))
+
+
+(defn- parse-line
+  "Parse a single line according to the line protocol, returning an event
+  constructed from the parsed data. Returns nil if the line is blank or
+  invalid."
+  [line]
+  (when-not (str/blank? line)
+    (let [[service metric & attrs] (str/split line #"\t+")]
+      (if-not (or (str/blank? service) (str/blank? metric))
+        (try
+          (let [metric (parse-metric metric)
+                attrs (into {} (keep parse-attribute) attrs)]
+            (assoc attrs
+                   :service service
+                   :metric metric))
+          (catch Exception ex
+            (log/warn "Failed to parse metrics line:"
+                      (pr-str line)
+                      (.getName (class ex))
+                      (.getMessage ex))))
+        (log/warn "Dropped invalid metrics line - missing service or metric:"
+                  (pr-str line))))))
+
+
+
+;; ## TCP Source
+
+(defrecord ShellSource
+  [shell command]
+
+  source/Source
+
+  (collect-events
+    [this]
+    (let [result (shell/sh shell "-s" :in command)]
+      (if (zero? (:exit result))
+        (->> (:out result)
+             (str/split-lines)
+             (into [] (keep parse-line)))
+        (log/warn "Failed to execute shell command:"
+                  (pr-str command)
+                  (pr-str (:err result)))))))
+
+
+(defmethod source/initialize :shell
+  [config]
+  (when-not (:command config)
+    (throw (IllegalArgumentException.
+             "Cannot initialize shell source without a command")))
+  (map->ShellSource
+    {:shell (:shell config (System/getenv "SHELL"))
+     :command (:command config)}))

test/solanum/config_test.clj

@@ -53,7 +53,6 @@ outputs:
 (deftest config-errors
   (write-configs!)
   (testing "file reading"
-    (is (nil? (#'config/read-file "target/test/not-a-file.yml")))
     (with-redefs [config/yaml->clj boom!]
       (is (nil? (#'config/read-file path-a)))))
   (testing "source configuration"