Merge pull request #2213 from jcantrill/log2032

LOG-2032: Update fluent kubernetes metadata filter to allow reading f…

fluentd/Gemfile

@@ -3,7 +3,7 @@ gem 'fluentd', ENV['FLUENTD_VERSION'], :source=>"https://rubygems.org"
 gem 'elasticsearch-transport'
 gem 'elasticsearch-api'
 gem 'elasticsearch'
-gem 'fluent-plugin-kubernetes_metadata_filter', '2.7.1'
+gem 'fluent-plugin-kubernetes_metadata_filter', '2.9.3'
 gem 'fluent-plugin-cloudwatch-logs'
 gem 'fluent-plugin-concat'
 gem 'fluent-plugin-detect-exceptions', "0.0.14"

fluentd/Gemfile.lock

@@ -63,9 +63,9 @@ GEM
       fluentd (>= 0.14.10, < 2)
       ltsv
       ruby-kafka (>= 0.7.8, < 2)
-    fluent-plugin-kubernetes_metadata_filter (2.7.1)
-      fluentd (>= 0.14.0, < 1.14)
-      kubeclient (< 5)
+    fluent-plugin-kubernetes_metadata_filter (2.9.3)
+      fluentd (>= 0.14.0, < 1.15)
+      kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
     fluent-plugin-label-router (0.2.4)
       fluentd (>= 0.14.10, < 2)
@@ -123,9 +123,9 @@ GEM
       rest-client (~> 2.0)
     lru_redux (1.1.0)
     ltsv (0.1.2)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0901)
+    mime-types-data (3.2021.1115)
     msgpack (1.3.3)
     multi_json (1.15.0)
     multipart-post (2.1.1)
@@ -180,7 +180,7 @@ DEPENDENCIES
   fluent-plugin-elasticsearch
   fluent-plugin-grafana-loki
   fluent-plugin-kafka
-  fluent-plugin-kubernetes_metadata_filter (= 2.7.1)
+  fluent-plugin-kubernetes_metadata_filter (= 2.9.3)
   fluent-plugin-label-router
   fluent-plugin-multi-format-parser
   fluent-plugin-prometheus

fluentd/rh-manifest.txt

@@ -37,7 +37,7 @@ rubygem-faraday 1.0.1 https://lostisland.github.io/faraday
 rubygem-ffi 1.11.3 https://github.com/ffi/ffi/wiki
 rubygem-fluent-mixin-config-placeholders 0.4.0 https://github.com/tagomoris/fluent-mixin-config-placeholders
 rubygem-fluent-plugin-elasticsearch 5.0.5 https://github.com/uken/fluent-plugin-elasticsearch
-rubygem-fluent-plugin-kubernetes_metadata_filter 2.7.1 https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
+rubygem-fluent-plugin-kubernetes_metadata_filter 2.9.3 https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
 rubygem-fluent-plugin-prometheus 1.7.3 https://github.com/fluent/fluent-plugin-prometheus
 rubygem-fluent-plugin-record-modifier 2.1.0 https://github.com/repeatedly/fluent-plugin-record-modifier
 rubygem-fluent-plugin-remote-syslog 1.1 https://github.com/docebo/fluent-plugin-remote-syslog
@@ -49,8 +49,8 @@ rubygem-http-form_data 2.3.0 https://github.com/httprb/form_data.rb
 rubygem-http_parser.rb 0.6.0 http://github.com/tmm1/http_parser.rb
 rubygem-kubeclient 4.9.2 https://github.com/abonas/kubeclient
 rubygem-lru_redux 1.1.0 https://github.com/SamSaffron/lru_redux
-rubygem-mime-types 3.3.1 https://github.com/mime-types/ruby-mime-types/
-rubygem-mime-types-data 3.2021.0901 https://github.com/mime-types/mime-types-data/
+rubygem-mime-types 3.4.1 https://github.com/mime-types/ruby-mime-types/
+rubygem-mime-types-data 3.2021.1115 https://github.com/mime-types/mime-types-data/
 rubygem-msgpack 1.3.3 http://msgpack.org/
 rubygem-multi_json 1.15.0 https://github.com/intridea/multi_json
 rubygem-multipart-post 2.1.1 https://github.com/nicksieger/multipart-post

fluentd/vendored_gem_src/fluent-plugin-kubernetes_metadata_filter/.circleci/config.yml

@@ -12,7 +12,7 @@ missingdeps: &missingdeps
             cat /etc/os-release
             printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
             sudo cp /tmp/sources.list /etc/apt/sources.list
-            sudo apt-get update
+            sudo apt-get --allow-releaseinfo-change update
             sudo apt-get install cmake libicu-dev libssl-dev
 
 test: &test

fluentd/vendored_gem_src/fluent-plugin-kubernetes_metadata_filter/Gemfile.lock

@@ -1,17 +1,17 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.7.1)
-      fluentd (>= 0.14.0, < 1.14)
-      kubeclient (< 5)
+    fluent-plugin-kubernetes_metadata_filter (2.9.3)
+      fluentd (>= 0.14.0, < 1.15)
+      kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.1)
+    ast (2.4.2)
     bump (0.10.0)
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
@@ -22,27 +22,27 @@ GEM
       github-linguist
     crack (0.4.5)
       rexml
-    docile (1.3.5)
+    docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
-    ffi (1.15.1)
+    ffi (1.15.4)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.13.0)
+    fluentd (1.14.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    github-linguist (7.12.2)
+    github-linguist (7.17.0)
       charlock_holmes (~> 0.7.7)
       escape_utils (~> 1.2.0)
       mini_mime (~> 1.0)
@@ -59,7 +59,7 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
-    http_parser.rb (0.6.0)
+    http_parser.rb (0.8.0)
     jsonpath (1.1.0)
       multi_json
     kubeclient (4.9.2)
@@ -68,43 +68,43 @@ GEM
       recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
     lru_redux (1.1.0)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0225)
-    mini_mime (1.0.2)
+    mime-types-data (3.2021.1115)
+    mini_mime (1.1.2)
     minitest (4.7.5)
     msgpack (1.4.2)
     multi_json (1.15.0)
     netrc (0.11.0)
-    parallel (1.20.1)
-    parser (3.0.0.0)
+    parallel (1.21.0)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
-    power_assert (1.2.0)
+    power_assert (2.0.1)
     public_suffix (4.0.6)
     rainbow (3.0.0)
-    rake (13.0.3)
+    rake (13.0.6)
     recursive-open-struct (1.1.3)
-    regexp_parser (2.0.3)
+    regexp_parser (2.1.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    rr (1.2.1)
-    rubocop (1.8.1)
+    rr (3.0.8)
+    rubocop (1.22.3)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.12.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.0)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
-    rugged (1.1.0)
+    rugged (1.2.0)
     serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -113,7 +113,7 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
+    simplecov_json_formatter (0.1.3)
     strptime (0.2.5)
     test-unit (3.0.9)
       power_assert
@@ -122,15 +122,15 @@ GEM
       test-unit (>= 2.5.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.1)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
-    unicode-display_width (2.0.0)
+    unf_ext (0.0.8)
+    unicode-display_width (2.1.0)
     vcr (6.0.0)
-    webmock (3.11.1)
-      addressable (>= 2.3.6)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)

fluentd/vendored_gem_src/fluent-plugin-kubernetes_metadata_filter/README.md

@@ -45,8 +45,10 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
+* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* `de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
+* `de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
@@ -74,7 +76,7 @@ payload.  The following configuration options are removed:
 
 One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser)
 
-**NOTE** As of this release, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
+**NOTE** As of release v2.1.4, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
 attempt to figure out the source of the metadata fields from the following:
 - If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
 `docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,

fluentd/vendored_gem_src/fluent-plugin-kubernetes_metadata_filter/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.7.1'
+  gem.version       = '2.9.3'
   gem.authors       = ['Jimmi Dyson']
   gem.email         = ['jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
@@ -17,8 +17,8 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.5.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.14']
-  gem.add_runtime_dependency 'kubeclient', '< 5'
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
+  gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
   gem.add_runtime_dependency 'lru_redux'
 
   gem.add_development_dependency 'bump'