Support Application Log collecting (#267)

* Support application log collecting: log4j2 and logback;
* Add application log description to user doc;
* Support classloader matcher to exclude/matcher classloader; 
* Update default configuration to enable app-log collecting.

Co-authored-by: yufu.deng <observeralone@gmail.com>

README.md

@@ -78,7 +78,8 @@ A lightweight & opening Java Agent for Cloud-Native and APM system
 
 * Standardization
     * The tracing data format is fully compatible with the Zipkin data format.
-    * Metric data format fully supports integration with Prometheus.
+    * Metric data format fully supports integration with `Prometheus`.
+    * The application log format is fully compatible with the `Opentelemetry` data format.
 
 
 ## Architecture Diagram

build/src/main/resources/agent.properties

@@ -31,6 +31,7 @@ observability.tracings.tag.response.headers.eg.3=X-EG-Time-Limiter
 
 # -------------------- plugin global config ---------------------
 plugin.observability.global.tracing.enabled=true
+
 plugin.observability.global.metric.enabled=true
 plugin.observability.global.metric.interval=30
 plugin.observability.global.metric.topic=application-metrics
@@ -49,6 +50,36 @@ plugin.integrability.global.forwarded.enabled=true
 plugin.hook.global.foundation.enabled=true
 
 
+plugin.observability.global.log.enabled=true
+plugin.observability.global.log.topic=application-log
+plugin.observability.global.log.url=/application-log
+#plugin.observability.global.log.appendType=console
+
+plugin.observability.global.log.level=INFO
+plugin.observability.global.log.encoder=LogDataJsonEncoder
+
+#plugin.observability.global.log.encoder.collectMdcKeys=
+
+# support pattern:
+# "logLevel": "%-5level",
+# "threadId": "%thread",
+# "location": "%logger{36}",
+# "message": "%msg%n",
+plugin.observability.global.log.encoder.timestamp=%d{UNIX_MILLIS}
+plugin.observability.global.log.encoder.logLevel=%-5level
+plugin.observability.global.log.encoder.threadId=%thread
+plugin.observability.global.log.encoder.location=%logger{36}
+plugin.observability.global.log.encoder.message=%msg%n%xEx{3}
+
+#
+# -------------------- access ---------------------
+## access: servlet and spring gateway
+plugin.observability.access.log.encoder=AccessLogJsonEncoder
+# plugin.observability.access.metric.appendType=kafka
+
+#plugin.observability.logback.log.enabled=false
+#plugin.observability.log4j2.log.enabled=false
+
 # ----------------------------------------------
 # if the plugin configuration is consistent with the global namespace,
 # do not add configuration items not commented out in this default configuration file.
@@ -174,15 +205,7 @@ plugin.observability.redis.metric.url=/platform-metrics
 # plugin.observability.feignClient.tracing.enabled=true
 ## restTemplate tracing
 # plugin.observability.restTemplate.tracing.enabled=true
-#
-# -------------------- access ---------------------
-## access: servlet and spring gateway
-# plugin.observability.access.metric.enabled=true
-# plugin.observability.access.metric.interval=30
-plugin.observability.access.metric.topic=application-log
-plugin.observability.access.metric.url=/application-log
-# plugin.observability.access.metric.appendType=kafka
-#
+
 # -------------------- service name ---------------------
 ## add service name to header by name for easemesh. default name: X-Mesh-RPC-Service
 # plugin.integrability.serviceName.addServiceNameHead.propagate.head=X-Mesh-RPC-Service
@@ -201,6 +224,7 @@ plugin.observability.mongodb.metric.url=/platform-metrics
 ## mongodb foundation
 # plugin.hook.mongodb.foundation.enabled=true
 
+
 # -------------- output ------------------
 ## http/kafka/zipkin server host and port for tracing and metric
 ###### example ######

config/src/main/java/com/megaease/easeagent/config/ConfigFactory.java

@@ -54,17 +54,19 @@ public class ConfigFactory {
         envKeys.add(AGENT_SERVER_PORT_KEY);
     }
 
-    static Map<String, String> updateEnvCfg(Map<String, String> fileCfgMap) {
+    static Map<String, String> updateEnvCfg() {
+        Map<String, String> envCfg = new TreeMap<>();
+
         for (String key : subEnvKeys) {
             String value = System.getProperty(key);
             if (!StringUtils.isEmpty(value)) {
-                fileCfgMap.put(key.substring("easeagent.".length()), value);
+                envCfg.put(key.substring("easeagent.".length()), value);
             }
         }
         for (String key : envKeys) {
             String value = System.getProperty(key);
             if (!StringUtils.isEmpty(value)) {
-                fileCfgMap.put(key, value);
+                envCfg.put(key, value);
             }
         }
 
@@ -77,10 +79,10 @@ static Map<String, String> updateEnvCfg(Map<String, String> fileCfgMap) {
                     strMap.put(entry.getKey(), entry.getValue().toString());
                 }
             }
-            fileCfgMap.putAll(strMap);
+            envCfg.putAll(strMap);
         }
 
-        return fileCfgMap;
+        return envCfg;
     }
 
     private ConfigFactory() {
@@ -134,16 +136,16 @@ public static GlobalConfigs loadConfigs(String pathname, ClassLoader loader) {
 
         // load yaml configuration file if exist
         GlobalConfigs yConfigs = ConfigFactory.loadFromClasspath(loader, CONFIG_YAML_FILE);
-        configs.updateConfigsNotNotify(yConfigs.getConfigs());
+        configs.mergeConfigs(yConfigs);
 
         // override by user special config file
         if (StringUtils.isNotEmpty(pathname)) {
-            Configs configsFromOuterFile = ConfigFactory.loadFromFile(new File(pathname));
-            configs.updateConfigsNotNotify(configsFromOuterFile.getConfigs());
+            GlobalConfigs configsFromOuterFile = ConfigFactory.loadFromFile(new File(pathname));
+            configs.mergeConfigs(configsFromOuterFile);
         }
 
         // check environment cfg override
-        configs.updateConfigsNotNotify(updateEnvCfg(configs.getConfigs()));
+        configs.updateConfigsNotNotify(updateEnvCfg());
 
         if (LOGGER.isDebugEnabled()) {
             final String display = configs.toPrettyDisplay();

config/src/main/java/com/megaease/easeagent/config/ConfigUtils.java

@@ -171,7 +171,7 @@ public static Map<String, String> extractByPrefix(Config config, String prefix)
     }
 
     public static Map<String, String> extractByPrefix(Map<String, String> cfg, String prefix) {
-        Map<String, String> extract = new HashMap<>();
+        Map<String, String> extract = new TreeMap<>();
 
         // override, new configuration KV override previous KV
         cfg.forEach((key, value) -> {

config/src/main/java/com/megaease/easeagent/config/Configs.java

@@ -83,6 +83,12 @@ public String getString(String name) {
         return this.source.get(name);
     }
 
+    public String getString(String name, String defVal) {
+        String val = this.source.get(name);
+
+        return val == null ? defVal : val;
+    }
+
     public Integer getInt(String name) {
         String value = this.source.get(name);
         if (value == null) {

config/src/main/java/com/megaease/easeagent/config/GlobalConfigs.java

@@ -47,8 +47,14 @@ public Configs getOriginalConfig() {
     @Override
     public void updateConfigsNotNotify(Map<String, String> changes) {
         // update original config
+        Map<String, String> newGlobalCfg = new TreeMap<>(this.originalConfig.getConfigs());
+        newGlobalCfg.putAll(changes);
         this.originalConfig.updateConfigsNotNotify(changes);
-        super.updateConfigsNotNotify(changes);
+
+        // report adapter
+        ReportConfigAdapter.convertConfig(newGlobalCfg);
+
+        super.updateConfigsNotNotify(newGlobalCfg);
     }
 
     @Override
@@ -64,6 +70,15 @@ public void updateConfigs(Map<String, String> changes) {
         super.updateConfigs(newGlobalCfg);
     }
 
+    public void mergeConfigs(GlobalConfigs configs) {
+        Map<String, String> merged = configs.getOriginalConfig().getConfigs();
+        if (merged.isEmpty()) {
+            return;
+        }
+        this.updateConfigsNotNotify(merged);
+        return;
+    }
+
     @Override
     public List<String> availableConfigNames() {
         throw new UnsupportedOperationException();

config/src/main/java/com/megaease/easeagent/config/PluginConfig.java

@@ -22,6 +22,7 @@
 import com.megaease.easeagent.plugin.api.config.Const;
 import com.megaease.easeagent.plugin.api.config.IPluginConfig;
 import com.megaease.easeagent.plugin.api.config.PluginConfigChangeListener;
+import com.megaease.easeagent.plugin.utils.common.StringUtils;
 
 import javax.annotation.Nonnull;
 import java.util.*;
@@ -162,10 +163,12 @@ public Long getLong(String property) {
     @Override
     public List<String> getStringList(String property) {
         String value = this.getString(property);
-        if (value == null) {
+        if (StringUtils.isEmpty(value)) {
             return Collections.emptyList();
         }
-        return Arrays.stream(value.split(",")).filter(Objects::nonNull).collect(Collectors.toList());
+        return Arrays.stream(value.split(","))
+            .map(String::trim)
+            .collect(Collectors.toList());
     }
 
     @Override

config/src/main/java/com/megaease/easeagent/config/PluginConfigManager.java

@@ -56,6 +56,11 @@ public String getConfig(String property) {
         return configs.getString(property);
     }
 
+    @Override
+    public String getConfig(String property, String defaultValue) {
+        return configs.getString(property, defaultValue);
+    }
+
     public PluginConfig getConfig(String domain, String namespace, String id) {
         return getConfig(domain, namespace, id, null);
     }