diff --git a/README.md b/README.md index b344b4fb1..9b06c3cfd 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,8 @@ A lightweight & opening Java Agent for Cloud-Native and APM system * Standardization * The tracing data format is fully compatible with the Zipkin data format. - * Metric data format fully supports integration with Prometheus. + * Metric data format fully supports integration with `Prometheus`. + * The application log format is fully compatible with the `Opentelemetry` data format. ## Architecture Diagram diff --git a/build/src/main/resources/agent.properties b/build/src/main/resources/agent.properties index ab089876a..57429a713 100644 --- a/build/src/main/resources/agent.properties +++ b/build/src/main/resources/agent.properties @@ -31,6 +31,7 @@ observability.tracings.tag.response.headers.eg.3=X-EG-Time-Limiter # -------------------- plugin global config --------------------- plugin.observability.global.tracing.enabled=true + plugin.observability.global.metric.enabled=true plugin.observability.global.metric.interval=30 plugin.observability.global.metric.topic=application-metrics @@ -49,6 +50,36 @@ plugin.integrability.global.forwarded.enabled=true plugin.hook.global.foundation.enabled=true +plugin.observability.global.log.enabled=true +plugin.observability.global.log.topic=application-log +plugin.observability.global.log.url=/application-log +#plugin.observability.global.log.appendType=console + +plugin.observability.global.log.level=INFO +plugin.observability.global.log.encoder=LogDataJsonEncoder + +#plugin.observability.global.log.encoder.collectMdcKeys= + +# support pattern: +# "logLevel": "%-5level", +# "threadId": "%thread", +# "location": "%logger{36}", +# "message": "%msg%n", +plugin.observability.global.log.encoder.timestamp=%d{UNIX_MILLIS} +plugin.observability.global.log.encoder.logLevel=%-5level +plugin.observability.global.log.encoder.threadId=%thread +plugin.observability.global.log.encoder.location=%logger{36} +plugin.observability.global.log.encoder.message=%msg%n%xEx{3} + +# +# -------------------- access --------------------- +## access: servlet and spring gateway +plugin.observability.access.log.encoder=AccessLogJsonEncoder +# plugin.observability.access.metric.appendType=kafka + +#plugin.observability.logback.log.enabled=false +#plugin.observability.log4j2.log.enabled=false + # ---------------------------------------------- # if the plugin configuration is consistent with the global namespace, # do not add configuration items not commented out in this default configuration file. @@ -174,15 +205,7 @@ plugin.observability.redis.metric.url=/platform-metrics # plugin.observability.feignClient.tracing.enabled=true ## restTemplate tracing # plugin.observability.restTemplate.tracing.enabled=true -# -# -------------------- access --------------------- -## access: servlet and spring gateway -# plugin.observability.access.metric.enabled=true -# plugin.observability.access.metric.interval=30 -plugin.observability.access.metric.topic=application-log -plugin.observability.access.metric.url=/application-log -# plugin.observability.access.metric.appendType=kafka -# + # -------------------- service name --------------------- ## add service name to header by name for easemesh. default name: X-Mesh-RPC-Service # plugin.integrability.serviceName.addServiceNameHead.propagate.head=X-Mesh-RPC-Service @@ -201,6 +224,7 @@ plugin.observability.mongodb.metric.url=/platform-metrics ## mongodb foundation # plugin.hook.mongodb.foundation.enabled=true + # -------------- output ------------------ ## http/kafka/zipkin server host and port for tracing and metric ###### example ###### diff --git a/config/src/main/java/com/megaease/easeagent/config/ConfigFactory.java b/config/src/main/java/com/megaease/easeagent/config/ConfigFactory.java index f2c5ce465..541a4dc05 100644 --- a/config/src/main/java/com/megaease/easeagent/config/ConfigFactory.java +++ b/config/src/main/java/com/megaease/easeagent/config/ConfigFactory.java @@ -54,17 +54,19 @@ public class ConfigFactory { envKeys.add(AGENT_SERVER_PORT_KEY); } - static Map updateEnvCfg(Map fileCfgMap) { + static Map updateEnvCfg() { + Map envCfg = new TreeMap<>(); + for (String key : subEnvKeys) { String value = System.getProperty(key); if (!StringUtils.isEmpty(value)) { - fileCfgMap.put(key.substring("easeagent.".length()), value); + envCfg.put(key.substring("easeagent.".length()), value); } } for (String key : envKeys) { String value = System.getProperty(key); if (!StringUtils.isEmpty(value)) { - fileCfgMap.put(key, value); + envCfg.put(key, value); } } @@ -77,10 +79,10 @@ static Map updateEnvCfg(Map fileCfgMap) { strMap.put(entry.getKey(), entry.getValue().toString()); } } - fileCfgMap.putAll(strMap); + envCfg.putAll(strMap); } - return fileCfgMap; + return envCfg; } private ConfigFactory() { @@ -134,16 +136,16 @@ public static GlobalConfigs loadConfigs(String pathname, ClassLoader loader) { // load yaml configuration file if exist GlobalConfigs yConfigs = ConfigFactory.loadFromClasspath(loader, CONFIG_YAML_FILE); - configs.updateConfigsNotNotify(yConfigs.getConfigs()); + configs.mergeConfigs(yConfigs); // override by user special config file if (StringUtils.isNotEmpty(pathname)) { - Configs configsFromOuterFile = ConfigFactory.loadFromFile(new File(pathname)); - configs.updateConfigsNotNotify(configsFromOuterFile.getConfigs()); + GlobalConfigs configsFromOuterFile = ConfigFactory.loadFromFile(new File(pathname)); + configs.mergeConfigs(configsFromOuterFile); } // check environment cfg override - configs.updateConfigsNotNotify(updateEnvCfg(configs.getConfigs())); + configs.updateConfigsNotNotify(updateEnvCfg()); if (LOGGER.isDebugEnabled()) { final String display = configs.toPrettyDisplay(); diff --git a/config/src/main/java/com/megaease/easeagent/config/ConfigUtils.java b/config/src/main/java/com/megaease/easeagent/config/ConfigUtils.java index dbc005825..7168dcbfc 100644 --- a/config/src/main/java/com/megaease/easeagent/config/ConfigUtils.java +++ b/config/src/main/java/com/megaease/easeagent/config/ConfigUtils.java @@ -171,7 +171,7 @@ public static Map extractByPrefix(Config config, String prefix) } public static Map extractByPrefix(Map cfg, String prefix) { - Map extract = new HashMap<>(); + Map extract = new TreeMap<>(); // override, new configuration KV override previous KV cfg.forEach((key, value) -> { diff --git a/config/src/main/java/com/megaease/easeagent/config/Configs.java b/config/src/main/java/com/megaease/easeagent/config/Configs.java index bcdea6fbf..38b177e2a 100644 --- a/config/src/main/java/com/megaease/easeagent/config/Configs.java +++ b/config/src/main/java/com/megaease/easeagent/config/Configs.java @@ -83,6 +83,12 @@ public String getString(String name) { return this.source.get(name); } + public String getString(String name, String defVal) { + String val = this.source.get(name); + + return val == null ? defVal : val; + } + public Integer getInt(String name) { String value = this.source.get(name); if (value == null) { diff --git a/config/src/main/java/com/megaease/easeagent/config/GlobalConfigs.java b/config/src/main/java/com/megaease/easeagent/config/GlobalConfigs.java index cd55c8644..b97694415 100644 --- a/config/src/main/java/com/megaease/easeagent/config/GlobalConfigs.java +++ b/config/src/main/java/com/megaease/easeagent/config/GlobalConfigs.java @@ -47,8 +47,14 @@ public Configs getOriginalConfig() { @Override public void updateConfigsNotNotify(Map changes) { // update original config + Map newGlobalCfg = new TreeMap<>(this.originalConfig.getConfigs()); + newGlobalCfg.putAll(changes); this.originalConfig.updateConfigsNotNotify(changes); - super.updateConfigsNotNotify(changes); + + // report adapter + ReportConfigAdapter.convertConfig(newGlobalCfg); + + super.updateConfigsNotNotify(newGlobalCfg); } @Override @@ -64,6 +70,15 @@ public void updateConfigs(Map changes) { super.updateConfigs(newGlobalCfg); } + public void mergeConfigs(GlobalConfigs configs) { + Map merged = configs.getOriginalConfig().getConfigs(); + if (merged.isEmpty()) { + return; + } + this.updateConfigsNotNotify(merged); + return; + } + @Override public List availableConfigNames() { throw new UnsupportedOperationException(); diff --git a/config/src/main/java/com/megaease/easeagent/config/PluginConfig.java b/config/src/main/java/com/megaease/easeagent/config/PluginConfig.java index 05438afbd..2b2126448 100644 --- a/config/src/main/java/com/megaease/easeagent/config/PluginConfig.java +++ b/config/src/main/java/com/megaease/easeagent/config/PluginConfig.java @@ -22,6 +22,7 @@ import com.megaease.easeagent.plugin.api.config.Const; import com.megaease.easeagent.plugin.api.config.IPluginConfig; import com.megaease.easeagent.plugin.api.config.PluginConfigChangeListener; +import com.megaease.easeagent.plugin.utils.common.StringUtils; import javax.annotation.Nonnull; import java.util.*; @@ -162,10 +163,12 @@ public Long getLong(String property) { @Override public List getStringList(String property) { String value = this.getString(property); - if (value == null) { + if (StringUtils.isEmpty(value)) { return Collections.emptyList(); } - return Arrays.stream(value.split(",")).filter(Objects::nonNull).collect(Collectors.toList()); + return Arrays.stream(value.split(",")) + .map(String::trim) + .collect(Collectors.toList()); } @Override diff --git a/config/src/main/java/com/megaease/easeagent/config/PluginConfigManager.java b/config/src/main/java/com/megaease/easeagent/config/PluginConfigManager.java index 922f843d3..2177186dc 100644 --- a/config/src/main/java/com/megaease/easeagent/config/PluginConfigManager.java +++ b/config/src/main/java/com/megaease/easeagent/config/PluginConfigManager.java @@ -56,6 +56,11 @@ public String getConfig(String property) { return configs.getString(property); } + @Override + public String getConfig(String property, String defaultValue) { + return configs.getString(property, defaultValue); + } + public PluginConfig getConfig(String domain, String namespace, String id) { return getConfig(domain, namespace, id, null); } diff --git a/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigAdapter.java b/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigAdapter.java index 8ba711f0a..7461e02e7 100644 --- a/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigAdapter.java +++ b/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigAdapter.java @@ -43,15 +43,17 @@ public static void convertConfig(Map config) { } public static Map extractReporterConfig(Config configs) { - Map cfg = extractAndConvertReporterConfig(configs.getConfigs()); + Map cfg = extractByPrefix(configs.getConfigs(), REPORT); // default config cfg.put(TRACE_ENCODER, NoNull.of(cfg.get(TRACE_ENCODER), SPAN_JSON_ENCODER_NAME)); cfg.put(METRIC_ENCODER, NoNull.of(cfg.get(METRIC_ENCODER), METRIC_JSON_ENCODER_NAME)); - cfg.put(LOG_ENCODER, NoNull.of(cfg.get(LOG_ENCODER), LOG_JSON_ENCODER_NAME)); + cfg.put(LOG_ENCODER, NoNull.of(cfg.get(LOG_ENCODER), LOG_DATA_JSON_ENCODER_NAME)); + cfg.put(LOG_ACCESS_ENCODER, NoNull.of(cfg.get(LOG_ACCESS_ENCODER), ACCESS_LOG_JSON_ENCODER_NAME)); cfg.put(TRACE_SENDER_NAME, NoNull.of(cfg.get(TRACE_SENDER_NAME), getDefaultAppender(cfg))); cfg.put(METRIC_SENDER_NAME, NoNull.of(cfg.get(METRIC_SENDER_NAME), getDefaultAppender(cfg))); + cfg.put(LOG_ACCESS_SENDER_NAME, NoNull.of(cfg.get(LOG_ACCESS_SENDER_NAME), getDefaultAppender(cfg))); cfg.put(LOG_SENDER_NAME, NoNull.of(cfg.get(LOG_SENDER_NAME), getDefaultAppender(cfg))); return cfg; @@ -75,10 +77,14 @@ private static Map extractAndConvertReporterConfig(Map extractTracingConfig(Map srcC } /** - * call after metric config adapter + * For Compatibility, call after metric config adapter * - * extract 'reporter.metric.access.*' to 'reporter.log.*' + * extract 'reporter.metric.access.*' to 'reporter.log.access.*' */ private static void updateAccessLogCfg(Map outputCfg) { // reporter.metric.access.* String prefix = join(METRIC_V2, ConfigConst.Namespace.ACCESS); - Map accessLog = extractAndConvertPrefix(outputCfg, prefix, LOGS); + Map metricAccess = extractByPrefix(outputCfg, prefix); + Map accessLog = extractAndConvertPrefix(metricAccess, prefix, LOG_ACCESS); + + // access log use `kafka` sender + if (METRIC_KAFKA_SENDER_NAME.equals(accessLog.get(LOG_ACCESS_SENDER_NAME))) { + accessLog.put(LOG_ACCESS_SENDER_NAME, KAFKA_SENDER_NAME); + } outputCfg.putAll(accessLog); } @@ -170,27 +182,27 @@ private static Map extractMetricPluginConfig(Map if (idx < 0) { continue; } - String namespace = key.substring(prefix.length(), idx); + String namespaceWithSeparator = key.substring(prefix.length(), idx); String suffix = key.substring(idx + metricKeyLength + 1); String newKey; - if (namespace.equals(globalKey)) { + if (namespaceWithSeparator.equals(globalKey)) { continue; } else { - if (!namespaces.contains(namespace)) { - namespaces.add(namespace); + if (!namespaces.contains(namespaceWithSeparator)) { + namespaces.add(namespaceWithSeparator); Map d = extractAndConvertPrefix(global, - METRIC_V2 + ".", METRIC_V2 + namespace); + METRIC_V2 + ".", METRIC_V2 + namespaceWithSeparator); metricConfigs.putAll(d); } } - if (suffix.equals(ENCODER_KEY)) { - newKey = METRIC_V2 + namespace + ENCODER_KEY; + if (suffix.startsWith(ENCODER_KEY) || suffix.startsWith(ASYNC_KEY)) { + newKey = METRIC_V2 + namespaceWithSeparator + suffix; } else if (suffix.equals(INTERVAL_KEY)) { - newKey = METRIC_V2 + namespace + join(ASYNC_KEY, suffix); + newKey = METRIC_V2 + namespaceWithSeparator + join(ASYNC_KEY, suffix); } else { - newKey = METRIC_V2 + namespace + join(SENDER_KEY, suffix); + newKey = METRIC_V2 + namespaceWithSeparator + join(SENDER_KEY, suffix); } if (newKey.endsWith(APPEND_TYPE_KEY) && e.getValue().equals("kafka")) { @@ -211,8 +223,8 @@ private static Map extractGlobalMetricConfig(Map Map extract = extractAndConvertPrefix(srcCfg, prefix, METRIC_SENDER); for (Map.Entry e : extract.entrySet()) { - if (e.getKey().endsWith(ENCODER_KEY)) { - global.put(join(METRIC_V2, ENCODER_KEY), e.getValue()); + if (e.getKey().startsWith(ENCODER_KEY, METRIC_SENDER.length() + 1)) { + global.put(join(METRIC_V2, e.getKey().substring(METRIC_SENDER.length() + 1)), e.getValue()); } else if (e.getKey().endsWith(INTERVAL_KEY)) { global.put(join(METRIC_ASYNC, INTERVAL_KEY), e.getValue()); } else if (e.getKey().endsWith(APPEND_TYPE_KEY) && e.getValue().equals("kafka")) { @@ -221,6 +233,108 @@ private static Map extractGlobalMetricConfig(Map global.put(e.getKey(), e.getValue()); } } + + + // global log level (async) + global.putAll(extractByPrefix(srcCfg, METRIC_SENDER)); + global.putAll(extractByPrefix(srcCfg, METRIC_ASYNC)); + global.putAll(extractByPrefix(srcCfg, METRIC_ENCODER)); + + return global; + } + + /** + * metric report configuration + * + * extract `plugin.observability.global.metric.*` config items to reporter.metric.sender.*` + * + * extract `plugin.observability.[namespace].metric.*` config items + * to reporter.metric.[namespace].sender.*` + * + * @param srcCfg source configuration map + * @return metric reporter config start with 'reporter.metric.[namespace].sender' + */ + private static Map extractLogPluginConfig(Map srcCfg) { + final String globalKey = "." + ConfigConst.PLUGIN_GLOBAL + "."; + final String prefix = join(ConfigConst.PLUGIN, ConfigConst.OBSERVABILITY); + + String typeKey = join("", ConfigConst.PluginID.LOG, ""); + int typeKeyLength = ConfigConst.PluginID.LOG.length(); + + final String reporterPrefix = LOGS; + + Map global = extractGlobalLogConfig(srcCfg); + HashSet namespaces = new HashSet<>(); + Map outputConfigs = new TreeMap<>(global); + + for (Map.Entry e : srcCfg.entrySet()) { + String key = e.getKey(); + if (!key.startsWith(prefix)) { + continue; + } + int idx = key.indexOf(typeKey, prefix.length()); + if (idx < 0) { + continue; + } else { + idx += 1; + } + String namespaceWithSeparator = key.substring(prefix.length(), idx); + String suffix = key.substring(idx + typeKeyLength + 1); + String newKey; + + if (namespaceWithSeparator.equals(globalKey)) { + continue; + } else { + if (!namespaces.contains(namespaceWithSeparator)) { + namespaces.add(namespaceWithSeparator); + Map d = extractAndConvertPrefix(global, + reporterPrefix + ".", reporterPrefix + namespaceWithSeparator); + outputConfigs.putAll(d); + } + } + + if (suffix.startsWith(ENCODER_KEY) || suffix.startsWith(ASYNC_KEY)) { + newKey = reporterPrefix + namespaceWithSeparator + suffix; + } else { + newKey = reporterPrefix + namespaceWithSeparator + join(SENDER_KEY, suffix); + } + + outputConfigs.put(newKey, e.getValue()); + } + + return outputConfigs; + } + + /** + * extract `plugin.observability.global.log.*` config items to `reporter.log.sender.*` + * extract `plugin.observability.global.log.output.*` config items to `reporter.log.output.*` + * + * @param srcCfg source config map + * @return reporter log config + */ + private static Map extractGlobalLogConfig(Map srcCfg) { + final String prefix = join(ConfigConst.PLUGIN, ConfigConst.OBSERVABILITY, + ConfigConst.PLUGIN_GLOBAL, + ConfigConst.PluginID.LOG); + Map global = new TreeMap<>(); + Map extract = extractAndConvertPrefix(srcCfg, prefix, LOG_SENDER); + + for (Map.Entry e : extract.entrySet()) { + String key = e.getKey(); + if (key.startsWith(ENCODER_KEY, LOG_SENDER.length() + 1)) { + global.put(join(LOGS, key.substring(LOG_SENDER.length() + 1)), e.getValue()); + } else if (key.startsWith(ASYNC_KEY, LOG_SENDER.length() + 1)) { + global.put(join(LOGS, key.substring(LOG_SENDER.length() + 1)), e.getValue()); + } else { + global.put(e.getKey(), e.getValue()); + } + } + + // global log level (async) + global.putAll(extractByPrefix(srcCfg, LOG_SENDER)); + global.putAll(extractByPrefix(srcCfg, LOG_ASYNC)); + global.putAll(extractByPrefix(srcCfg, LOG_ENCODER)); + return global; } } diff --git a/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigConst.java b/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigConst.java index f19898183..1ed31fdef 100644 --- a/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigConst.java +++ b/config/src/main/java/com/megaease/easeagent/config/report/ReportConfigConst.java @@ -28,7 +28,8 @@ private ReportConfigConst() {} public static final String SPAN_JSON_ENCODER_NAME = "SpanJsonEncoder"; public static final String METRIC_JSON_ENCODER_NAME = "MetricJsonEncoder"; - public static final String LOG_JSON_ENCODER_NAME = "LogJsonEncoder"; + public static final String LOG_DATA_JSON_ENCODER_NAME = "LogDataJsonEncoder"; + public static final String ACCESS_LOG_JSON_ENCODER_NAME = "AccessLogJsonEncoder"; public static final String HTTP_SPAN_JSON_ENCODER_NAME = "HttpSpanJsonEncoder"; @@ -49,6 +50,8 @@ private ReportConfigConst() {} public static final String ASYNC_MSG_MAX_BYTES_KEY = "messageMaxBytes"; public static final String ASYNC_MSG_TIMEOUT_KEY = "messageTimeout"; public static final String ASYNC_QUEUE_MAX_SIZE_KEY = "queuedMaxSize"; + public static final String ASYNC_QUEUE_MAX_LOGS_KEY = "queuedMaxLogs"; + public static final String ASYNC_QUEUE_MAX_ITEMS_KEY = "queuedMaxItems"; /** * Reporter v2 configuration @@ -69,9 +72,14 @@ private ReportConfigConst() {} public static final String OUTPUT_SECURITY_PROTOCOL_V2 = join(OUTPUT_SERVER_V2, "security.protocol"); public static final String OUTPUT_SERVERS_SSL = join(OUTPUT_SERVER_V2, "ssl"); + public static final String LOG_ASYNC = join(LOGS, ASYNC_KEY); + public static final String LOG_SENDER = join(LOGS, SENDER_KEY); public static final String LOG_ENCODER = join(LOGS, ENCODER_KEY); - public static final String LOG_ASYNC = join(LOGS, ASYNC_KEY); + + public static final String LOG_ACCESS = join(LOGS, "access"); + public static final String LOG_ACCESS_SENDER = join(LOG_ACCESS, SENDER_KEY); + public static final String LOG_ACCESS_ENCODER = join(LOG_ACCESS, ENCODER_KEY); public static final String TRACE_SENDER = join(TRACE_V2, SENDER_KEY); public static final String TRACE_ENCODER = join(TRACE_V2, ENCODER_KEY); @@ -82,14 +90,17 @@ private ReportConfigConst() {} public static final String METRIC_ASYNC = join(METRIC_V2, ASYNC_KEY); // -------- lv4 -------- + public static final String LOG_SENDER_TOPIC = join(LOG_SENDER, TOPIC_KEY); public static final String LOG_SENDER_NAME = join(LOG_SENDER, APPEND_TYPE_KEY); - public static final String LOG_SENDER_ENABLED_V2 = join(LOG_SENDER, ENABLED_KEY); - public static final String LOG_SENDER_TOPIC_V2 = join(LOG_SENDER, TOPIC_KEY); + + public static final String LOG_ACCESS_SENDER_NAME = join(LOG_ACCESS_SENDER, APPEND_TYPE_KEY); + public static final String LOG_ACCESS_SENDER_ENABLED = join(LOG_ACCESS_SENDER, ENABLED_KEY); + public static final String LOG_ACCESS_SENDER_TOPIC = join(LOG_ACCESS_SENDER, TOPIC_KEY); public static final String LOG_ASYNC_MESSAGE_MAX_BYTES = join(LOG_ASYNC, ASYNC_MSG_MAX_BYTES_KEY); public static final String LOG_ASYNC_REPORT_THREAD = join(LOG_ASYNC, ASYNC_THREAD_KEY); public static final String LOG_ASYNC_MESSAGE_TIMEOUT = join(LOG_ASYNC, ASYNC_MSG_TIMEOUT_KEY); - public static final String LOG_ASYNC_QUEUED_MAX_LOGS = join(LOG_ASYNC, "queuedMaxLogs"); + public static final String LOG_ASYNC_QUEUED_MAX_LOGS = join(LOG_ASYNC, ASYNC_QUEUE_MAX_LOGS_KEY); public static final String LOG_ASYNC_QUEUED_MAX_SIZE = join(LOG_ASYNC, ASYNC_QUEUE_MAX_SIZE_KEY); public static final String TRACE_SENDER_NAME = join(TRACE_SENDER, APPEND_TYPE_KEY); @@ -108,6 +119,8 @@ private ReportConfigConst() {} public static final String METRIC_SENDER_APPENDER = join(METRIC_SENDER, LOG_APPENDER_KEY); public static final String METRIC_ASYNC_INTERVAL = join(METRIC_ASYNC, INTERVAL_KEY); + public static final String METRIC_ASYNC_QUEUED_MAX_ITEMS = join(METRIC_ASYNC, ASYNC_QUEUE_MAX_ITEMS_KEY); + public static final String METRIC_ASYNC_MESSAGE_MAX_BYTES = join(METRIC_ASYNC, ASYNC_MSG_MAX_BYTES_KEY); public static final String OUTPUT_SSL_KEYSTORE_TYPE_V2 = join(OUTPUT_SERVERS_SSL, "keystore.type"); public static final String OUTPUT_KEY_V2 = join(OUTPUT_SERVERS_SSL, "keystore.key"); diff --git a/config/src/test/java/com/megaease/easeagent/config/report/ReportConfigAdapterTest.java b/config/src/test/java/com/megaease/easeagent/config/report/ReportConfigAdapterTest.java index db357ccbe..4e254a3de 100644 --- a/config/src/test/java/com/megaease/easeagent/config/report/ReportConfigAdapterTest.java +++ b/config/src/test/java/com/megaease/easeagent/config/report/ReportConfigAdapterTest.java @@ -116,4 +116,135 @@ public void test_metric_global_v1() { Assert.assertEquals("application-meter", configs.getString(METRIC_SENDER_TOPIC)); Assert.assertEquals(METRIC_KAFKA_SENDER_NAME, configs.getString(METRIC_SENDER_NAME)); } + + @Test + public void test_metric_async_update() { + HashMap cfg = new HashMap<>(); + cfg.put("plugin.observability.global.metric.output.messageMaxBytes", "100"); + cfg.put("plugin.observability.access.metric.output.interval", "100"); + cfg.put(METRIC_ASYNC_MESSAGE_MAX_BYTES, "1000"); + cfg.put(METRIC_ASYNC_INTERVAL, "200"); + + GlobalConfigs configs = new GlobalConfigs(cfg); + Assert.assertEquals("1000", configs.getString(METRIC_ASYNC_MESSAGE_MAX_BYTES)); + Assert.assertEquals("1000", configs.getString(join(METRIC_V2, "access", ASYNC_KEY, ASYNC_MSG_MAX_BYTES_KEY))); + Assert.assertEquals("100", configs.getString(join(METRIC_V2, "access", ASYNC_KEY, INTERVAL_KEY))); + + HashMap changes = new HashMap<>(); + changes.put(METRIC_ASYNC_MESSAGE_MAX_BYTES, "2000"); + changes.put(METRIC_ASYNC_INTERVAL, "150"); + configs.updateConfigs(changes); + Assert.assertEquals("2000", configs.getString(join(METRIC_V2, "access", ASYNC_KEY, ASYNC_MSG_MAX_BYTES_KEY))); + Assert.assertEquals("2000", configs.getString(METRIC_ASYNC_MESSAGE_MAX_BYTES)); + Assert.assertEquals("100", configs.getString(join(METRIC_V2, "access", ASYNC_KEY, INTERVAL_KEY))); + } + + @Test + public void test_log_global() { + HashMap cfg = new HashMap<>(); + cfg.put("plugin.observability.global.log.topic", "application-log"); + cfg.put("plugin.observability.global.log.url", "/application-log"); + cfg.put("plugin.observability.global.log.appendType", "kafka"); + cfg.put("plugin.observability.global.log.output.messageMaxBytes", "100"); + + GlobalConfigs configs = new GlobalConfigs(cfg); + Assert.assertEquals("application-log", configs.getString(LOG_SENDER_TOPIC)); + Assert.assertEquals(KAFKA_SENDER_NAME, configs.getString(LOG_SENDER_NAME)); + Assert.assertEquals("100", configs.getString(LOG_ASYNC_MESSAGE_MAX_BYTES)); + } + + @Test + public void test_access_in_metric() { + HashMap cfg = new HashMap<>(); + cfg.put("plugin.observability.global.log.topic", "application-log"); + cfg.put("plugin.observability.global.log.url", "/application-log"); + cfg.put("plugin.observability.global.log.appendType", "kafka"); + cfg.put("plugin.observability.global.log.encoder", "APP_JSON"); + cfg.put("plugin.observability.global.log.output.messageMaxBytes", "100"); + cfg.put("plugin.observability.access.log.topic", "access-log"); + cfg.put("plugin.observability.access.log.encoder", "LOG_ACCESS_JSON"); + cfg.put("plugin.observability.access.metric.encoder", "ACCESS_JSON"); + + GlobalConfigs configs = new GlobalConfigs(cfg); + Assert.assertEquals("application-log", configs.getString(LOG_SENDER_TOPIC)); + Assert.assertEquals("access-log", configs.getString(LOG_ACCESS_SENDER_TOPIC)); + Assert.assertEquals("ACCESS_JSON", configs.getString(LOG_ACCESS_ENCODER)); + Assert.assertEquals(KAFKA_SENDER_NAME, configs.getString(LOG_ACCESS_SENDER_NAME)); + Assert.assertEquals("100", configs.getString(LOG_ASYNC_MESSAGE_MAX_BYTES)); + } + + @Test + public void test_access_log_update() { + HashMap cfg = new HashMap<>(); + cfg.put("plugin.observability.global.log.topic", "application-log"); + cfg.put("plugin.observability.global.log.url", "/application-log"); + cfg.put("plugin.observability.global.log.appendType", "kafka"); + cfg.put("plugin.observability.global.log.encoder", "APP_JSON"); + cfg.put("plugin.observability.global.log.output.messageMaxBytes", "100"); + cfg.put("plugin.observability.access.log.topic", "access-log"); + cfg.put("plugin.observability.access.log.encoder", "LOG_ACCESS_JSON"); + + GlobalConfigs configs = new GlobalConfigs(cfg); + Assert.assertEquals("application-log", configs.getString(LOG_SENDER_TOPIC)); + Assert.assertEquals("access-log", configs.getString(LOG_ACCESS_SENDER_TOPIC)); + Assert.assertEquals("LOG_ACCESS_JSON", configs.getString(LOG_ACCESS_ENCODER)); + Assert.assertEquals(KAFKA_SENDER_NAME, configs.getString(LOG_ACCESS_SENDER_NAME)); + Assert.assertEquals("100", configs.getString(LOG_ASYNC_MESSAGE_MAX_BYTES)); + + // update + HashMap changes = new HashMap<>(); + changes.put("plugin.observability.access.log.appendType", CONSOLE_SENDER_NAME); + configs.updateConfigs(changes); + Assert.assertEquals(CONSOLE_SENDER_NAME, configs.getString(LOG_ACCESS_SENDER_NAME)); + + changes.put("plugin.observability.access.metric.appendType", KAFKA_SENDER_NAME); + changes.put("plugin.observability.access.log.appendType", CONSOLE_SENDER_NAME); + configs.updateConfigs(changes); + Assert.assertEquals(KAFKA_SENDER_NAME, configs.getString(LOG_ACCESS_SENDER_NAME)); + } + + @Test + public void test_log_global_async() { + HashMap cfg = new HashMap<>(); + cfg.put("plugin.observability.global.log.topic", "application-log"); + cfg.put("plugin.observability.global.log.url", "/application-log"); + cfg.put("plugin.observability.global.log.appendType", "kafka"); + cfg.put("plugin.observability.global.log.output.messageMaxBytes", "100"); + cfg.put("plugin.observability.access.log.topic", "access-log"); + cfg.put("plugin.observability.access.log.encoder", "LOG_ACCESS_JSON"); + + cfg.put(LOG_ASYNC_MESSAGE_MAX_BYTES, "1000"); + cfg.put(LOG_ASYNC_QUEUED_MAX_LOGS, "200"); + + GlobalConfigs configs = new GlobalConfigs(cfg); + Assert.assertEquals("application-log", configs.getString(LOG_SENDER_TOPIC)); + Assert.assertEquals("access-log", configs.getString(LOG_ACCESS_SENDER_TOPIC)); + Assert.assertEquals("LOG_ACCESS_JSON", configs.getString(LOG_ACCESS_ENCODER)); + Assert.assertEquals(KAFKA_SENDER_NAME, configs.getString(LOG_ACCESS_SENDER_NAME)); + + Assert.assertEquals("1000", configs.getString(LOG_ASYNC_MESSAGE_MAX_BYTES)); + Assert.assertEquals("1000", configs.getString(join(LOG_ACCESS, ASYNC_KEY, ASYNC_MSG_MAX_BYTES_KEY))); + } + + @Test + public void test_log_async_update() { + HashMap cfg = new HashMap<>(); + cfg.put("plugin.observability.global.log.output.messageMaxBytes", "100"); + cfg.put("plugin.observability.access.log.output.queuedMaxLogs", "100"); + cfg.put(LOG_ASYNC_MESSAGE_MAX_BYTES, "1000"); + cfg.put(LOG_ASYNC_QUEUED_MAX_LOGS, "200"); + + GlobalConfigs configs = new GlobalConfigs(cfg); + Assert.assertEquals("1000", configs.getString(LOG_ASYNC_MESSAGE_MAX_BYTES)); + Assert.assertEquals("1000", configs.getString(join(LOG_ACCESS, ASYNC_KEY, ASYNC_MSG_MAX_BYTES_KEY))); + Assert.assertEquals("100", configs.getString(join(LOG_ACCESS, ASYNC_KEY, ASYNC_QUEUE_MAX_LOGS_KEY))); + + HashMap changes = new HashMap<>(); + changes.put(LOG_ASYNC_MESSAGE_MAX_BYTES, "2000"); + changes.put(LOG_ASYNC_QUEUED_MAX_LOGS, "150"); + configs.updateConfigs(changes); + Assert.assertEquals("2000", configs.getString(join(LOG_ACCESS, ASYNC_KEY, ASYNC_MSG_MAX_BYTES_KEY))); + Assert.assertEquals("2000", configs.getString(LOG_ASYNC_MESSAGE_MAX_BYTES)); + Assert.assertEquals("100", configs.getString(join(LOG_ACCESS, ASYNC_KEY, ASYNC_QUEUE_MAX_LOGS_KEY))); + } } diff --git a/config/src/test/resources/user.properties b/config/src/test/resources/user.properties new file mode 100644 index 000000000..dadb76546 --- /dev/null +++ b/config/src/test/resources/user.properties @@ -0,0 +1,8 @@ +name=demo-user +system=demo-system + +## topic for kafka use +reporter.metric.sender.topic=application-meter + +#reporter.metric.encoder=MetricJsonEncoder +reporter.metric.output.interval=30 diff --git a/core/src/main/java/com/megaease/easeagent/core/Bootstrap.java b/core/src/main/java/com/megaease/easeagent/core/Bootstrap.java index 821af7809..78c0c6ec8 100644 --- a/core/src/main/java/com/megaease/easeagent/core/Bootstrap.java +++ b/core/src/main/java/com/megaease/easeagent/core/Bootstrap.java @@ -50,6 +50,7 @@ import javax.management.ObjectName; import java.lang.instrument.Instrumentation; import java.lang.management.ManagementFactory; +import java.net.URLClassLoader; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -94,6 +95,10 @@ public static void start(String args, Instrumentation inst) { final GlobalConfigs conf = ConfigFactory.loadConfigs(configPath, Bootstrap.class.getClassLoader()); wrapConfig(conf); + // loader check + GlobalAgentHolder.setAgentClassLoader((URLClassLoader) Bootstrap.class.getClassLoader()); + EaseAgent.agentClassloader = GlobalAgentHolder::getAgentClassLoader; + // init Context/API contextManager = ContextManager.build(conf); EaseAgent.dispatcher = new BridgeDispatcher(); @@ -194,7 +199,8 @@ public static AgentBuilder getAgentBuilder(Configs config, boolean test) { .or(nameStartsWith("brave.")) .or(nameStartsWith("zipkin2.")) .or(nameStartsWith("com.fasterxml")) - .or(nameStartsWith("org.apache.logging")) + .or(nameStartsWith("org.apache.logging") + .and(not(hasSuperClass(named("org.apache.logging.log4j.spi.AbstractLogger"))))) .or(nameStartsWith("kotlin.")) .or(nameStartsWith("javax.")) .or(nameStartsWith("net.bytebuddy.")) diff --git a/core/src/main/java/com/megaease/easeagent/core/GlobalAgentHolder.java b/core/src/main/java/com/megaease/easeagent/core/GlobalAgentHolder.java index ad15ca69e..d0cbe5acd 100644 --- a/core/src/main/java/com/megaease/easeagent/core/GlobalAgentHolder.java +++ b/core/src/main/java/com/megaease/easeagent/core/GlobalAgentHolder.java @@ -21,10 +21,13 @@ import com.megaease.easeagent.httpserver.nano.AgentHttpServer; import com.megaease.easeagent.plugin.report.AgentReport; +import java.net.URLClassLoader; + public class GlobalAgentHolder { private static WrappedConfigManager wrappedConfigManager; private static AgentHttpServer agentHttpServer; private static AgentReport agentReport; + private static URLClassLoader agentLoader; private GlobalAgentHolder() {} @@ -51,4 +54,12 @@ public static void setAgentReport(AgentReport report) { public static AgentReport getAgentReport() { return agentReport; } + + public static void setAgentClassLoader(URLClassLoader loader) { + agentLoader = loader; + } + + public static URLClassLoader getAgentClassLoader() { + return agentLoader; + } } diff --git a/core/src/main/java/com/megaease/easeagent/core/plugin/PluginLoader.java b/core/src/main/java/com/megaease/easeagent/core/plugin/PluginLoader.java index 6e12eea7c..ceda39f23 100644 --- a/core/src/main/java/com/megaease/easeagent/core/plugin/PluginLoader.java +++ b/core/src/main/java/com/megaease/easeagent/core/plugin/PluginLoader.java @@ -50,7 +50,7 @@ public static AgentBuilder load(AgentBuilder ab, Configs conf) { Set sortedTransformations = pointsLoad(); for (ClassTransformation transformation : sortedTransformations) { - ab = ab.type(transformation.getClassMatcher()) + ab = ab.type(transformation.getClassMatcher(), transformation.getClassloaderMatcher()) .transform(compound(transformation.isHasDynamicField(), transformation.getMethodTransformations())); } return ab; diff --git a/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/ClassLoaderMatcherConvert.java b/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/ClassLoaderMatcherConvert.java new file mode 100644 index 000000000..5db2dc584 --- /dev/null +++ b/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/ClassLoaderMatcherConvert.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.core.plugin.matcher; + +import com.megaease.easeagent.core.Bootstrap; +import com.megaease.easeagent.log4j2.FinalClassloaderSupplier; +import com.megaease.easeagent.plugin.matcher.loader.IClassLoaderMatcher; +import com.megaease.easeagent.plugin.matcher.loader.NegateClassLoaderMatcher; +import net.bytebuddy.matcher.ElementMatcher; + +import static com.megaease.easeagent.plugin.matcher.loader.ClassLoaderMatcher.*; +import static net.bytebuddy.matcher.ElementMatchers.*; + +public class ClassLoaderMatcherConvert implements Converter> { + public static final ClassLoaderMatcherConvert INSTANCE = new ClassLoaderMatcherConvert(); + + private static final ElementMatcher agentLoaderMatcher = is(Bootstrap.class.getClassLoader()) + .or(is(FinalClassloaderSupplier.CLASSLOADER)); + + @Override + public ElementMatcher convert(IClassLoaderMatcher source) { + boolean negate; + ElementMatcher matcher; + if (source instanceof NegateClassLoaderMatcher) { + negate = true; + source = source.negate(); + } else { + negate = false; + } + + if (ALL.equals(source)) { + matcher = any(); + } else { + switch (source.getClassLoaderName()) { + case BOOTSTRAP_NAME: + matcher = isBootstrapClassLoader(); + break; + case EXTERNAL_NAME: + matcher = isExtensionClassLoader(); + break; + case SYSTEM_NAME: + matcher = isSystemClassLoader(); + break; + case AGENT_NAME: + matcher = agentLoaderMatcher; + break; + default: + matcher = new NameMatcher(source.getClassLoaderName()); + break; + } + } + + if (negate) { + return not(matcher); + } else { + return matcher; + } + } + + static class NameMatcher implements ElementMatcher { + final String className; + + public NameMatcher(String name) { + this.className = name; + } + + @Override + public boolean matches(ClassLoader target) { + if (target == null) { + return this.className == null; + } + return this.className.equals(target.getClass().getCanonicalName()); + } + } +} diff --git a/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/ClassTransformation.java b/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/ClassTransformation.java index 7c7b43bb0..81a4cc646 100644 --- a/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/ClassTransformation.java +++ b/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/ClassTransformation.java @@ -20,39 +20,50 @@ import com.megaease.easeagent.plugin.Ordered; import lombok.Data; import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; import net.bytebuddy.matcher.ElementMatcher.Junction; import java.util.Set; +import static net.bytebuddy.matcher.ElementMatchers.any; + @Data public class ClassTransformation implements Ordered { private int order; private Junction classMatcher; + private ElementMatcher classloaderMatcher; private Set methodTransformations; private boolean hasDynamicField; public ClassTransformation(int order, + ElementMatcher classloaderMatcher, Junction classMatcher, Set methodTransformations, boolean hasDynamicField) { this.order = order; + if (classloaderMatcher == null) { + this.classloaderMatcher = any(); + } else { + this.classloaderMatcher = classloaderMatcher; + } this.classMatcher = classMatcher; this.methodTransformations = methodTransformations; this.hasDynamicField = hasDynamicField; } + public static Builder builder() { + return new Builder(); + } + @Override public int order() { return this.order; } - public static Builder builder() { - return new Builder(); - } - public static class Builder { private int order; private Junction classMatcher; + private ElementMatcher classloaderMatcher = null; private Set methodTransformations; private boolean hasDynamicField; @@ -64,6 +75,11 @@ public Builder order(int order) { return this; } + public Builder classloaderMatcher(ElementMatcher clmMatcher) { + this.classloaderMatcher = clmMatcher; + return this; + } + public Builder classMatcher(Junction classMatcher) { this.classMatcher = classMatcher; return this; @@ -80,7 +96,8 @@ public Builder hasDynamicField(boolean hasDynamicField) { } public ClassTransformation build() { - return new ClassTransformation(order, classMatcher, methodTransformations, hasDynamicField); + return new ClassTransformation(order, classloaderMatcher, classMatcher, + methodTransformations, hasDynamicField); } public String toString() { diff --git a/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/MethodMatcherConvert.java b/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/MethodMatcherConvert.java index 6c7e3ad8b..831bf5bbf 100644 --- a/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/MethodMatcherConvert.java +++ b/core/src/main/java/com/megaease/easeagent/core/plugin/matcher/MethodMatcherConvert.java @@ -25,6 +25,7 @@ import com.megaease.easeagent.plugin.matcher.operator.NegateMethodMatcher; import com.megaease.easeagent.plugin.matcher.operator.OrMethodMatcher; import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher.Junction; import net.bytebuddy.matcher.NegatingMatcher; @@ -121,7 +122,8 @@ private Junction convert(MethodMatcher matcher) { } if (matcher.getOverriddenFrom() != null) { - mc = isOverriddenFrom(ClassMatcherConvert.INSTANCE.convert(matcher.getOverriddenFrom())); + Junction cls = ClassMatcherConvert.INSTANCE.convert(matcher.getOverriddenFrom()); + mc = isOverriddenFrom(cls); c = c == null ? mc : c.and(mc); } diff --git a/core/src/main/java/com/megaease/easeagent/core/plugin/registry/PluginRegistry.java b/core/src/main/java/com/megaease/easeagent/core/plugin/registry/PluginRegistry.java index 71c8f0269..9bec3d287 100644 --- a/core/src/main/java/com/megaease/easeagent/core/plugin/registry/PluginRegistry.java +++ b/core/src/main/java/com/megaease/easeagent/core/plugin/registry/PluginRegistry.java @@ -21,20 +21,18 @@ import com.megaease.easeagent.core.plugin.interceptor.ProviderChain; import com.megaease.easeagent.core.plugin.interceptor.ProviderChain.Builder; import com.megaease.easeagent.core.plugin.interceptor.ProviderPluginDecorator; -import com.megaease.easeagent.core.plugin.matcher.ClassMatcherConvert; -import com.megaease.easeagent.core.plugin.matcher.ClassTransformation; -import com.megaease.easeagent.core.plugin.matcher.MethodMatcherConvert; -import com.megaease.easeagent.core.plugin.matcher.MethodTransformation; +import com.megaease.easeagent.core.plugin.matcher.*; import com.megaease.easeagent.core.utils.AgentArray; import com.megaease.easeagent.plugin.AgentPlugin; import com.megaease.easeagent.plugin.Points; -import com.megaease.easeagent.plugin.interceptor.InterceptorProvider; import com.megaease.easeagent.plugin.api.logging.Logger; import com.megaease.easeagent.plugin.bridge.EaseAgent; +import com.megaease.easeagent.plugin.interceptor.InterceptorProvider; import com.megaease.easeagent.plugin.matcher.IClassMatcher; import com.megaease.easeagent.plugin.matcher.IMethodMatcher; import net.bytebuddy.description.method.MethodDescription; import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; import net.bytebuddy.matcher.ElementMatcher.Junction; import java.util.Objects; @@ -68,6 +66,8 @@ public static ClassTransformation register(Points points) { IClassMatcher classMatcher = points.getClassMatcher(); boolean hasDynamicField = points.isAddDynamicField(); Junction innerClassMatcher = ClassMatcherConvert.INSTANCE.convert(classMatcher); + ElementMatcher loaderMatcher = ClassLoaderMatcherConvert.INSTANCE + .convert(points.getClassLoaderMatcher()); Set methodMatchers = points.getMethodMatcher(); @@ -87,12 +87,14 @@ public static ClassTransformation register(Points points) { } return mt; }).filter(Objects::nonNull).collect(Collectors.toSet()); + AgentPlugin plugin = POINTS_TO_PLUGIN.get(pointsClassName); int order = plugin.order(); return ClassTransformation.builder().classMatcher(innerClassMatcher) .hasDynamicField(hasDynamicField) .methodTransformations(mInfo) + .classloaderMatcher(loaderMatcher) .order(order).build(); } diff --git a/core/src/test/java/com/megaease/easeagent/core/matcher/ClassLoaderMatcherTest.java b/core/src/test/java/com/megaease/easeagent/core/matcher/ClassLoaderMatcherTest.java new file mode 100644 index 000000000..cfc17188b --- /dev/null +++ b/core/src/test/java/com/megaease/easeagent/core/matcher/ClassLoaderMatcherTest.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.core.matcher; + +import com.megaease.easeagent.core.Bootstrap; +import com.megaease.easeagent.core.plugin.matcher.ClassLoaderMatcherConvert; +import com.megaease.easeagent.plugin.matcher.loader.ClassLoaderMatcher; +import net.bytebuddy.matcher.ElementMatcher; +import org.junit.Assert; +import org.junit.Test; + +import java.net.URL; +import java.net.URLClassLoader; + +public class ClassLoaderMatcherTest { + @Test + public void test_convert() { + ElementMatcher matcher; + // bootstrap + matcher = ClassLoaderMatcherConvert.INSTANCE.convert(ClassLoaderMatcher.BOOTSTRAP); + Assert.assertTrue(matcher.matches(null)); + Assert.assertFalse(matcher.matches(ClassLoader.getSystemClassLoader())); + + // external + matcher = ClassLoaderMatcherConvert.INSTANCE.convert(ClassLoaderMatcher.EXTERNAL); + Assert.assertFalse(matcher.matches(null)); + Assert.assertFalse(matcher.matches(ClassLoader.getSystemClassLoader())); + Assert.assertTrue(matcher.matches(ClassLoader.getSystemClassLoader().getParent())); + + // system + matcher = ClassLoaderMatcherConvert.INSTANCE.convert(ClassLoaderMatcher.SYSTEM); + Assert.assertFalse(matcher.matches(ClassLoader.getSystemClassLoader().getParent())); + Assert.assertTrue(matcher.matches(ClassLoader.getSystemClassLoader())); + + // agent + matcher = ClassLoaderMatcherConvert.INSTANCE.convert(ClassLoaderMatcher.AGENT); + Assert.assertTrue(matcher.matches(Bootstrap.class.getClassLoader())); + Assert.assertFalse(matcher.matches(ClassLoader.getSystemClassLoader().getParent())); + + // name + matcher = ClassLoaderMatcherConvert.INSTANCE + .convert(new ClassLoaderMatcher("com.megaease.easeagent.core.matcher.ClassLoaderMatcherTest.TestClassLoader")); + URL[] urls = new URL[1]; + urls[0] = this.getClass().getProtectionDomain().getCodeSource().getLocation(); + Assert.assertFalse(matcher.matches(Bootstrap.class.getClassLoader())); + Assert.assertTrue(matcher.matches(new TestClassLoader(urls))); + } + + static class TestClassLoader extends URLClassLoader { + public TestClassLoader(URL[] urls) { + super(urls); + } + } +} diff --git a/doc/user-manual.md b/doc/user-manual.md index ac4a03f31..3e6737477 100644 --- a/doc/user-manual.md +++ b/doc/user-manual.md @@ -12,6 +12,7 @@ - [Tracing config](#tracing-config) - [Plugin Configuration](#plugin-configuration) - [Tracing and Metric](#tracing-and-metric) + - [Application Log](#application-log) - [Redirect](#redirect) - [Forwarded headers plugin enabled](#forwarded-headers-plugin-enabled) - [Service Name Head](#service-name-head) @@ -39,6 +40,7 @@ - [Spring AMQP on Message Listener](#spring-amqp-on-message-listener) - [Elasticsearch](#elasticsearch) - [MongoDB](#mongodb) + - [Application Log](#application-log-1) ## Configuration The EaseAgent configuration information can be divided into two categories, one is the **global configuration** and the other is the **plugin configuration**. @@ -242,6 +244,45 @@ Supported components and corresponding namespaces: | jvmGc | `jvmGc` | JVM GC Metric | | JVM Memory | `jvmMemory` | JVM Memory Metric | +#### Application Log +Application log modules collecting application logs printed by the application. + +Supported components/plugins and corresponding namespaces: + +| Plugin/Components | Namespace | Description | +| ----------------- | --------------- | ------------------------- | +| logback | `logback` | Support logback library | +| log4j2 | `log4j2 ` | Support log4j2 library | +| access | `access` | Access log module | + +The default configuration is as follows: + +``` +plugin.observability.global.log.enabled=true +plugin.observability.global.log.appendType=console +plugin.observability.global.log.topic=application-log +plugin.observability.global.log.url=/application-log +plugin.observability.global.log.level=INFO + + +plugin.observability.global.log.encoder=LogDataJsonEncoder +plugin.observability.global.log.encoder.timestamp=%d{UNIX_MILLIS} +plugin.observability.global.log.encoder.logLevel=%-5level +plugin.observability.global.log.encoder.threadId=%thread +plugin.observability.global.log.encoder.location=%logger{36} +plugin.observability.global.log.encoder.message=%msg%n + +plugin.observability.access.log.encoder=AccessLogJsonEncoder + +plugin.observability.logback.log.enabled=false +plugin.observability.log4j2.log.enabled=false +``` + +The `logback` and `log4j2` modules are disabled by default, and they can be `enabled` in the user configuration file to enable collecting logs printed by the application. +The `LogDataJsonEncoder` supports `log4j2` style pattern configuration for each field. + +To send logs data to `Opentelemetry` compatible backend, the corresponding `Encoder` need to be developed. + #### Redirect Redirection feature combined with `EaseMesh` to direct traffic to shadow services to simulate real traffic for the whole site performance test in the production environment in an effective and safe way. For more detail, please reference [EaseMesh](https://megaease.com/easemesh/) documents. @@ -751,3 +792,6 @@ MongoDB schema describes key metrics of MongoDB client invoking, which include: | p95 | double | TP95: The MongoDB execution duration in milliseconds for 95% user. | | p98 | double | TP98: The MongoDB execution duration in milliseconds for 98% user. | | p99 | double | TP99: The MongoDB execution duration in milliseconds for 99% user. | + +## Application Log + diff --git a/metrics/src/test/java/com/megaease/easeagent/metrics/MetricProviderImplTest.java b/metrics/src/test/java/com/megaease/easeagent/metrics/MetricProviderImplTest.java index da854d1b3..573b06008 100644 --- a/metrics/src/test/java/com/megaease/easeagent/metrics/MetricProviderImplTest.java +++ b/metrics/src/test/java/com/megaease/easeagent/metrics/MetricProviderImplTest.java @@ -19,10 +19,10 @@ import com.megaease.easeagent.mock.config.MockConfig; import com.megaease.easeagent.plugin.api.Reporter; -import com.megaease.easeagent.plugin.api.config.ChangeItem; import com.megaease.easeagent.plugin.api.config.IPluginConfig; import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; import com.megaease.easeagent.plugin.api.metric.MetricRegistrySupplier; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; import com.megaease.easeagent.plugin.field.AgentFieldReflectAccessor; import com.megaease.easeagent.plugin.report.EncodedData; import com.megaease.easeagent.plugin.report.tracing.ReportSpan; @@ -30,8 +30,6 @@ import com.megaease.easeagent.plugin.report.metric.MetricReporterFactory; import org.junit.Test; -import java.util.List; - import static org.junit.Assert.*; public class MetricProviderImplTest { @@ -51,6 +49,11 @@ public void report(AccessLogInfo log) { // skip } + @Override + public void report(AgentLogData log) { + // skip + } + @Override public MetricReporterFactory metricReporter() { return new MetricReporterFactory() { diff --git a/metrics/src/test/java/com/megaease/easeagent/metrics/impl/MetricRegistryImplTest.java b/metrics/src/test/java/com/megaease/easeagent/metrics/impl/MetricRegistryImplTest.java index 42601970c..692c67254 100644 --- a/metrics/src/test/java/com/megaease/easeagent/metrics/impl/MetricRegistryImplTest.java +++ b/metrics/src/test/java/com/megaease/easeagent/metrics/impl/MetricRegistryImplTest.java @@ -176,6 +176,7 @@ public void timer() throws InterruptedException { context.stop(); assertEquals(3, timer.getCount()); Snapshot snapshot = timer.getSnapshot(); + assertEquals(3, snapshot.size()); assertEquals(TimeUnit.MILLISECONDS.toNanos(10), snapshot.getMin()); @@ -187,7 +188,7 @@ public void timer() throws InterruptedException { double median = snapshot.getMedian(); String info = "median = " + (int) median; Assert.assertTrue(info, median > TimeUnit.MILLISECONDS.toNanos(20)); - Assert.assertTrue(info, median < TimeUnit.MILLISECONDS.toNanos(80)); + Assert.assertTrue(info, median < TimeUnit.MILLISECONDS.toNanos(120)); assertEquals(TimeUnit.MILLISECONDS.toNanos(200), snapshot.getMax()); } diff --git a/mock/config-mock/src/main/java/com/megaease/easeagent/mock/config/MockConfig.java b/mock/config-mock/src/main/java/com/megaease/easeagent/mock/config/MockConfig.java index 7c447519b..ed3a6978c 100644 --- a/mock/config-mock/src/main/java/com/megaease/easeagent/mock/config/MockConfig.java +++ b/mock/config-mock/src/main/java/com/megaease/easeagent/mock/config/MockConfig.java @@ -19,6 +19,7 @@ import com.megaease.easeagent.config.ConfigFactory; import com.megaease.easeagent.config.Configs; +import com.megaease.easeagent.config.GlobalConfigs; import com.megaease.easeagent.config.PluginConfigManager; import java.io.File; @@ -29,7 +30,7 @@ public class MockConfig { private static final String MOCK_CONFIG_YAML_FILE = "mock_agent.yaml"; private static final String MOCK_CONFIG_PROP_FILE = "mock_agent.properties"; - private static final Configs CONFIGS; + private static final GlobalConfigs CONFIGS; private static final PluginConfigManager PLUGIN_CONFIG_MANAGER; static { @@ -42,15 +43,15 @@ public class MockConfig { initConfigs.put("observability.tracings.output.enabled", "true"); initConfigs.put("plugin.observability.global.tracing.enabled", "true"); initConfigs.put("plugin.observability.global.metric.enabled", "true"); - CONFIGS = new Configs(initConfigs); + CONFIGS = new GlobalConfigs(initConfigs); ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); URL url = classLoader.getResource(MOCK_CONFIG_YAML_FILE); if (url == null) { url = classLoader.getResource(MOCK_CONFIG_PROP_FILE); } if (url != null) { - Configs configsFromOuterFile = ConfigFactory.loadFromFile(new File(url.getFile())); - CONFIGS.updateConfigsNotNotify(configsFromOuterFile.getConfigs()); + GlobalConfigs configsFromOuterFile = ConfigFactory.loadFromFile(new File(url.getFile())); + CONFIGS.mergeConfigs(configsFromOuterFile); } PLUGIN_CONFIG_MANAGER = PluginConfigManager.builder(CONFIGS).build(); } diff --git a/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/MockEaseAgent.java b/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/MockEaseAgent.java index af2b027e8..b8f7927ad 100644 --- a/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/MockEaseAgent.java +++ b/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/MockEaseAgent.java @@ -116,14 +116,14 @@ public static void cleanLastSpan() { * @return Access Log */ public static AccessLogInfo getLastLog() { - return MockReport.getLastLog(); + return MockReport.getLastAccessLog(); } /** * clean last Span cache from Report */ public static void clearLastLog() { - MockReport.cleanLastLog(); + MockReport.cleanLastAccessLog(); } /** diff --git a/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/utils/ContextUtils.java b/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/utils/ContextUtils.java index 3c2892641..c094ac6a8 100644 --- a/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/utils/ContextUtils.java +++ b/mock/plugin-api-mock/src/main/java/com/megaease/easeagent/mock/plugin/api/utils/ContextUtils.java @@ -38,9 +38,10 @@ public static void resetAll() { EaseAgent.initializeContextSupplier.getContext().clear(); MockMetricProvider.clearAll(); OldRedirect.resetRedirect(); + MockTracingProvider.cleanCurrentSpan(); MockTracingProvider.cleanPendingSpans(); MockReport.cleanLastSpan(); - MockReport.cleanLastLog(); + MockReport.cleanLastAccessLog(); MockReport.cleanSkipSpan(); } diff --git a/mock/report-mock/src/main/java/com/megaease/easeagent/mock/report/MockReport.java b/mock/report-mock/src/main/java/com/megaease/easeagent/mock/report/MockReport.java index d590d79fa..1947e0471 100644 --- a/mock/report-mock/src/main/java/com/megaease/easeagent/mock/report/MockReport.java +++ b/mock/report-mock/src/main/java/com/megaease/easeagent/mock/report/MockReport.java @@ -22,9 +22,9 @@ import com.megaease.easeagent.mock.config.MockConfig; import com.megaease.easeagent.mock.report.impl.LastJsonReporter; import com.megaease.easeagent.plugin.api.Reporter; -import com.megaease.easeagent.plugin.api.config.ChangeItem; import com.megaease.easeagent.plugin.api.config.IPluginConfig; import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; import com.megaease.easeagent.plugin.report.EncodedData; import com.megaease.easeagent.plugin.report.tracing.ReportSpan; import com.megaease.easeagent.plugin.utils.common.JsonUtil; @@ -44,7 +44,8 @@ public class MockReport { private static final Logger LOGGER = LoggerFactory.getLogger(MockReport.class); private static final AgentReport AGENT_REPORT = new MockAgentReport(DefaultAgentReport.create(MockConfig.getCONFIGS())); - private static final AtomicReference LAST_LOG = new AtomicReference<>(); + private static final AtomicReference LAST_ACCESS_LOG = new AtomicReference<>(); + private static final AtomicReference LAST_APP_LOG = new AtomicReference<>(); private static final AtomicReference LAST_SPAN = new AtomicReference<>(); private static final AtomicReference LAST_SKIP_SPAN = new AtomicReference<>(); private static volatile MetricFlushable metricFlushable; @@ -136,7 +137,12 @@ public void report(ReportSpan span) { @Override public void report(AccessLogInfo log) { // this.agentReport.report(log); - LAST_LOG.set(log); + LAST_ACCESS_LOG.set(log); + } + + @Override + public void report(AgentLogData log) { + LAST_APP_LOG.set(log); } @Override @@ -193,12 +199,20 @@ public static void cleanLastSpan() { LAST_SPAN.set(null); } - public static AccessLogInfo getLastLog() { - return LAST_LOG.get(); + public static AccessLogInfo getLastAccessLog() { + return LAST_ACCESS_LOG.get(); + } + + public static void cleanLastAccessLog() { + LAST_ACCESS_LOG.set(null); + } + + public static io.opentelemetry.sdk.logs.data.LogData getLastAppLog() { + return LAST_APP_LOG.get(); } - public static void cleanLastLog() { - LAST_LOG.set(null); + public static void cleanLastAppLog() { + LAST_APP_LOG.set(null); } public static ReportSpan getLastSkipSpan() { diff --git a/mock/zipkin-mock/src/main/java/com/megaease/easeagent/mock/zipkin/MockTracingProvider.java b/mock/zipkin-mock/src/main/java/com/megaease/easeagent/mock/zipkin/MockTracingProvider.java index 67afa5bfe..5bb0afeea 100644 --- a/mock/zipkin-mock/src/main/java/com/megaease/easeagent/mock/zipkin/MockTracingProvider.java +++ b/mock/zipkin-mock/src/main/java/com/megaease/easeagent/mock/zipkin/MockTracingProvider.java @@ -19,6 +19,8 @@ import brave.TracerTestUtils; import brave.Tracing; +import brave.propagation.CurrentTraceContext; +import brave.propagation.ThreadLocalCurrentTraceContext; import com.megaease.easeagent.mock.config.MockConfig; import com.megaease.easeagent.mock.report.MockReport; import com.megaease.easeagent.mock.utils.MockProvider; @@ -51,4 +53,11 @@ public Object get() { public static synchronized void cleanPendingSpans() { TracerTestUtils.clean(TRACING.tracer()); } + + public static synchronized void cleanCurrentSpan() { + CurrentTraceContext currentContext = TRACING.currentTraceContext(); + if (currentContext instanceof ThreadLocalCurrentTraceContext) { + ((ThreadLocalCurrentTraceContext) currentContext).clear(); + } + } } diff --git a/plugin-api/pom.xml b/plugin-api/pom.xml index d3101961e..0c5b880ea 100644 --- a/plugin-api/pom.xml +++ b/plugin-api/pom.xml @@ -33,7 +33,6 @@ com.squareup javapoet - com.fasterxml.jackson.core jackson-core @@ -47,11 +46,35 @@ auto-service provided + + + com.google.auto.value + auto-value-annotations + 1.9 + provided + + com.google.code.findbugs jsr305 compile + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-sdk-common + + + io.opentelemetry + opentelemetry-semconv + + + io.opentelemetry + opentelemetry-sdk-logs + diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/Points.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/Points.java index 7e8e4c735..f0309331c 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/Points.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/Points.java @@ -19,6 +19,8 @@ import com.megaease.easeagent.plugin.matcher.IClassMatcher; import com.megaease.easeagent.plugin.matcher.IMethodMatcher; +import com.megaease.easeagent.plugin.matcher.loader.ClassLoaderMatcher; +import com.megaease.easeagent.plugin.matcher.loader.IClassLoaderMatcher; import java.util.Set; @@ -34,11 +36,10 @@ public interface Points { * .hadInterface(A) * .isPublic() * .isAbstract() - * .build() * .or() * .hasSuperClass(B) * .isPublic() - * .build()) + * .build() */ IClassMatcher getClassMatcher(); @@ -76,4 +77,14 @@ public interface Points { default boolean isAddDynamicField() { return false; } + + /** + * Only match classes loaded by the ClassLoaderMatcher + * default as all classloader + * + * @return classloader matcher + */ + default IClassLoaderMatcher getClassLoaderMatcher() { + return ClassLoaderMatcher.ALL; + } } diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/Config.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/Config.java index 56990414f..4042fd183 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/Config.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/Config.java @@ -27,6 +27,8 @@ public interface Config { String getString(String name); + String getString(String name, String defVal); + Integer getInt(String name); Integer getInt(String name, int defValue); diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/ConfigConst.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/ConfigConst.java index 3136aca23..23c7c1baf 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/ConfigConst.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/ConfigConst.java @@ -33,6 +33,7 @@ public interface ConfigConst { // ServiceId String METRIC_SERVICE_ID = "metric"; + String LOG_SERVICE_ID = "log"; String TRACING_SERVICE_ID = "tracing"; String SERVICE_ID_ENABLED_KEY = "enabled"; @@ -157,7 +158,7 @@ interface Namespace { interface PluginID { String TRACING = "tracing"; String METRIC = "metric"; - String LOG = "access-log"; + String LOG = "log"; String REDIRECT = "redirect"; String FORWARDED = "forwarded"; } diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/IConfigFactory.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/IConfigFactory.java index 66830a0bb..a70417a4f 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/IConfigFactory.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/config/IConfigFactory.java @@ -32,6 +32,8 @@ public interface IConfigFactory { */ String getConfig(String property); + String getConfig(String property, String defaultValue); + /** * Returns the agent's plugin configuration. * diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentAttributes.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentAttributes.java new file mode 100644 index 000000000..4b1bd2f7d --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentAttributes.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.plugin.api.otlp.common; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; + +import javax.annotation.Nullable; +import javax.annotation.ParametersAreNonnullByDefault; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; + +@ParametersAreNonnullByDefault +@SuppressWarnings("unchecked, rawtype, unused") +public final class AgentAttributes extends HashMap, Object> implements Attributes { + @Nullable + @Override + public T get(AttributeKey key) { + Object v = super.get(key); + if (v == null) { + return null; + } + return (T)v; + } + + @Override + public void forEach(BiConsumer, ? super Object> consumer) { + super.forEach(consumer); + } + + @Override + public Map, Object> asMap() { + return this; + } + + @Override + public AttributesBuilder toBuilder() { + return new Builder(this); + } + + public static AttributesBuilder builder() { + return new Builder(); + } + + static class Builder implements AttributesBuilder { + AgentAttributes attrs; + private final long capacity; + private final int lengthLimit; + + public Builder(AgentAttributes from) { + this.attrs = from; + this.capacity = Integer.MAX_VALUE; + this.lengthLimit = Integer.MAX_VALUE; + } + + public Builder() { + this(Integer.MAX_VALUE, Integer.MAX_VALUE); + } + + public Builder(int capacity, int limit) { + this.capacity = capacity; + this.lengthLimit = limit; + this.attrs = new AgentAttributes(); + } + + @Override + public Attributes build() { + return this.attrs; + } + + @Override + public AttributesBuilder put(AttributeKey key, int value) { + if (this.attrs.size() > this.capacity) { + return this; + } + this.attrs.put(key, value); + return this; + } + + @Override + public AttributesBuilder put(AttributeKey key, T value) { + if (this.attrs.size() > this.capacity) { + return this; + } + this.attrs.put(key, value); + return this; + } + + @Override + public AttributesBuilder putAll(Attributes attributes) { + if (attributes.size() + this.attrs.size() > this.capacity) { + return this; + } + this.attrs.putAll(attributes.asMap()); + return this; + } + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentInstrumentLibInfo.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentInstrumentLibInfo.java new file mode 100644 index 000000000..4b34df7ed --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentInstrumentLibInfo.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.plugin.api.otlp.common; + +import io.opentelemetry.sdk.common.InstrumentationLibraryInfo; + +import java.util.concurrent.ConcurrentHashMap; + +public class AgentInstrumentLibInfo { + static ConcurrentHashMap infoMap = new ConcurrentHashMap<>(); + + public static InstrumentationLibraryInfo getInfo(String loggerName) { + InstrumentationLibraryInfo info = infoMap.get(loggerName); + if (info != null) { + return info; + } + info = InstrumentationLibraryInfo.create(loggerName, null); + infoMap.putIfAbsent(loggerName, info); + return info; + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentLogData.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentLogData.java new file mode 100644 index 000000000..8ff2f4c96 --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentLogData.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.plugin.api.otlp.common; + +import com.megaease.easeagent.plugin.report.EncodedData; +import io.opentelemetry.sdk.logs.data.LogData; +import io.opentelemetry.sdk.resources.EaseAgentResource; + +import java.util.Map; + +public interface AgentLogData extends LogData { + /** + * get logger thread name + * @return thread name + */ + String getThreadName(); + + /** + * get logger name + * @return logger name + */ + String getLocation(); + + /** + * get unix timestamp in milliseconds + * @return timestamp + */ + long getEpochMillis(); + + /** + * get agent resource - system/service + * @return agent resource + */ + EaseAgentResource getAgentResource(); + + /** + * complete attributes + */ + void completeAttributes(); + + /** + * return pattern map + * @return pattern map + */ + Map getPatternMap(); + + /** + * return throwable/Exception + * @return throwbale + */ + Throwable getThrowable(); + + /** + * return encoded data + * @return encoded data + */ + EncodedData getEncodedData(); + + void setEncodedData(EncodedData data); +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentLogDataImpl.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentLogDataImpl.java new file mode 100644 index 000000000..f5da3b0b6 --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/AgentLogDataImpl.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.plugin.api.otlp.common; + +import com.megaease.easeagent.plugin.report.EncodedData; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.sdk.common.InstrumentationLibraryInfo; +import io.opentelemetry.sdk.logs.data.Body; +import io.opentelemetry.sdk.logs.data.Severity; +import io.opentelemetry.sdk.resources.EaseAgentResource; +import lombok.Data; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@Data +@SuppressWarnings("unused") +public class AgentLogDataImpl implements AgentLogData { + private EaseAgentResource resource = EaseAgentResource.getResource(); + private InstrumentationLibraryInfo instrumentationLibraryInfo; + private long epochMillis; + private SpanContext spanContext; + private Severity severity; + private String severityText; + private String name = null; + + private Body body; + private Attributes attributes; + + private String threadName; + private long threadId; + private Throwable throwable; + + private Map patternMap = null; + private EncodedData encodedData; + + public AgentLogDataImpl(Builder builder) { + this.epochMillis = builder.epochMills; + this.spanContext = builder.spanContext == null ? SpanContext.getInvalid() : builder.spanContext; + this.severity = builder.severity; + this.severityText = builder.severityText != null ? builder.severityText : this.severity.name(); + this.body = builder.body; + + this.attributes = builder.attributesBuilder != null + ? builder.attributesBuilder.build() + : AgentAttributes.builder().build(); + + this.instrumentationLibraryInfo = AgentInstrumentLibInfo.getInfo(builder.logger); + this.threadName = builder.threadName; + this.threadId = builder.threadId; + this.throwable = builder.throwable; + } + + @Override + public String getThreadName() { + return this.threadName; + } + + @Override + public String getLocation() { + return this.instrumentationLibraryInfo.getName(); + } + + @Override + public EaseAgentResource getAgentResource() { + return this.resource; + } + + @Override + public void completeAttributes() { + AttributesBuilder attrsBuilder = this.attributes.toBuilder(); + if (this.throwable != null) { + attrsBuilder.put(SemanticKey.EXCEPTION_TYPE, throwable.getClass().getName()); + attrsBuilder.put(SemanticKey.EXCEPTION_MESSAGE, throwable.getMessage()); + + StringWriter writer = new StringWriter(); + throwable.printStackTrace(new PrintWriter(writer)); + attrsBuilder.put(SemanticKey.EXCEPTION_STACKTRACE, writer.toString()); + } + + attrsBuilder.put(SemanticKey.THREAD_NAME, threadName); + attrsBuilder.put(SemanticKey.THREAD_ID, threadId); + } + + @Override + public Map getPatternMap() { + if (this.patternMap == null) { + this.patternMap = new HashMap<>(); + } + return this.patternMap; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public long getEpochNanos() { + return TimeUnit.MILLISECONDS.toNanos(this.epochMillis); + } + + @SuppressWarnings("UnusedReturnValue") + public static class Builder { + private String logger; + private long epochMills; + private SpanContext spanContext; + private Severity severity; + private String severityText; + private Body body; + private Throwable throwable; + + private AttributesBuilder attributesBuilder = null; + + private String threadName; + private long threadId; + + public Builder logger(String logger) { + this.logger = logger; + return this; + } + + public Builder epochMills(long timestamp) { + this.epochMills = timestamp; + return this; + } + + public Builder spanContext() { + this.spanContext = OtlpSpanContext.getLogSpanContext(); + return this; + } + + public Builder severity(Severity level) { + this.severity = level; + return this; + } + + public Builder severityText(String level) { + this.severityText = level; + return this; + } + + public Builder body(String msg) { + this.body = Body.string(msg); + return this; + } + + public Builder thread(Thread thread) { + this.threadName = thread.getName(); + this.threadId = thread.getId(); + return this; + } + + public Builder throwable(Throwable throwable) { + this.throwable = throwable; + return this; + } + + public Builder contextData(Collection keys, Map data) { + if (keys == null || keys.isEmpty()) { + if (data.isEmpty()) { + return this; + } + keys = data.keySet(); + } + + AttributesBuilder ab = getAttributesBuilder(); + for (String key : keys) { + ab.put(SemanticKey.stringKey(key), data.get(key)); + } + return this; + } + + public AttributesBuilder getAttributesBuilder() { + if (attributesBuilder == null) { + attributesBuilder = AgentAttributes.builder(); + } + return attributesBuilder; + } + + public AgentLogData build() { + return new AgentLogDataImpl(this); + } + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/LogMapper.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/LogMapper.java new file mode 100644 index 000000000..4f13be61f --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/LogMapper.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.megaease.easeagent.plugin.api.otlp.common; + +import com.megaease.easeagent.plugin.api.config.IPluginConfig; +import com.megaease.easeagent.plugin.interceptor.MethodInfo; + +public interface LogMapper { + public String MDC_KEYS = "encoder.collectMDCKeys"; + + AgentLogData mapLoggingEvent(MethodInfo methodInfo, int levelInt, IPluginConfig config); +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/OtlpSpanContext.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/OtlpSpanContext.java new file mode 100644 index 000000000..fc89732c5 --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/OtlpSpanContext.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.plugin.api.otlp.common; + +import com.megaease.easeagent.plugin.api.Context; +import com.megaease.easeagent.plugin.api.trace.Span; +import com.megaease.easeagent.plugin.bridge.EaseAgent; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; + +public class OtlpSpanContext { + OtlpSpanContext() {} + + public static SpanContext getLogSpanContext() { + Context context = EaseAgent.getContext(); + SpanContext spanContext; + + if (!context.currentTracing().hasCurrentSpan()) { + spanContext = SpanContext.getInvalid(); + } else { + Span span = context.currentTracing().currentSpan(); + spanContext = new AgentSpanContext(span, TraceFlags.getSampled(), TraceState.getDefault()); + } + return spanContext; + } + + static class AgentSpanContext implements SpanContext { + String traceId; + String spanId; + TraceFlags flags; + TraceState state; + boolean remote; + + public AgentSpanContext(Span span, TraceFlags flags, TraceState state) { + this(span, flags, state, false); + } + + public AgentSpanContext(Span span, TraceFlags flags, TraceState state, boolean isRemote) { + this.traceId = Long.toHexString(span.traceId()); + this.spanId = Long.toHexString(span.spanId()); + this.flags = flags; + this.state = state; + this.remote = isRemote; + } + + @Override + public String getTraceId() { + return this.traceId; + } + + @Override + public String getSpanId() { + return this.spanId; + } + + @Override + public TraceFlags getTraceFlags() { + return this.flags; + } + + @Override + public TraceState getTraceState() { + return this.state; + } + + @Override + public boolean isRemote() { + return this.remote; + } + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/SemanticKey.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/SemanticKey.java new file mode 100644 index 000000000..d247a6e1c --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/api/otlp/common/SemanticKey.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.plugin.api.otlp.common; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.semconv.trace.attributes.SemanticAttributes; +import org.checkerframework.checker.units.qual.C; + +import java.util.concurrent.ConcurrentHashMap; + +public class SemanticKey { + public static final String SCHEMA_URL = SemanticAttributes.SCHEMA_URL; + public static final AttributeKey THREAD_NAME = SemanticAttributes.THREAD_NAME; + public static final AttributeKey THREAD_ID = SemanticAttributes.THREAD_ID; + + public static final AttributeKey EXCEPTION_TYPE = SemanticAttributes.EXCEPTION_TYPE; + public static final AttributeKey EXCEPTION_MESSAGE = SemanticAttributes.EXCEPTION_MESSAGE; + public static final AttributeKey EXCEPTION_STACKTRACE = SemanticAttributes.EXCEPTION_STACKTRACE; + + private static final ConcurrentHashMap> keysMap = new ConcurrentHashMap<>(); + + public static AttributeKey stringKey(String key) { + AttributeKey vk = keysMap.get(key); + return vk != null ? vk : keysMap.computeIfAbsent(key, AttributeKey::stringKey); + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/EaseAgent.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/EaseAgent.java index 6486625af..302429588 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/EaseAgent.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/EaseAgent.java @@ -30,6 +30,9 @@ import com.megaease.easeagent.plugin.api.metric.name.Tags; import com.megaease.easeagent.plugin.report.AgentReport; +import java.net.URLClassLoader; +import java.util.function.Supplier; + /** * the bridge api will be initiated when agent startup */ @@ -43,6 +46,12 @@ public final class EaseAgent { public static IDispatcher dispatcher = new NoOpDispatcher(); + public static Supplier agentClassloader = () -> null; + + public static URLClassLoader getAgentClassLoader() { + return agentClassloader.get(); + } + public static AgentReport getAgentReport() { return agentReport; } @@ -77,7 +86,7 @@ public static Config getConfig() { } /** - * Returns a configuration property from the agent's all configuration. + * Returns a configuration property from the agent's global configuration. * * @return The configuration of this Java agent. */ @@ -85,6 +94,17 @@ public static String getConfig(String property) { return configFactory.getConfig(property); } + /** + * find the configuration property from the agent's global configuration. + * if not exist, then return @{defaultValue} + * + * @param defaultValue default value returned when the property is not exist + * @return The configuration of this Java agent. + */ + public static String getConfig(String property, String defaultValue) { + return configFactory.getConfig(property, defaultValue); + } + /** * get a Config by domain, namespace and name * diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpAgentReporter.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpAgentReporter.java index f11aaec3a..885fc8cd2 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpAgentReporter.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpAgentReporter.java @@ -18,15 +18,13 @@ package com.megaease.easeagent.plugin.bridge; import com.megaease.easeagent.plugin.api.Reporter; -import com.megaease.easeagent.plugin.api.config.ChangeItem; import com.megaease.easeagent.plugin.api.config.IPluginConfig; import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; import com.megaease.easeagent.plugin.report.AgentReport; import com.megaease.easeagent.plugin.report.metric.MetricReporterFactory; import com.megaease.easeagent.plugin.report.tracing.ReportSpan; -import java.util.List; - public class NoOpAgentReporter implements AgentReport { @Override public void report(ReportSpan span) { @@ -38,6 +36,11 @@ public void report(AccessLogInfo log) { // ignored } + @Override + public void report(AgentLogData log) { + // ignored + } + @Override public MetricReporterFactory metricReporter() { return new MetricReporterFactory() { diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpConfigFactory.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpConfigFactory.java index a8ba8442d..9a7639523 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpConfigFactory.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/bridge/NoOpConfigFactory.java @@ -35,6 +35,11 @@ public String getConfig(String property) { return null; } + @Override + public String getConfig(String property, String defaultValue) { + return defaultValue; + } + @Override public IPluginConfig getConfig(String domain, String namespace, String id) { return new NoOpIPluginConfig(domain, namespace, id); @@ -54,6 +59,11 @@ public String getString(String name) { return null; } + @Override + public String getString(String name, String defVal) { + return defVal; + } + @Override public Integer getInt(String name) { return null; @@ -94,11 +104,7 @@ public Double getDouble(String name) { @Override public Double getDouble(String name, double defValue) { - Double aDouble = getDouble(name); - if (aDouble == null) { - return defValue; - } - return aDouble; + return defValue; } @Override @@ -108,11 +114,7 @@ public Long getLong(String name) { @Override public Long getLong(String name, long defValue) { - Long aLong = getLong(name); - if (aLong == null) { - return defValue; - } - return aLong; + return defValue; } @Override diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/field/NullObject.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/field/NullObject.java index 730e74b39..634cc5fc5 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/field/NullObject.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/field/NullObject.java @@ -22,7 +22,7 @@ * avoiding NullPointerException when serialized */ public class NullObject { - public static Object NULL = new Object(); + public static final Object NULL = new Object(); public String toString() { return "null"; diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/ClassMatcher.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/ClassMatcher.java index 66bd66e4e..7916dadbc 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/ClassMatcher.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/ClassMatcher.java @@ -31,7 +31,6 @@ public class ClassMatcher implements IClassMatcher { private ClassMatch matchType; private int modifier = Modifier.ACC_NONE; private int notModifier = Modifier.ACC_NONE; - private String classLoader; public static final int MODIFIER_MASK = Modifier.ACC_ABSTRACT | Modifier.ACC_INTERFACE | Modifier.ACC_PRIVATE | Modifier.ACC_PUBLIC | Modifier.ACC_PROTECTED; @@ -39,12 +38,11 @@ public class ClassMatcher implements IClassMatcher { protected ClassMatcher() { } - private ClassMatcher(String name, ClassMatch type, int modifier, int notModifier, String loaderName) { + private ClassMatcher(String name, ClassMatch type, int modifier, int notModifier) { this.name = name; this.matchType = type; this.modifier = modifier; this.notModifier = notModifier; - this.classLoader = loaderName; } public static ClassMatcherBuilder builder() { @@ -56,7 +54,6 @@ public static class ClassMatcherBuilder { private ClassMatch matchType; private int modifier; private int notModifier; - private String classLoader; private IClassMatcher left; private Operator operator = Operator.AND; @@ -131,7 +128,6 @@ public ClassMatcherBuilder hasInterface(String className) { return this.name(className).matchType(ClassMatch.INTERFACE); } - public ClassMatcherBuilder matchType(ClassMatch matchType) { this.matchType = matchType; return this; @@ -147,11 +143,6 @@ public ClassMatcherBuilder notModifier(int notModifier) { return this; } - public ClassMatcherBuilder classLoader(String classLoader) { - this.classLoader = classLoader; - return this; - } - public ClassMatcherBuilder isPublic() { this.modifier |= Modifier.ACC_PUBLIC; return this; @@ -193,7 +184,7 @@ protected ClassMatcherBuilder name(String name) { } public IClassMatcher build() { - IClassMatcher matcher = new ClassMatcher(name, matchType, modifier, notModifier, classLoader); + IClassMatcher matcher = new ClassMatcher(name, matchType, modifier, notModifier); if (this.isNegate) { matcher = matcher.negate(); @@ -203,6 +194,7 @@ public IClassMatcher build() { return matcher; } + IClassMatcher points; switch (this.operator) { case OR: return new OrClassMatcher(this.left, matcher); @@ -220,8 +212,7 @@ public String toString() { + ", notModifier=" + this.notModifier + ", isNegate=" + this.isNegate + ", operate=" + this.operator - + ", left=" + this.left - + ", classLoader=" + this.classLoader + ")"; + + ", left=" + this.left + ")"; } } } diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/ClassLoaderMatcher.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/ClassLoaderMatcher.java new file mode 100644 index 000000000..366e2371b --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/ClassLoaderMatcher.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.megaease.easeagent.plugin.matcher.loader; + +import com.megaease.easeagent.plugin.utils.common.StringUtils; + +@SuppressWarnings("unused") +public class ClassLoaderMatcher implements IClassLoaderMatcher { + public static final String BOOTSTRAP_NAME = "bootstrap"; + public static final String EXTERNAL_NAME = "external"; + public static final String SYSTEM_NAME = "system"; + public static final String AGENT_NAME = "agent"; + + // predefined classloader name and matcher + public static final ClassLoaderMatcher ALL = new ClassLoaderMatcher("all"); + public static final ClassLoaderMatcher BOOTSTRAP = new ClassLoaderMatcher(BOOTSTRAP_NAME); + public static final ClassLoaderMatcher EXTERNAL = new ClassLoaderMatcher(EXTERNAL_NAME); + public static final ClassLoaderMatcher SYSTEM = new ClassLoaderMatcher(SYSTEM_NAME); + public static final ClassLoaderMatcher AGENT = new ClassLoaderMatcher(AGENT_NAME); + + // classloader class name or Predefined names + String classLoaderName; + + public ClassLoaderMatcher(String loaderName) { + if (StringUtils.isEmpty(loaderName)) { + this.classLoaderName = BOOTSTRAP_NAME; + } else { + this.classLoaderName = loaderName; + } + } + + @Override + public String getClassLoaderName() { + return this.classLoaderName; + } + + @Override + public IClassLoaderMatcher negate() { + return new NegateClassLoaderMatcher(this); + } + + @Override + public int hashCode() { + return this.classLoaderName.hashCode(); + } + + public boolean equals(Object o) { + if (!(o instanceof ClassLoaderMatcher)) { + return false; + } + return this.classLoaderName.equals(((ClassLoaderMatcher) o).classLoaderName); + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/IClassLoaderMatcher.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/IClassLoaderMatcher.java new file mode 100644 index 000000000..38faf8e47 --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/IClassLoaderMatcher.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.megaease.easeagent.plugin.matcher.loader; + +import com.megaease.easeagent.plugin.matcher.Matcher; + +public interface IClassLoaderMatcher extends Matcher { + String getClassLoaderName(); + + IClassLoaderMatcher negate(); +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/NegateClassLoaderMatcher.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/NegateClassLoaderMatcher.java new file mode 100644 index 000000000..ac70231e1 --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/matcher/loader/NegateClassLoaderMatcher.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.plugin.matcher.loader; + +public class NegateClassLoaderMatcher implements IClassLoaderMatcher { + IClassLoaderMatcher matcher; + + public NegateClassLoaderMatcher(IClassLoaderMatcher matcher) { + this.matcher = matcher; + } + + @Override + public String getClassLoaderName() { + return this.matcher.getClassLoaderName(); + } + + @Override + public IClassLoaderMatcher negate() { + return this.matcher; + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/AgentReport.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/AgentReport.java index e86a3f084..cac6f55bb 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/AgentReport.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/AgentReport.java @@ -18,6 +18,7 @@ package com.megaease.easeagent.plugin.report; import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; import com.megaease.easeagent.plugin.report.tracing.ReportSpan; import com.megaease.easeagent.plugin.report.metric.MetricReporterFactory; @@ -33,11 +34,17 @@ public interface AgentReport { void report(ReportSpan span); /** - * report log + * report access-log * @param log log info */ void report(AccessLogInfo log); + /** + * report application log + * @param log log info + */ + void report(AgentLogData log); + /** * Metric reporters factory * @return metric reporters factory diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/EncodedData.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/EncodedData.java index 7e02b3563..2f0d09594 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/EncodedData.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/report/EncodedData.java @@ -21,4 +21,16 @@ public interface EncodedData { int size(); byte[] getData(); + + EncodedData EMPTY = new EncodedData() { + @Override + public int size() { + return 0; + } + + @Override + public byte[] getData() { + return new byte[0]; + } + }; } diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/tools/loader/AgentHelperClassLoader.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/tools/loader/AgentHelperClassLoader.java new file mode 100644 index 000000000..52edc7739 --- /dev/null +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/tools/loader/AgentHelperClassLoader.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.megaease.easeagent.plugin.tools.loader; + +import com.megaease.easeagent.plugin.utils.common.WeakConcurrentMap; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.concurrent.ConcurrentHashMap; + +/** + * if there are classes with the same classname in user classloaders and Agent classloader, + * to avoid class cast exception in plugins, only load these classes by user classloaders in plugin context. + * Other related plugin classes loaded by this classloader. + */ +public class AgentHelperClassLoader extends URLClassLoader { + private static final ConcurrentHashMap helpUrls = new ConcurrentHashMap<>(); + private static final WeakConcurrentMap helpLoaders = new WeakConcurrentMap<>(); + + private final URLClassLoader agentClassLoader; + + public AgentHelperClassLoader(URL[] urls, ClassLoader parent, URLClassLoader agent) { + // may lead to classloader leak here + super(urls, parent); + this.agentClassLoader = agent; + } + + public static void registryUrls(Class clazz) { + URL url = clazz.getProtectionDomain().getCodeSource().getLocation(); + helpUrls.putIfAbsent(url, url); + } + + public static AgentHelperClassLoader getClassLoader(ClassLoader parent, URLClassLoader agent) { + AgentHelperClassLoader help = helpLoaders.getIfPresent(parent); + if (help != null) { + return help; + } else { + URL[] urls; + if (helpUrls.isEmpty()) { + urls = new URL[0]; + } else { + urls = helpUrls.keySet().toArray(new URL[1]); + } + help = new AgentHelperClassLoader(urls, parent, agent); + if (helpLoaders.putIfProbablyAbsent(parent, help) == null) { + return help; + } else { + return helpLoaders.getIfPresent(parent); + } + } + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + try { + return super.loadClass(name, resolve); + } catch (ClassNotFoundException e) { + try { + final Class aClass = this.agentClassLoader.loadClass(name); + if (resolve) { + resolveClass(aClass); + } + return aClass; + } catch (ClassNotFoundException ignored) { + // ignored + } + throw e; + } + } + + @Override + public URL findResource(String name) { + URL url = super.findResource(name); + try { + url = this.agentClassLoader.getResource(name); + if (url != null) { + return url; + } + } catch (Exception ignored) { + // ignored + } + return url; + } +} diff --git a/plugin-api/src/main/java/com/megaease/easeagent/plugin/tools/metrics/HttpLog.java b/plugin-api/src/main/java/com/megaease/easeagent/plugin/tools/metrics/HttpLog.java index a4af7479b..a68fa8967 100644 --- a/plugin-api/src/main/java/com/megaease/easeagent/plugin/tools/metrics/HttpLog.java +++ b/plugin-api/src/main/java/com/megaease/easeagent/plugin/tools/metrics/HttpLog.java @@ -33,30 +33,30 @@ public class HttpLog { public AccessLogInfo prepare(String system, String serviceName, Long beginTime, Span span, AccessLogServerInfo serverInfo) { - AccessLogInfo accessLogInfo = prepare(system, serviceName, beginTime, serverInfo); + AccessLogInfo accessLog = prepare(system, serviceName, beginTime, serverInfo); if (span == null) { - return accessLogInfo; + return accessLog; } - accessLogInfo.setTraceId(span.traceIdString()); - accessLogInfo.setSpanId(span.spanIdString()); - accessLogInfo.setParentSpanId(span.parentIdString()); - return accessLogInfo; + accessLog.setTraceId(span.traceIdString()); + accessLog.setSpanId(span.spanIdString()); + accessLog.setParentSpanId(span.parentIdString()); + return accessLog; } private AccessLogInfo prepare(String system, String serviceName, Long beginTime, AccessLogServerInfo serverInfo) { - AccessLogInfo accessLogInfo = new AccessLogInfo(); - accessLogInfo.setSystem(system); - accessLogInfo.setService(serviceName); - accessLogInfo.setHostName(HostAddress.localhost()); - accessLogInfo.setHostIpv4(HostAddress.getHostIpv4()); - accessLogInfo.setUrl(serverInfo.getMethod() + " " + serverInfo.getRequestURI()); - accessLogInfo.setMethod(serverInfo.getMethod()); - accessLogInfo.setHeaders(serverInfo.findHeaders()); - accessLogInfo.setBeginTime(beginTime); - accessLogInfo.setQueries(getQueries(serverInfo)); - accessLogInfo.setClientIP(serverInfo.getClientIP()); - accessLogInfo.setBeginCpuTime(System.nanoTime()); - return accessLogInfo; + AccessLogInfo accessLog = new AccessLogInfo(); + accessLog.setSystem(system); + accessLog.setService(serviceName); + accessLog.setHostName(HostAddress.localhost()); + accessLog.setHostIpv4(HostAddress.getHostIpv4()); + accessLog.setUrl(serverInfo.getMethod() + " " + serverInfo.getRequestURI()); + accessLog.setMethod(serverInfo.getMethod()); + accessLog.setHeaders(serverInfo.findHeaders()); + accessLog.setBeginTime(beginTime); + accessLog.setQueries(getQueries(serverInfo)); + accessLog.setClientIP(serverInfo.getClientIP()); + accessLog.setBeginCpuTime(System.nanoTime()); + return accessLog; } private Map getQueries(AccessLogServerInfo serverInfo) { @@ -71,26 +71,26 @@ private Map getQueries(AccessLogServerInfo serverInfo) { return queries; } - public String getLogString(AccessLogInfo accessLogInfo, boolean success, Long beginTime, AccessLogServerInfo serverInfo) { - this.finish(accessLogInfo, success, beginTime, serverInfo); + public String getLogString(AccessLogInfo accessLog, boolean success, Long beginTime, AccessLogServerInfo serverInfo) { + this.finish(accessLog, success, beginTime, serverInfo); List list = new ArrayList<>(1); - list.add(accessLogInfo); + list.add(accessLog); return JsonUtil.toJson(list); } - public void finish(AccessLogInfo accessLogInfo, boolean success, Long beginTime, AccessLogServerInfo serverInfo) { - accessLogInfo.setStatusCode(serverInfo.getStatusCode()); + public void finish(AccessLogInfo accessLog, boolean success, Long beginTime, AccessLogServerInfo serverInfo) { + accessLog.setStatusCode(serverInfo.getStatusCode()); if (!success) { - accessLogInfo.setStatusCode("500"); + accessLog.setStatusCode("500"); } long now = SystemClock.now(); - accessLogInfo.setTimestamp(now); - accessLogInfo.setRequestTime(now - beginTime); - accessLogInfo.setCpuElapsedTime(System.nanoTime() - accessLogInfo.getBeginCpuTime()); - accessLogInfo.setResponseSize(serverInfo.getResponseBufferSize()); - accessLogInfo.setMatchUrl(serverInfo.getMatchURL()); + accessLog.setTimestamp(now); + accessLog.setRequestTime(now - beginTime); + accessLog.setCpuElapsedTime(System.nanoTime() - accessLog.getBeginCpuTime()); + accessLog.setResponseSize(serverInfo.getResponseBufferSize()); + accessLog.setMatchUrl(serverInfo.getMatchURL()); } } diff --git a/plugin-api/src/main/java/io/opentelemetry/sdk/resources/EaseAgentResource.java b/plugin-api/src/main/java/io/opentelemetry/sdk/resources/EaseAgentResource.java new file mode 100644 index 000000000..e55bffde5 --- /dev/null +++ b/plugin-api/src/main/java/io/opentelemetry/sdk/resources/EaseAgentResource.java @@ -0,0 +1,83 @@ +package io.opentelemetry.sdk.resources; + +import com.megaease.easeagent.plugin.api.config.ChangeItem; +import com.megaease.easeagent.plugin.api.config.ConfigChangeListener; +import com.megaease.easeagent.plugin.api.otlp.common.AgentAttributes; +import com.megaease.easeagent.plugin.api.otlp.common.SemanticKey; +import com.megaease.easeagent.plugin.bridge.EaseAgent; +import io.opentelemetry.api.common.Attributes; +import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; +import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAMESPACE; + +import javax.annotation.Nullable; +import java.util.List; + +public class EaseAgentResource extends Resource implements ConfigChangeListener { + static volatile EaseAgentResource agentResource = null; + + private EaseAgentResource() { + super(); + this.system = EaseAgent.getConfig("system", "demo-system"); + this.service = EaseAgent.getConfig("name", "demo-service"); + EaseAgent.getConfig().addChangeListener(this); + + this.resource = Resource.getDefault() + .merge(Resource.create( + AgentAttributes.builder() + .put(SERVICE_NAME, this.service) + .put(SERVICE_NAMESPACE, this.system) + .build())); + } + + private final Resource resource; + private String service; + private String system; + + public String getService() { + return this.service; + } + + public String getSystem() { + return this.system; + } + + public static EaseAgentResource getResource() { + if (agentResource == null) { + synchronized (EaseAgentResource.class) { + if (agentResource == null) { + agentResource = new EaseAgentResource(); + } + } + } + + return agentResource; + } + + @Nullable + @Override + public String getSchemaUrl() { + return SemanticKey.SCHEMA_URL; + } + + @Override + public Attributes getAttributes() { + return this.resource.getAttributes(); + } + + @Override + public void onChange(List list) { + list.forEach(change -> { + if (change.getFullName().equals("name")) { + this.service = change.getNewValue(); + } else if (change.getFullName().equals("system")) { + this.system = change.getNewValue(); + } + + this.resource.merge(Resource.create( + Attributes.builder() + .put(SERVICE_NAME, this.service) + .put(SERVICE_NAMESPACE, this.system) + .build())); + }); + } +} diff --git a/plugins/httpservlet/src/main/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptor.java b/plugins/httpservlet/src/main/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptor.java index e7f8808fc..6bc177214 100644 --- a/plugins/httpservlet/src/main/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptor.java +++ b/plugins/httpservlet/src/main/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptor.java @@ -17,20 +17,19 @@ package com.megaease.easeagent.plugin.httpservlet.interceptor; -import com.megaease.easeagent.plugin.interceptor.MethodInfo; import com.megaease.easeagent.plugin.annotation.AdviceTo; import com.megaease.easeagent.plugin.api.Context; -import com.megaease.easeagent.plugin.api.config.IPluginConfig; import com.megaease.easeagent.plugin.api.context.RequestContext; +import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; import com.megaease.easeagent.plugin.api.trace.Span; import com.megaease.easeagent.plugin.bridge.EaseAgent; import com.megaease.easeagent.plugin.enums.Order; import com.megaease.easeagent.plugin.httpservlet.AccessPlugin; import com.megaease.easeagent.plugin.httpservlet.advice.DoFilterPoints; import com.megaease.easeagent.plugin.httpservlet.utils.ServletUtils; +import com.megaease.easeagent.plugin.interceptor.MethodInfo; import com.megaease.easeagent.plugin.tools.metrics.AccessLogServerInfo; import com.megaease.easeagent.plugin.tools.metrics.HttpLog; -import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -41,10 +40,6 @@ public class ServletHttpLogInterceptor extends BaseServletInterceptor { private static final String AFTER_MARK = ServletHttpLogInterceptor.class.getName() + "$AfterMark"; private final HttpLog httpLog = new HttpLog(); - @Override - public void init(IPluginConfig config, String className, String methodName, String methodDescriptor) { - } - public AccessLogServerInfo serverInfo(HttpServletRequest request, HttpServletResponse response) { ServletAccessLogServerInfo serverInfo = (ServletAccessLogServerInfo) request.getAttribute(ServletAccessLogServerInfo.class.getName()); if (serverInfo == null) { @@ -82,8 +77,8 @@ public void doBefore(MethodInfo methodInfo, Context context) { Long beginTime = ServletUtils.startTime(httpServletRequest); Span span = getSpan(httpServletRequest, context); AccessLogServerInfo serverInfo = this.serverInfo(httpServletRequest, httpServletResponse); - AccessLogInfo accessLogInfo = this.httpLog.prepare(getSystem(), getServiceName(), beginTime, span, serverInfo); - httpServletRequest.setAttribute(AccessLogInfo.class.getName(), accessLogInfo); + AccessLogInfo accessLog = this.httpLog.prepare(getSystem(), getServiceName(), beginTime, span, serverInfo); + httpServletRequest.setAttribute(AccessLogInfo.class.getName(), accessLog); } @Override @@ -94,20 +89,19 @@ String getAfterMark() { @Override void internalAfter(Throwable throwable, String key, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, long start) { Long beginTime = ServletUtils.startTime(httpServletRequest); - AccessLogInfo accessLogInfo = (AccessLogInfo) httpServletRequest.getAttribute(AccessLogInfo.class.getName()); + AccessLogInfo accessLog = (AccessLogInfo) httpServletRequest.getAttribute(AccessLogInfo.class.getName()); AccessLogServerInfo serverInfo = this.serverInfo(httpServletRequest, httpServletResponse); - this.httpLog.finish(accessLogInfo, throwable == null, beginTime, serverInfo); - EaseAgent.agentReport.report(accessLogInfo); + this.httpLog.finish(accessLog, throwable == null, beginTime, serverInfo); + EaseAgent.agentReport.report(accessLog); } @Override public String getType() { - // xxx change to LOG? - return Order.METRIC.getName(); + return Order.LOG.getName(); } @Override public int order() { - return Order.TRACING.getOrder(); + return Order.LOG.getOrder(); } } diff --git a/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletAccessLogServerInfoTest.java b/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletAccessLogInfoServerInfoTest.java similarity index 98% rename from plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletAccessLogServerInfoTest.java rename to plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletAccessLogInfoServerInfoTest.java index a296d1843..ff6911a86 100644 --- a/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletAccessLogServerInfoTest.java +++ b/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletAccessLogInfoServerInfoTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.*; -public class ServletAccessLogServerInfoTest { +public class ServletAccessLogInfoServerInfoTest { private ServletAccessLogServerInfo loadMock() { ServletAccessLogServerInfo servletAccessLogServerInfo = new ServletAccessLogServerInfo(); diff --git a/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptorTest.java b/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptorTest.java index 1ecfc36c0..d13806348 100644 --- a/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptorTest.java +++ b/plugins/httpservlet/src/test/java/com/megaease/easeagent/plugin/httpservlet/interceptor/ServletHttpLogInterceptorTest.java @@ -57,19 +57,19 @@ public void doBefore() throws JsonProcessingException { internalAfter(); } - public void verify(AccessLogInfo accessLogInfo, long startTime) { - assertEquals(EaseAgent.getConfig("system"), accessLogInfo.getSystem()); - assertEquals(EaseAgent.getConfig("name"), accessLogInfo.getService()); - assertEquals(HostAddress.localhost(), accessLogInfo.getHostName()); - assertEquals(HostAddress.getHostIpv4(), accessLogInfo.getHostIpv4()); - assertEquals(TestConst.METHOD + " " + TestConst.URL, accessLogInfo.getUrl()); - assertEquals(TestConst.METHOD, accessLogInfo.getMethod()); - assertEquals(TestConst.FORWARDED_VALUE, accessLogInfo.getHeaders().get(TestConst.FORWARDED_NAME)); - assertEquals(startTime, accessLogInfo.getBeginTime()); - assertEquals("10", accessLogInfo.getQueries().get("q1")); - assertEquals("testq", accessLogInfo.getQueries().get("q2")); - assertEquals(TestConst.FORWARDED_VALUE, accessLogInfo.getClientIP()); - assertTrue(accessLogInfo.getBeginCpuTime() > 0); + public void verify(AccessLogInfo accessLog, long startTime) { + assertEquals(EaseAgent.getConfig("system"), accessLog.getSystem()); + assertEquals(EaseAgent.getConfig("name"), accessLog.getService()); + assertEquals(HostAddress.localhost(), accessLog.getHostName()); + assertEquals(HostAddress.getHostIpv4(), accessLog.getHostIpv4()); + assertEquals(TestConst.METHOD + " " + TestConst.URL, accessLog.getUrl()); + assertEquals(TestConst.METHOD, accessLog.getMethod()); + assertEquals(TestConst.FORWARDED_VALUE, accessLog.getHeaders().get(TestConst.FORWARDED_NAME)); + assertEquals(startTime, accessLog.getBeginTime()); + assertEquals("10", accessLog.getQueries().get("q1")); + assertEquals("testq", accessLog.getQueries().get("q2")); + assertEquals(TestConst.FORWARDED_VALUE, accessLog.getClientIP()); + assertTrue(accessLog.getBeginCpuTime() > 0); } @Test @@ -99,9 +99,9 @@ public void internalAfter() throws JsonProcessingException { Object requestInfoO = httpServletRequest.getAttribute(AccessLogInfo.class.getName()); assertNotNull(requestInfoO); assertTrue(requestInfoO instanceof AccessLogInfo); - AccessLogInfo accessLogInfo = (AccessLogInfo) requestInfoO; + AccessLogInfo accessLog = (AccessLogInfo) requestInfoO; long start = (long) httpServletRequest.getAttribute(ServletUtils.START_TIME); - verify(accessLogInfo, start); + verify(accessLog, start); LastJsonReporter lastJsonReporter = MockEaseAgent.lastMetricJsonReporter(stringObjectMap -> { Object type = stringObjectMap.get("type"); return type instanceof String && "access-log".equals(type); @@ -128,6 +128,6 @@ public void internalAfter() throws JsonProcessingException { @Test public void getType() { ServletHttpLogInterceptor servletHttpLogInterceptor = new ServletHttpLogInterceptor(); - assertEquals(Order.METRIC.getName(), servletHttpLogInterceptor.getType()); + assertEquals(Order.LOG.getName(), servletHttpLogInterceptor.getType()); } } diff --git a/plugins/kafka/src/test/java/com/megaease/easeagent/plugin/kafka/interceptor/metric/KafkaMessageListenerMetricInterceptorTest.java b/plugins/kafka/src/test/java/com/megaease/easeagent/plugin/kafka/interceptor/metric/KafkaMessageListenerMetricInterceptorTest.java index 9097e76b9..cd37a8d5e 100644 --- a/plugins/kafka/src/test/java/com/megaease/easeagent/plugin/kafka/interceptor/metric/KafkaMessageListenerMetricInterceptorTest.java +++ b/plugins/kafka/src/test/java/com/megaease/easeagent/plugin/kafka/interceptor/metric/KafkaMessageListenerMetricInterceptorTest.java @@ -40,7 +40,7 @@ public class KafkaMessageListenerMetricInterceptorTest { public void init() { KafkaMessageListenerMetricInterceptor interceptor = new KafkaMessageListenerMetricInterceptor(); KafkaMetricTest.init(interceptor); - assertNotNull(KafkaConsumerMetricInterceptor.getKafkaMetric()); + assertNotNull(KafkaMessageListenerMetricInterceptor.getKafkaMetric()); } @Test diff --git a/plugins/log4j2-log-plugin/pom.xml b/plugins/log4j2-log-plugin/pom.xml new file mode 100644 index 000000000..9f1f53e0d --- /dev/null +++ b/plugins/log4j2-log-plugin/pom.xml @@ -0,0 +1,95 @@ + + + + + 4.0.0 + + + com.megaease.easeagent + plugins + 2.0.0 + + + + log4j2-log-plugin + + jar + + + + com.megaease.easeagent + plugin-api + provided + + + junit + junit + test + + + com.megaease.easeagent + plugin-api-mock + ${project.version} + test + + + org.apache.logging.log4j + log4j-core + 2.17.1 + provided + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.maven.plugins + maven-resources-plugin + 3.0.1 + + + + + + diff --git a/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/Log4j2Plugin.java b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/Log4j2Plugin.java new file mode 100644 index 000000000..def15f15e --- /dev/null +++ b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/Log4j2Plugin.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.log4j2; + +import com.megaease.easeagent.plugin.AgentPlugin; +import com.megaease.easeagent.plugin.api.config.ConfigConst; +import com.megaease.easeagent.plugin.enums.Order; + +public class Log4j2Plugin implements AgentPlugin { + @Override + public String getNamespace() { + return "log4j2"; + } + + @Override + public String getDomain() { + return ConfigConst.OBSERVABILITY; + } + + @Override + public int order() { + return Order.LOG.getOrder(); + } +} diff --git a/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/interceptor/Log4j2AppenderInterceptor.java b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/interceptor/Log4j2AppenderInterceptor.java new file mode 100644 index 000000000..dbc37a00e --- /dev/null +++ b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/interceptor/Log4j2AppenderInterceptor.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.log4j2.interceptor; + +import com.megaease.easeagent.log4j2.points.AbstractLoggerPoints; +import com.megaease.easeagent.plugin.annotation.AdviceTo; +import com.megaease.easeagent.plugin.api.Context; +import com.megaease.easeagent.plugin.api.config.IPluginConfig; +import com.megaease.easeagent.plugin.api.config.PluginConfigChangeListener; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.api.otlp.common.LogMapper; +import com.megaease.easeagent.plugin.bridge.EaseAgent; +import com.megaease.easeagent.plugin.enums.Order; +import com.megaease.easeagent.plugin.interceptor.MethodInfo; +import com.megaease.easeagent.plugin.interceptor.NonReentrantInterceptor; +import com.megaease.easeagent.plugin.tools.loader.AgentHelperClassLoader; +import com.megaease.easeagent.plugin.utils.common.StringUtils; +import com.megaease.easeagent.plugin.utils.common.WeakConcurrentMap; +import org.apache.logging.log4j.Level; + +import java.lang.reflect.InvocationTargetException; + +@AdviceTo(AbstractLoggerPoints.class) +public class Log4j2AppenderInterceptor implements NonReentrantInterceptor, PluginConfigChangeListener { + static WeakConcurrentMap logMappers = new WeakConcurrentMap<>(); + int collectLevel = Level.INFO.intLevel(); + @Override + public void init(IPluginConfig config, int uniqueIndex) { + String lv = config.getString("level"); + if (StringUtils.isNotEmpty(lv)) { + collectLevel = Level.toLevel(lv, Level.OFF).intLevel(); + } + config.addChangeListener(this); + AgentHelperClassLoader.registryUrls(this.getClass()); + } + + @Override + public void doBefore(MethodInfo methodInfo, Context context) { + ClassLoader appLoader = methodInfo.getInvoker().getClass().getClassLoader(); + LogMapper mapper = logMappers.getIfPresent(appLoader); + + if (mapper == null) { + ClassLoader help = AgentHelperClassLoader.getClassLoader(appLoader, EaseAgent.getAgentClassLoader()); + try { + Class cls = help.loadClass("com.megaease.easeagent.log4j2.log.Log4jLogMapper"); + mapper = (LogMapper) cls.getConstructor().newInstance(); + logMappers.putIfProbablyAbsent(appLoader, mapper); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException + | InvocationTargetException | InstantiationException e) { + return; + } + } + + AgentLogData log = mapper.mapLoggingEvent(methodInfo, this.collectLevel, context.getConfig()); + if (log != null) { + EaseAgent.getAgentReport().report(log); + } + } + + @Override + public String getType() { + return Order.LOG.getName(); + } + + @Override + public int order() { + return Order.LOG.getOrder(); + } + + @Override + public void onChange(IPluginConfig oldConfig, IPluginConfig newConfig) { + String lv = newConfig.getString("level"); + + if (!StringUtils.isEmpty(lv)) { + this.collectLevel = Level.toLevel(lv, Level.OFF).intLevel(); + } else { + this.collectLevel = Level.OFF.intLevel(); + } + } +} diff --git a/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/log/Log4jLogMapper.java b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/log/Log4jLogMapper.java new file mode 100644 index 000000000..955cb2637 --- /dev/null +++ b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/log/Log4jLogMapper.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.megaease.easeagent.log4j2.log; + +import com.megaease.easeagent.plugin.api.config.IPluginConfig; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogDataImpl; +import com.megaease.easeagent.plugin.api.otlp.common.LogMapper; +import com.megaease.easeagent.plugin.interceptor.MethodInfo; +import com.megaease.easeagent.plugin.utils.SystemClock; +import io.opentelemetry.sdk.logs.data.Severity; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.ThreadContext; +import org.apache.logging.log4j.message.MapMessage; +import org.apache.logging.log4j.message.Message; + +import java.util.Map; + +/** + * reference to Opentelemetry instrumentation + */ +public class Log4jLogMapper implements LogMapper { + private static final String SPECIAL_MAP_MESSAGE_ATTRIBUTE = "message"; + + public AgentLogData mapLoggingEvent(MethodInfo logInfo, int levelInt, IPluginConfig config) { + Object[] args = logInfo.getArgs(); + + if (args == null) { + return null; + } + + AgentLogDataImpl.Builder builder = AgentLogDataImpl.builder(); + + for (int i = 0; i < args.length; i++) { + switch (i) { + case 0: + // level + Level level = (Level)args[i]; + if (level.intLevel() > levelInt) { + return null; + } + builder.severity(levelToSeverity(level)); + builder.severityText(level.name()); + break; + case 4: + // message + Message message = (Message)args[i]; + if (!(message instanceof MapMessage)) { + builder.body(message.getFormattedMessage()); + } else { + MapMessage mapMessage = (MapMessage) message; + + String body = mapMessage.getFormat(); + boolean checkSpecialMapMessageAttribute = (body == null || body.isEmpty()); + if (checkSpecialMapMessageAttribute) { + body = mapMessage.get(SPECIAL_MAP_MESSAGE_ATTRIBUTE); + } + if (body != null && !body.isEmpty()) { + builder.body(body); + } + } + break; + case 5: + // throwable + Throwable throwable = (Throwable) args[5]; + if (throwable != null) { + builder.throwable(throwable); + } + break; + default: + break; + } + } + + // logger + Logger logger = (Logger)logInfo.getInvoker(); + if (logger.getName() == null || logger.getName().isEmpty()) { + builder.logger("ROOT"); + } else { + builder.logger(logger.getName()); + } + + // thread + builder.thread(Thread.currentThread()); + builder.epochMills(SystemClock.now()); + + // MDC + Map contextData = ThreadContext.getImmutableContext(); + builder.contextData(config.getStringList(LogMapper.MDC_KEYS), contextData); + + // span context + builder.spanContext(); + + return builder.build(); + } + + private static Severity levelToSeverity(Level level) { + switch (level.getStandardLevel()) { + case ALL: + case TRACE: + return Severity.TRACE; + case DEBUG: + return Severity.DEBUG; + case INFO: + return Severity.INFO; + case WARN: + return Severity.WARN; + case ERROR: + return Severity.ERROR; + case FATAL: + return Severity.FATAL; + case OFF: + return Severity.UNDEFINED_SEVERITY_NUMBER; + } + return Severity.UNDEFINED_SEVERITY_NUMBER; + } +} diff --git a/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/points/AbstractLoggerPoints.java b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/points/AbstractLoggerPoints.java new file mode 100644 index 000000000..d7e041d8a --- /dev/null +++ b/plugins/log4j2-log-plugin/src/main/java/com/megaease/easeagent/log4j2/points/AbstractLoggerPoints.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.megaease.easeagent.log4j2.points; + +import com.megaease.easeagent.plugin.Points; +import com.megaease.easeagent.plugin.matcher.ClassMatcher; +import com.megaease.easeagent.plugin.matcher.IClassMatcher; +import com.megaease.easeagent.plugin.matcher.IMethodMatcher; +import com.megaease.easeagent.plugin.matcher.MethodMatcher; +import com.megaease.easeagent.plugin.matcher.loader.ClassLoaderMatcher; +import com.megaease.easeagent.plugin.matcher.loader.IClassLoaderMatcher; + +import java.util.Set; + +public class AbstractLoggerPoints implements Points { + @Override + public IClassMatcher getClassMatcher() { + return ClassMatcher.builder() + .hasSuperClass("org.apache.logging.log4j.spi.AbstractLogger") + .build(); + } + + @Override + public Set getMethodMatcher() { + return MethodMatcher.builder() + .named("log") + .argsLength(6) + .arg(0, "org.apache.logging.log4j.Level") + .arg(1, "org.apache.logging.log4j.Marker") + .arg(2, "java.lang.String") + .arg(3, "java.lang.StackTraceElement") + .arg(4, "org.apache.logging.log4j.message.Message") + .arg(5, "java.lang.Throwable") + .build() + .toSet(); + } + + /** + * Do not match classes loaded by Agent classloader + */ + @Override + public IClassLoaderMatcher getClassLoaderMatcher() { + return ClassLoaderMatcher.AGENT.negate(); + } +} diff --git a/plugins/logback/pom.xml b/plugins/logback/pom.xml new file mode 100644 index 000000000..cf2bb2336 --- /dev/null +++ b/plugins/logback/pom.xml @@ -0,0 +1,95 @@ + + + + + 4.0.0 + + + com.megaease.easeagent + plugins + 2.0.0 + + + + logback + + jar + + + + com.megaease.easeagent + plugin-api + provided + + + junit + junit + test + + + com.megaease.easeagent + plugin-api-mock + ${project.version} + test + + + ch.qos.logback + logback-classic + 1.2.11 + provided + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.maven.plugins + maven-resources-plugin + 3.0.1 + + + + + + diff --git a/plugins/logback/src/main/java/com/megaease/easeagent/logback/LogbackPlugin.java b/plugins/logback/src/main/java/com/megaease/easeagent/logback/LogbackPlugin.java new file mode 100644 index 000000000..e1f4803a3 --- /dev/null +++ b/plugins/logback/src/main/java/com/megaease/easeagent/logback/LogbackPlugin.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.logback; + +import com.megaease.easeagent.plugin.AgentPlugin; +import com.megaease.easeagent.plugin.api.config.ConfigConst; + +public class LogbackPlugin implements AgentPlugin { + @Override + public String getNamespace() { + return "logback"; + } + + @Override + public String getDomain() { + return ConfigConst.OBSERVABILITY; + } +} diff --git a/plugins/logback/src/main/java/com/megaease/easeagent/logback/interceptor/LogbackAppenderInterceptor.java b/plugins/logback/src/main/java/com/megaease/easeagent/logback/interceptor/LogbackAppenderInterceptor.java new file mode 100644 index 000000000..957b891f8 --- /dev/null +++ b/plugins/logback/src/main/java/com/megaease/easeagent/logback/interceptor/LogbackAppenderInterceptor.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.logback.interceptor; + +import ch.qos.logback.classic.Level; +import com.megaease.easeagent.logback.log.LogbackLogMapper; +import com.megaease.easeagent.logback.points.LoggerPoints; +import com.megaease.easeagent.plugin.annotation.AdviceTo; +import com.megaease.easeagent.plugin.api.Context; +import com.megaease.easeagent.plugin.api.config.IPluginConfig; +import com.megaease.easeagent.plugin.api.config.PluginConfigChangeListener; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.bridge.EaseAgent; +import com.megaease.easeagent.plugin.enums.Order; +import com.megaease.easeagent.plugin.interceptor.MethodInfo; +import com.megaease.easeagent.plugin.interceptor.NonReentrantInterceptor; +import com.megaease.easeagent.plugin.utils.common.StringUtils; + +@AdviceTo(LoggerPoints.class) +public class LogbackAppenderInterceptor implements NonReentrantInterceptor, PluginConfigChangeListener { + int collectLevel = Level.INFO.levelInt; + + @Override + public void init(IPluginConfig config, int uniqueIndex) { + String lv = config.getString("level"); + if (StringUtils.isNotEmpty(lv)) { + collectLevel = Level.toLevel(lv, Level.OFF).levelInt; + } + config.addChangeListener(this); + } + + @Override + public void doBefore(MethodInfo methodInfo, Context context) { + AgentLogData log = LogbackLogMapper.INSTANCE.mapLoggingEvent(methodInfo, collectLevel, context.getConfig()); + if (log == null) { + return; + } + EaseAgent.getAgentReport().report(log); + } + + @Override + public String getType() { + return Order.LOG.getName(); + } + + @Override + public int order() { + return Order.LOG.getOrder(); + } + + @Override + public void onChange(IPluginConfig oldConfig, IPluginConfig newConfig) { + String lv = newConfig.getString("level"); + if (StringUtils.isNotEmpty(lv)) { + collectLevel = Level.toLevel(lv, Level.OFF).levelInt; + } + } +} diff --git a/plugins/logback/src/main/java/com/megaease/easeagent/logback/log/LogbackLogMapper.java b/plugins/logback/src/main/java/com/megaease/easeagent/logback/log/LogbackLogMapper.java new file mode 100644 index 000000000..57bd7c298 --- /dev/null +++ b/plugins/logback/src/main/java/com/megaease/easeagent/logback/log/LogbackLogMapper.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.logback.log; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxy; +import com.megaease.easeagent.plugin.api.config.IPluginConfig; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogDataImpl; +import com.megaease.easeagent.plugin.api.otlp.common.LogMapper; +import com.megaease.easeagent.plugin.interceptor.MethodInfo; +import io.opentelemetry.sdk.logs.data.Severity; + +import java.util.Map; + +/** + * reference to opentelemetry logback instrumentation lib + */ +public class LogbackLogMapper implements LogMapper { + public static final LogbackLogMapper INSTANCE = new LogbackLogMapper(); + + @Override + public AgentLogData mapLoggingEvent(MethodInfo methodInfo, int levelInt, IPluginConfig config) { + if (methodInfo.getArgs() == null || methodInfo.getArgs().length < 1) { + return null; + } + + ILoggingEvent loggingEvent = (ILoggingEvent)methodInfo.getArgs()[0]; + Level level = loggingEvent.getLevel(); + if (level == null || level.levelInt < levelInt) { + return null; + } + + AgentLogDataImpl.Builder builder = AgentLogDataImpl.builder(); + // logger + String logger = loggingEvent.getLoggerName(); + if (logger == null || logger.isEmpty()) { + logger = "ROOT"; + } + builder.logger(logger); + // message + String message = loggingEvent.getFormattedMessage(); + if (message != null) { + builder.body(message); + } + + // time + long timestamp = loggingEvent.getTimeStamp(); + builder.epochMills(timestamp); + + // level + builder.severity(levelToSeverity(level)); + builder.severityText(level.levelStr); + + // throwable + Object throwableProxy = loggingEvent.getThrowableProxy(); + Throwable throwable = null; + if (throwableProxy instanceof ThrowableProxy) { + // there is only one other subclass of ch.qos.logback.classic.spi. + // IThrowableProxy and it is only used for logging exceptions over the wire + throwable = ((ThrowableProxy) throwableProxy).getThrowable(); + } + if (throwable != null) { + builder.throwable(throwable); + } + + Thread currentThread = Thread.currentThread(); + builder.thread(currentThread); + + // MDC + Map contextData = loggingEvent.getMDCPropertyMap(); + builder.contextData(config.getStringList(LogMapper.MDC_KEYS), contextData); + + // span context + builder.spanContext(); + + return builder.build(); + } + + private static Severity levelToSeverity(Level level) { + switch (level.levelInt) { + case Level.ALL_INT: + case Level.TRACE_INT: + return Severity.TRACE; + case Level.DEBUG_INT: + return Severity.DEBUG; + case Level.INFO_INT: + return Severity.INFO; + case Level.WARN_INT: + return Severity.WARN; + case Level.ERROR_INT: + return Severity.ERROR; + case Level.OFF_INT: + default: + return Severity.UNDEFINED_SEVERITY_NUMBER; + } + } +} diff --git a/plugins/logback/src/main/java/com/megaease/easeagent/logback/points/LoggerPoints.java b/plugins/logback/src/main/java/com/megaease/easeagent/logback/points/LoggerPoints.java new file mode 100644 index 000000000..0fbc0c882 --- /dev/null +++ b/plugins/logback/src/main/java/com/megaease/easeagent/logback/points/LoggerPoints.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.megaease.easeagent.logback.points; + +import com.megaease.easeagent.plugin.Points; +import com.megaease.easeagent.plugin.matcher.ClassMatcher; +import com.megaease.easeagent.plugin.matcher.IClassMatcher; +import com.megaease.easeagent.plugin.matcher.IMethodMatcher; +import com.megaease.easeagent.plugin.matcher.MethodMatcher; +import com.megaease.easeagent.plugin.matcher.loader.ClassLoaderMatcher; +import com.megaease.easeagent.plugin.matcher.loader.IClassLoaderMatcher; + +import java.util.Set; + +public class LoggerPoints implements Points { + @Override + public IClassMatcher getClassMatcher() { + return ClassMatcher.builder() + .hasClassName("ch.qos.logback.classic.Logger") + .build(); + } + + @Override + public Set getMethodMatcher() { + return MethodMatcher.builder() + .named("callAppenders") + .argsLength(1) + .arg(0, "ch.qos.logback.classic.spi.ILoggingEvent") + .build().toSet(); + } + + /** + * Do not match classes loaded by Agent classloader + */ + @Override + public IClassLoaderMatcher getClassLoaderMatcher() { + return ClassLoaderMatcher.AGENT.negate(); + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index b4f0481ea..8f7200bd5 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -47,6 +47,8 @@ elasticsearch healthy mongodb + logback + log4j2-log-plugin diff --git a/plugins/spring-gateway/src/main/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInterceptor.java b/plugins/spring-gateway/src/main/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInterceptor.java index 2d87f9741..701ce02f2 100644 --- a/plugins/spring-gateway/src/main/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInterceptor.java +++ b/plugins/spring-gateway/src/main/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInterceptor.java @@ -19,10 +19,9 @@ import com.megaease.easeagent.plugin.annotation.AdviceTo; import com.megaease.easeagent.plugin.api.Context; -import com.megaease.easeagent.plugin.api.Reporter; -import com.megaease.easeagent.plugin.api.config.IPluginConfig; import com.megaease.easeagent.plugin.api.context.AsyncContext; import com.megaease.easeagent.plugin.api.context.RequestContext; +import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; import com.megaease.easeagent.plugin.api.trace.Span; import com.megaease.easeagent.plugin.bridge.EaseAgent; import com.megaease.easeagent.plugin.enums.Order; @@ -30,7 +29,6 @@ import com.megaease.easeagent.plugin.interceptor.MethodInfo; import com.megaease.easeagent.plugin.tools.metrics.AccessLogServerInfo; import com.megaease.easeagent.plugin.tools.metrics.HttpLog; -import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; import easeagent.plugin.spring.gateway.AccessPlugin; import easeagent.plugin.spring.gateway.advice.AgentGlobalFilterAdvice; import easeagent.plugin.spring.gateway.interceptor.GatewayCons; @@ -44,22 +42,16 @@ @AdviceTo(value = AgentGlobalFilterAdvice.class, plugin = AccessPlugin.class) public class GatewayAccessLogInterceptor implements Interceptor { private static final Object START_TIME = new Object(); - private static Reporter reportConsumer; private final HttpLog httpLog = new HttpLog(); - @Override - public void init(IPluginConfig config, String className, String methodName, String methodDescriptor) { - reportConsumer = EaseAgent.metricReporter(config); - } - @Override public void before(MethodInfo methodInfo, Context context) { ServerWebExchange exchange = (ServerWebExchange) methodInfo.getArgs()[0]; AccessLogServerInfo serverInfo = this.serverInfo(exchange); Long beginTime = startTime(context, START_TIME); - AccessLogInfo accessLogInfo = this.httpLog.prepare(getSystem(), + AccessLogInfo accessLog = this.httpLog.prepare(getSystem(), getServiceName(), beginTime, getSpan(exchange), serverInfo); - exchange.getAttributes().put(AccessLogInfo.class.getName(), accessLogInfo); + exchange.getAttributes().put(AccessLogInfo.class.getName(), accessLog); } @Override @@ -84,18 +76,14 @@ Span getSpan(ServerWebExchange exchange) { private void finishCallback(MethodInfo methodInfo, AsyncContext ctx) { ServerWebExchange exchange = (ServerWebExchange) methodInfo.getArgs()[0]; - AccessLogInfo accessLogInfo = exchange.getAttribute(AccessLogInfo.class.getName()); - if (accessLogInfo == null) { + AccessLogInfo accessLog = exchange.getAttribute(AccessLogInfo.class.getName()); + if (accessLog == null) { return; } Long beginTime = ctx.get(START_TIME); AccessLogServerInfo serverInfo = this.serverInfo(exchange); - this.httpLog.finish(accessLogInfo, methodInfo.isSuccess(), beginTime, serverInfo); - EaseAgent.getAgentReport().report(accessLogInfo); - /* - String logString = this.httpLog.getLogString(accessLogInfo, methodInfo.isSuccess(), beginTime, serverInfo); - reportConsumer.report(logString); - */ + this.httpLog.finish(accessLog, methodInfo.isSuccess(), beginTime, serverInfo); + EaseAgent.getAgentReport().report(accessLog); } AccessLogServerInfo serverInfo(ServerWebExchange exchange) { @@ -114,11 +102,11 @@ String getServiceName() { @Override public String getType() { - return Order.METRIC.getName(); + return Order.LOG.getName(); } @Override public int order() { - return Order.METRIC.getOrder(); + return Order.LOG.getOrder(); } } diff --git a/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInterceptorTest.java b/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInfoInterceptorTest.java similarity index 71% rename from plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInterceptorTest.java rename to plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInfoInterceptorTest.java index 9092e2f2f..cb37f763b 100644 --- a/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInterceptorTest.java +++ b/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/GatewayAccessLogInfoInterceptorTest.java @@ -46,16 +46,9 @@ import static org.junit.Assert.*; @RunWith(EaseAgentJunit4ClassRunner.class) -public class GatewayAccessLogInterceptorTest { +public class GatewayAccessLogInfoInterceptorTest { private Object startTime = AgentFieldReflectAccessor.getStaticFieldValue(GatewayAccessLogInterceptor.class, "START_TIME"); - @Test - public void init() { - GatewayAccessLogInterceptor interceptor = new GatewayAccessLogInterceptor(); - InterceptorTestUtils.init(interceptor, new AccessPlugin()); - assertNotNull(AgentFieldReflectAccessor.getFieldValue(interceptor, "reportConsumer")); - } - @Test public void before() { GatewayAccessLogInterceptor interceptor = new GatewayAccessLogInterceptor(); @@ -63,12 +56,12 @@ public void before() { MockServerWebExchange mockServerWebExchange = TestServerWebExchangeUtils.mockServerWebExchange(); MethodInfo methodInfo = MethodInfo.builder().args(new Object[]{mockServerWebExchange}).build(); interceptor.before(methodInfo, context); - AccessLogInfo accessLogInfo = (AccessLogInfo) mockServerWebExchange.getAttributes().get(AccessLogInfo.class.getName()); - assertNotNull(accessLogInfo); - verify(accessLogInfo, TimeUtils.startTime(context, startTime)); - assertNull(accessLogInfo.getTraceId()); - assertNull(accessLogInfo.getSpanId()); - assertNull(accessLogInfo.getParentSpanId()); + AccessLogInfo accessLog = (AccessLogInfo) mockServerWebExchange.getAttributes().get(AccessLogInfo.class.getName()); + assertNotNull(accessLog); + verify(accessLog, TimeUtils.startTime(context, startTime)); + assertNull(accessLog.getTraceId()); + assertNull(accessLog.getSpanId()); + assertNull(accessLog.getParentSpanId()); mockServerWebExchange = TestServerWebExchangeUtils.mockServerWebExchange(); methodInfo = MethodInfo.builder().args(new Object[]{mockServerWebExchange}).build(); @@ -77,28 +70,28 @@ public void before() { Span span = requestContext.span(); try (Scope ignored = requestContext.scope()) { interceptor.before(methodInfo, context); - accessLogInfo = (AccessLogInfo) mockServerWebExchange.getAttributes().get(AccessLogInfo.class.getName()); - assertNotNull(accessLogInfo); - verify(accessLogInfo, TimeUtils.startTime(context, startTime)); - assertEquals(span.traceIdString(), accessLogInfo.getTraceId()); - assertEquals(span.spanIdString(), accessLogInfo.getSpanId()); - assertEquals(span.parentIdString(), accessLogInfo.getParentSpanId()); + accessLog = (AccessLogInfo) mockServerWebExchange.getAttributes().get(AccessLogInfo.class.getName()); + assertNotNull(accessLog); + verify(accessLog, TimeUtils.startTime(context, startTime)); + assertEquals(span.traceIdString(), accessLog.getTraceId()); + assertEquals(span.spanIdString(), accessLog.getSpanId()); + assertEquals(span.parentIdString(), accessLog.getParentSpanId()); } } - public void verify(AccessLogInfo accessLogInfo, long startTime) { - assertEquals("test-gateway-system", accessLogInfo.getSystem()); - assertEquals("test-gateway-service", accessLogInfo.getService()); - assertEquals(HostAddress.localhost(), accessLogInfo.getHostName()); - assertEquals(HostAddress.getHostIpv4(), accessLogInfo.getHostIpv4()); - assertEquals("GET http://192.168.0.12:8080/test?a=b", accessLogInfo.getUrl()); - assertEquals("GET", accessLogInfo.getMethod()); - assertEquals(TestConst.FORWARDED_VALUE, accessLogInfo.getHeaders().get(TestConst.FORWARDED_NAME)); - assertEquals(startTime, accessLogInfo.getBeginTime()); - assertEquals("b", accessLogInfo.getQueries().get("a")); - assertEquals(TestConst.FORWARDED_VALUE, accessLogInfo.getClientIP()); - assertTrue(accessLogInfo.getBeginCpuTime() > 0); + public void verify(AccessLogInfo accessLog, long startTime) { + assertEquals("test-gateway-system", accessLog.getSystem()); + assertEquals("test-gateway-service", accessLog.getService()); + assertEquals(HostAddress.localhost(), accessLog.getHostName()); + assertEquals(HostAddress.getHostIpv4(), accessLog.getHostIpv4()); + assertEquals("GET http://192.168.0.12:8080/test?a=b", accessLog.getUrl()); + assertEquals("GET", accessLog.getMethod()); + assertEquals(TestConst.FORWARDED_VALUE, accessLog.getHeaders().get(TestConst.FORWARDED_NAME)); + assertEquals(startTime, accessLog.getBeginTime()); + assertEquals("b", accessLog.getQueries().get("a")); + assertEquals(TestConst.FORWARDED_VALUE, accessLog.getClientIP()); + assertTrue(accessLog.getBeginCpuTime() > 0); } @Test @@ -121,8 +114,8 @@ public void after() throws InterruptedException { Thread thread = new Thread(() -> agentMono.getFinish().accept(agentMono.getMethodInfo(), agentMono.getAsyncContext())); thread.start(); thread.join(); - AccessLogInfo accessLogInfo = MockEaseAgent.getLastLog(); - verify(accessLogInfo, start); + AccessLogInfo accessLog = MockEaseAgent.getLastLog(); + verify(accessLog, start); } @Test @@ -147,12 +140,12 @@ public void getServiceName() { @Test public void getType() { GatewayAccessLogInterceptor interceptor = new GatewayAccessLogInterceptor(); - assertEquals(ConfigConst.PluginID.METRIC, interceptor.getType()); + assertEquals(ConfigConst.PluginID.LOG, interceptor.getType()); } @Test public void order() { GatewayAccessLogInterceptor interceptor = new GatewayAccessLogInterceptor(); - assertEquals(Order.METRIC.getOrder(), interceptor.order()); + assertEquals(Order.LOG.getOrder(), interceptor.order()); } } diff --git a/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/SpringGatewayAccessLogServerInfoTest.java b/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/SpringGatewayAccessLogInfoServerInfoTest.java similarity index 98% rename from plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/SpringGatewayAccessLogServerInfoTest.java rename to plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/SpringGatewayAccessLogInfoServerInfoTest.java index e399a4bfc..a8f514122 100644 --- a/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/SpringGatewayAccessLogServerInfoTest.java +++ b/plugins/spring-gateway/src/test/java/easeagent/plugin/spring/gateway/interceptor/metric/log/SpringGatewayAccessLogInfoServerInfoTest.java @@ -32,7 +32,7 @@ import static org.junit.Assert.*; -public class SpringGatewayAccessLogServerInfoTest { +public class SpringGatewayAccessLogInfoServerInfoTest { @Test public void load() { diff --git a/pom.xml b/pom.xml index 09de82cb0..ecf550a4f 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,7 @@ 3.1.0 5.13.3 1.12.0 + 1.12.0-alpha 2.8.0 2.15.0 1.1.0 @@ -212,6 +213,24 @@ ${version.otel} + + io.opentelemetry + opentelemetry-sdk-common + ${version.otel} + + + + io.opentelemetry + opentelemetry-semconv + ${version.otel-alpha} + + + + io.opentelemetry + opentelemetry-sdk-logs + ${version.otel-alpha} + + io.zipkin.brave brave diff --git a/report/pom.xml b/report/pom.xml index 5403a5415..468996976 100644 --- a/report/pom.xml +++ b/report/pom.xml @@ -117,6 +117,10 @@ com.fasterxml.jackson.core jackson-databind + + io.opentelemetry + opentelemetry-sdk-logs + diff --git a/report/src/main/java/com/megaease/easeagent/report/DefaultAgentReport.java b/report/src/main/java/com/megaease/easeagent/report/DefaultAgentReport.java index 9bffc0e7c..f7f835fba 100644 --- a/report/src/main/java/com/megaease/easeagent/report/DefaultAgentReport.java +++ b/report/src/main/java/com/megaease/easeagent/report/DefaultAgentReport.java @@ -23,10 +23,12 @@ import com.megaease.easeagent.plugin.api.config.Config; import com.megaease.easeagent.plugin.api.config.ConfigChangeListener; import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; import com.megaease.easeagent.plugin.report.AgentReport; import com.megaease.easeagent.plugin.report.metric.MetricReporterFactory; import com.megaease.easeagent.plugin.report.tracing.ReportSpan; -import com.megaease.easeagent.report.async.log.LogReporter; +import com.megaease.easeagent.report.async.log.AccessLogReporter; +import com.megaease.easeagent.report.async.log.ApplicationLogReporter; import com.megaease.easeagent.report.metric.MetricReporterFactoryImpl; import com.megaease.easeagent.report.plugin.ReporterLoader; import com.megaease.easeagent.report.trace.TraceReport; @@ -39,7 +41,8 @@ public class DefaultAgentReport implements AgentReport, ConfigChangeListener { private final TraceReport traceReport; private final MetricReporterFactory metricReporterFactory; - private final LogReporter logReporter; + private final AccessLogReporter accessLogReporter; + private final ApplicationLogReporter appLogReporter; private final Config config; private final Config reportConfig; @@ -47,7 +50,8 @@ public class DefaultAgentReport implements AgentReport, ConfigChangeListener { this.config = config; this.reportConfig = new Configs(ReportConfigAdapter.extractReporterConfig(config)); this.traceReport = new TraceReport(this.reportConfig); - this.logReporter = new LogReporter(this.reportConfig); + this.accessLogReporter = new AccessLogReporter(this.reportConfig); + this.appLogReporter = new ApplicationLogReporter(this.reportConfig); this.metricReporterFactory = MetricReporterFactoryImpl.create(this.reportConfig); this.config.addChangeListener(this); } @@ -64,7 +68,12 @@ public void report(ReportSpan span) { @Override public void report(AccessLogInfo log) { - this.logReporter.report(log); + this.accessLogReporter.report(log); + } + + @Override + public void report(AgentLogData log) { + this.appLogReporter.report(log); } @Override diff --git a/report/src/main/java/com/megaease/easeagent/report/async/log/AccessLogReporter.java b/report/src/main/java/com/megaease/easeagent/report/async/log/AccessLogReporter.java new file mode 100644 index 000000000..58943b3c5 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/async/log/AccessLogReporter.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.async.log; + +import com.megaease.easeagent.config.ConfigUtils; +import com.megaease.easeagent.config.Configs; +import com.megaease.easeagent.config.report.ReportConfigConst; +import com.megaease.easeagent.plugin.api.config.ChangeItem; +import com.megaease.easeagent.plugin.api.config.Config; +import com.megaease.easeagent.plugin.api.config.ConfigChangeListener; +import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; +import com.megaease.easeagent.plugin.utils.common.StringUtils; +import com.megaease.easeagent.report.async.AsyncProps; +import com.megaease.easeagent.report.async.AsyncReporter; +import com.megaease.easeagent.report.async.DefaultAsyncReporter; +import com.megaease.easeagent.report.plugin.ReporterRegistry; +import com.megaease.easeagent.report.sender.SenderWithEncoder; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static com.megaease.easeagent.config.report.ReportConfigConst.*; + +@SuppressWarnings("unused") +public class AccessLogReporter implements ConfigChangeListener { + Config config; + AsyncReporter asyncReporter; + + public AccessLogReporter(Config configs) { + Map cfg = ConfigUtils.extractByPrefix(configs.getConfigs(), LOG_ACCESS); + cfg.putAll(ConfigUtils.extractByPrefix(configs.getConfigs(), OUTPUT_SERVER_V2)); + cfg.putAll(ConfigUtils.extractByPrefix(configs.getConfigs(), LOG_ASYNC)); + + this.config = new Configs(cfg); + configs.addChangeListener(this); + + AsyncProps asyncProperties = new LogAsyncProps(this.config, LOG_ACCESS); + SenderWithEncoder sender = ReporterRegistry.getSender(ReportConfigConst.LOG_ACCESS_SENDER, this.config); + this.asyncReporter = DefaultAsyncReporter.builderAsyncReporter(sender, asyncProperties); + this.asyncReporter.startFlushThread(); + } + + public void report(AccessLogInfo log) { + this.asyncReporter.report(log); + } + + @Override + public void onChange(List list) { + Map changes = filterChanges(list); + if (changes.isEmpty()) { + return; + } + this.config.updateConfigs(changes); + this.refresh(this.config.getConfigs()); + } + + private Map filterChanges(List list) { + Map cfg = new HashMap<>(); + list.stream() + .filter(item -> item.getFullName().startsWith(LOG_ACCESS) + || item.getFullName().startsWith(LOG_ASYNC) + || item.getFullName().startsWith(OUTPUT_SERVER_V2)) + .forEach(one -> cfg.put(one.getFullName(), one.getNewValue())); + return cfg; + } + + public synchronized void refresh(Map cfg) { + String name = cfg.get(LOG_ACCESS_SENDER_NAME); + SenderWithEncoder sender = asyncReporter.getSender(); + if (sender != null) { + if (StringUtils.isNotEmpty(name) && !sender.name().equals(name)) { + try { + sender.close(); + } catch (Exception ignored) { + // ignored + } + sender = ReporterRegistry.getSender(LOG_ACCESS_SENDER, this.config); + asyncReporter.setSender(sender); + } + } else { + sender = ReporterRegistry.getSender(LOG_ACCESS_SENDER, this.config); + asyncReporter.setSender(sender); + } + + AsyncProps asyncProperties = new LogAsyncProps(this.config, LOG_ACCESS); + asyncReporter.closeFlushThread(); + asyncReporter.setPending(asyncProperties.getQueuedMaxItems(), asyncProperties.getQueuedMaxSize()); + asyncReporter.setMessageTimeoutNanos(messageTimeout(asyncProperties.getMessageTimeout())); + asyncReporter.startFlushThread(); // start thread + } + + protected long messageTimeout(long timeout) { + if (timeout < 0) { + timeout = 1000L; + } + return TimeUnit.MILLISECONDS.toNanos(timeout); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/async/log/LogReporter.java b/report/src/main/java/com/megaease/easeagent/report/async/log/ApplicationLogReporter.java similarity index 91% rename from report/src/main/java/com/megaease/easeagent/report/async/log/LogReporter.java rename to report/src/main/java/com/megaease/easeagent/report/async/log/ApplicationLogReporter.java index 6843699ef..59c5f7e26 100644 --- a/report/src/main/java/com/megaease/easeagent/report/async/log/LogReporter.java +++ b/report/src/main/java/com/megaease/easeagent/report/async/log/ApplicationLogReporter.java @@ -23,8 +23,6 @@ import com.megaease.easeagent.plugin.api.config.ChangeItem; import com.megaease.easeagent.plugin.api.config.Config; import com.megaease.easeagent.plugin.api.config.ConfigChangeListener; -import com.megaease.easeagent.plugin.api.logging.AccessLogInfo; -import com.megaease.easeagent.plugin.bridge.EaseAgent; import com.megaease.easeagent.plugin.utils.common.StringUtils; import com.megaease.easeagent.report.async.AsyncProps; import com.megaease.easeagent.report.async.AsyncReporter; @@ -32,6 +30,8 @@ import com.megaease.easeagent.report.plugin.ReporterRegistry; import com.megaease.easeagent.report.sender.SenderWithEncoder; +import io.opentelemetry.sdk.logs.data.LogData; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,23 +40,24 @@ import static com.megaease.easeagent.config.report.ReportConfigConst.*; @SuppressWarnings("unused") -public class LogReporter implements ConfigChangeListener { +public class ApplicationLogReporter implements ConfigChangeListener { Config config; - AsyncReporter asyncReporter; + AsyncReporter asyncReporter; - public LogReporter(Config configs) { + public ApplicationLogReporter(Config configs) { Map cfg = ConfigUtils.extractByPrefix(configs.getConfigs(), LOGS); cfg.putAll(ConfigUtils.extractByPrefix(configs.getConfigs(), OUTPUT_SERVER_V2)); + cfg.putAll(ConfigUtils.extractByPrefix(configs.getConfigs(), LOG_ASYNC)); this.config = new Configs(cfg); configs.addChangeListener(this); SenderWithEncoder sender = ReporterRegistry.getSender(ReportConfigConst.LOG_SENDER, configs); - AsyncProps asyncProperties = new LogAsyncProps(this.config); + AsyncProps asyncProperties = new LogAsyncProps(this.config, null); this.asyncReporter = DefaultAsyncReporter.builderAsyncReporter(sender, asyncProperties); this.asyncReporter.startFlushThread(); } - public void report(AccessLogInfo log) { + public void report(LogData log) { this.asyncReporter.report(log); } @@ -80,7 +81,7 @@ private Map filterChanges(List list) { } public synchronized void refresh(Map cfg) { - String name = cfg.get(LOG_SENDER_NAME); + String name = cfg.get(LOG_ACCESS_SENDER_NAME); SenderWithEncoder sender = asyncReporter.getSender(); if (sender != null) { if (StringUtils.isNotEmpty(name) && !sender.name().equals(name)) { @@ -97,7 +98,7 @@ public synchronized void refresh(Map cfg) { asyncReporter.setSender(sender); } - AsyncProps asyncProperties = new LogAsyncProps(this.config); + AsyncProps asyncProperties = new LogAsyncProps(this.config, null); asyncReporter.closeFlushThread(); asyncReporter.setPending(asyncProperties.getQueuedMaxItems(), asyncProperties.getQueuedMaxSize()); asyncReporter.setMessageTimeoutNanos(messageTimeout(asyncProperties.getMessageTimeout())); diff --git a/report/src/main/java/com/megaease/easeagent/report/async/log/LogAsyncProps.java b/report/src/main/java/com/megaease/easeagent/report/async/log/LogAsyncProps.java index c9f7d670f..a39340d68 100644 --- a/report/src/main/java/com/megaease/easeagent/report/async/log/LogAsyncProps.java +++ b/report/src/main/java/com/megaease/easeagent/report/async/log/LogAsyncProps.java @@ -18,6 +18,7 @@ package com.megaease.easeagent.report.async.log; import com.megaease.easeagent.plugin.api.config.Config; +import com.megaease.easeagent.plugin.utils.common.StringUtils; import com.megaease.easeagent.report.async.AsyncProps; import static com.megaease.easeagent.config.ConfigUtils.bindProp; @@ -30,13 +31,24 @@ public class LogAsyncProps implements AsyncProps { private volatile int messageTimeout; private volatile int messageMaxBytes; - public LogAsyncProps(Config config) { + public LogAsyncProps(Config config, String prefix) { int onePercentageMemory = AsyncProps.onePercentOfMemory(); - bindProp(LOG_ASYNC_REPORT_THREAD, config, Config::getInt, v -> this.reportThread = v, 1); - bindProp(LOG_ASYNC_QUEUED_MAX_SIZE, config, Config::getInt, v -> this.queuedMaxSize = v, onePercentageMemory); - bindProp(LOG_ASYNC_QUEUED_MAX_LOGS, config, Config::getInt, v -> this.queuedMaxLogs = v, 500); - bindProp(LOG_ASYNC_MESSAGE_MAX_BYTES, config, Config::getInt, v -> this.messageMaxBytes = v, 999900); - bindProp(LOG_ASYNC_MESSAGE_TIMEOUT, config, Config::getInt, v -> this.messageTimeout = v, 1000); + String keyPrefix = StringUtils.isEmpty(prefix) ? LOG_ASYNC : prefix; + + bindProp(join(keyPrefix, join(ASYNC_KEY, ASYNC_THREAD_KEY)), + config, Config::getInt, v -> this.reportThread = v, 1); + + bindProp(join(keyPrefix, join(ASYNC_KEY, ASYNC_QUEUE_MAX_SIZE_KEY)), + config, Config::getInt, v -> this.queuedMaxSize = v, onePercentageMemory); + + bindProp(join(keyPrefix, join(ASYNC_KEY, ASYNC_QUEUE_MAX_LOGS_KEY)), + config, Config::getInt, v -> this.queuedMaxLogs = v, 500); + + bindProp(join(keyPrefix, join(ASYNC_KEY, ASYNC_MSG_MAX_BYTES_KEY)), + config, Config::getInt, v -> this.messageMaxBytes = v, 999900); + + bindProp(join(keyPrefix, join(ASYNC_KEY, ASYNC_MSG_TIMEOUT_KEY)), + config, Config::getInt, v -> this.messageTimeout = v, 1000); } @Override diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogJsonEncoder.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/AccessLogJsonEncoder.java similarity index 88% rename from report/src/main/java/com/megaease/easeagent/report/encoder/log/LogJsonEncoder.java rename to report/src/main/java/com/megaease/easeagent/report/encoder/log/AccessLogJsonEncoder.java index f4d111bee..d6d63f03d 100644 --- a/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogJsonEncoder.java +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/AccessLogJsonEncoder.java @@ -17,8 +17,6 @@ */ package com.megaease.easeagent.report.encoder.log; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.auto.service.AutoService; import com.megaease.easeagent.config.report.ReportConfigConst; import com.megaease.easeagent.plugin.api.config.Config; @@ -30,10 +28,10 @@ import zipkin2.internal.JsonCodec; @AutoService(Encoder.class) -public class LogJsonEncoder extends JsonEncoder { - public static final String ENCODER_NAME = ReportConfigConst.LOG_JSON_ENCODER_NAME; +public class AccessLogJsonEncoder extends JsonEncoder { + public static final String ENCODER_NAME = ReportConfigConst.ACCESS_LOG_JSON_ENCODER_NAME; - LogWriter writer; + AccessLogWriter writer; @Override public String name() { @@ -42,7 +40,7 @@ public String name() { @Override public void init(Config config) { - this.writer = new LogWriter(); + this.writer = new AccessLogWriter(); } @Override diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogWriter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/AccessLogWriter.java similarity index 99% rename from report/src/main/java/com/megaease/easeagent/report/encoder/log/LogWriter.java rename to report/src/main/java/com/megaease/easeagent/report/encoder/log/AccessLogWriter.java index 4ca850cfe..72182b976 100644 --- a/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogWriter.java +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/AccessLogWriter.java @@ -23,7 +23,7 @@ import java.util.Map; -public class LogWriter implements WriteBuffer.Writer { +public class AccessLogWriter implements WriteBuffer.Writer { static final String TYPE_FIELD_NAME = "\"type\":\""; static final String TRACE_ID_FIELD_NAME = ",\"trace_id\":\""; static final String SPAN_ID_FIELD_NAME = ",\"span_id\":\""; diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogDataJsonEncoder.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogDataJsonEncoder.java new file mode 100644 index 000000000..0c97948e3 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogDataJsonEncoder.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log; + +import com.google.auto.service.AutoService; +import com.megaease.easeagent.config.Configs; +import com.megaease.easeagent.config.report.ReportConfigConst; +import com.megaease.easeagent.plugin.api.config.ChangeItem; +import com.megaease.easeagent.plugin.api.config.Config; +import com.megaease.easeagent.plugin.api.config.ConfigChangeListener; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.report.ByteWrapper; +import com.megaease.easeagent.plugin.report.EncodedData; +import com.megaease.easeagent.plugin.report.Encoder; +import com.megaease.easeagent.plugin.report.encoder.JsonEncoder; +import zipkin2.internal.JsonCodec; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static com.megaease.easeagent.config.report.ReportConfigConst.ENCODER_KEY; + +@AutoService(Encoder.class) +public class LogDataJsonEncoder extends JsonEncoder implements ConfigChangeListener { + public static final String ENCODER_NAME = ReportConfigConst.LOG_DATA_JSON_ENCODER_NAME; + + Config encoderConfig; + LogDataWriter writer; + + @Override + public void init(Config config) { + config.addChangeListener(this); + this.encoderConfig = new Configs(getEncoderConfig(config.getConfigs())); + this.writer = new LogDataWriter(this.encoderConfig); + } + + @Override + public int sizeInBytes(AgentLogData input) { + return this.writer.sizeInBytes(input); + } + + @Override + public EncodedData encode(AgentLogData input) { + try { + EncodedData d = input.getEncodedData(); + if (d == null) { + d = new ByteWrapper(JsonCodec.write(writer, input)); + input.setEncodedData(d); + } + return d; + } catch (Exception e) { + return new ByteWrapper(new byte[0]); + } + } + + @Override + public String name() { + return ENCODER_NAME; + } + + @Override + public void onChange(List list) { + if (list.isEmpty()) { + return; + } + Map changes = new HashMap<>(); + list.forEach(change -> changes.put(change.getFullName(), change.getNewValue())); + Map encoderChanges = getEncoderConfig(changes); + if (encoderChanges.isEmpty()) { + return; + } + Map cfg = this.encoderConfig.getConfigs(); + cfg.putAll(encoderChanges); + this.encoderConfig = new Configs(cfg); + this.writer = new LogDataWriter(this.encoderConfig); + } + + private Map getEncoderConfig(Map cfgMap) { + Map encoderMap = new TreeMap<>(); + + cfgMap.forEach((k, v) -> { + if (k.contains(ENCODER_KEY) && !k.endsWith(ENCODER_KEY)) { + encoderMap.put(k.substring(k.lastIndexOf('.') + 1), v); + } + }); + + return encoderMap; + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogDataWriter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogDataWriter.java new file mode 100644 index 000000000..88bce86f2 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/LogDataWriter.java @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log; + +import com.megaease.easeagent.plugin.api.config.Config; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.utils.common.StringUtils; +import com.megaease.easeagent.report.encoder.log.pattern.LogDataPatternFormatter; +import io.opentelemetry.api.trace.SpanContext; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.apache.logging.log4j.core.pattern.PatternParser; +import zipkin2.internal.JsonEscaper; +import zipkin2.internal.WriteBuffer; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + + + timestamp + [UNIX_TIMESTAMP_AS_NUMBER] + UTC + + + + { + "service": "${APP_NAME}", + "traceId": "%X{traceId}", + "id": "%X{spanId}", + "logLevel": "%-5level", + "threadId": "%thread", + "location": "%logger{36}", + "message": "%msg%n", + "type": "application-log" + } + + + +**/ +public class LogDataWriter implements WriteBuffer.Writer { + static final String TYPE_FIELD_NAME = "\"type\":\"application-log\""; + + static final String TRACE_ID_FIELD_NAME = ",\"traceId\":\""; + static final String SPAN_ID_FIELD_NAME = ",\"id\":\""; + + static final String SERVICE_FIELD_NAME = ",\"service\":\""; + static final String SYSTEM_FIELD_NAME = ",\"system\":\""; + static final String TIMESTAMP_FILED_NAME = ",\"timestamp\":\""; + static final String TIMESTAMP_NUM_FILED_NAME = ",\"timestamp\":"; + static final String LOG_LEVEL_FIELD_NAME = ",\"logLevel\":\""; + static final String THREAD_ID_FIELD_NAME = ",\"threadId\":\""; + static final String LOCATION_FIELD_NAME = ",\"location\":\""; + static final String MESSAGE_FIELD_NAME = ",\"message\":\""; + + static final String TIMESTAMP = "timestamp"; + static final String LOG_LEVEL = "logLevel"; + static final String THREAD_ID = "threadId"; + static final String LOCATION = "location"; + static final String MESSAGE = "message"; + + static final int STATIC_SIZE = 2 + + TYPE_FIELD_NAME.length() + + SERVICE_FIELD_NAME.length() + 1 + + SYSTEM_FIELD_NAME.length() + 1; + + private static final ThreadLocal threadLocal = new ThreadLocal<>(); + + protected static final int DEFAULT_STRING_BUILDER_SIZE = 1024; + protected static final int MAX_STRING_BUILDER_SIZE = 2048; + + Config config; + + PatternParser parser; + + boolean dateTypeIsNumber = false; + List dateFormats; + List threadIdFormats; + List levelFormats; + List locationFormats; + List messageFormats; + + Map> customFields = new HashMap<>(); + + public LogDataWriter(Config cfg) { + this.config = cfg; + this.parser = PatternLayout.createPatternParser(null); + initFormatters(); + } + + @Override + public int sizeInBytes(AgentLogData value) { + int size = STATIC_SIZE; + + size += JsonEscaper.jsonEscapedSizeInBytes(value.getAgentResource().getService()); + size += JsonEscaper.jsonEscapedSizeInBytes(value.getAgentResource().getSystem()); + + if (!value.getSpanContext().equals(SpanContext.getInvalid())) { + size += TRACE_ID_FIELD_NAME.length() + value.getSpanContext().getTraceId().length() + 1; + size += SPAN_ID_FIELD_NAME.length() + value.getSpanContext().getSpanId().length() + 1; + } + + StringBuilder sb = getStringBuilder(); + if (this.dateTypeIsNumber) { + size += TIMESTAMP_NUM_FILED_NAME.length(); + size += WriteBuffer.asciiSizeInBytes(value.getEpochMillis()); + } else { + size += kvLength(TIMESTAMP_FILED_NAME, value, this.dateFormats, sb, false); + } + + size += kvLength(LOG_LEVEL_FIELD_NAME, value, this.levelFormats, sb, false); + size += kvLength(THREAD_ID_FIELD_NAME, value, this.threadIdFormats, sb, false); + + // instrumentInfo - loggerName + size += kvLength(LOCATION_FIELD_NAME, value, this.locationFormats, sb, false); + + if (!this.customFields.isEmpty()) { + for (Map.Entry> c : this.customFields.entrySet()) { + size += kvLength(c.getKey(), value, c.getValue(), sb, true); + } + } + + size += kvLength(MESSAGE_FIELD_NAME, value, this.messageFormats, sb, true); + + return size; + } + + @Override + public void write(AgentLogData value, WriteBuffer b) { + StringBuilder sb = getStringBuilder(); + + // fix items + b.writeByte(123); + b.writeAscii(TYPE_FIELD_NAME); + + if (!value.getSpanContext().equals(SpanContext.getInvalid())) { + // traceId + b.writeAscii(TRACE_ID_FIELD_NAME); + b.writeAscii(value.getSpanContext().getTraceId()); + b.writeByte('\"'); + + b.writeAscii(SPAN_ID_FIELD_NAME); + b.writeAscii(value.getSpanContext().getSpanId()); + b.writeByte('\"'); + } + + // resource - system/service + b.writeAscii(SERVICE_FIELD_NAME); + b.writeUtf8(JsonEscaper.jsonEscape(value.getAgentResource().getService())); + b.writeByte('\"'); + + b.writeAscii(SYSTEM_FIELD_NAME); + b.writeUtf8(JsonEscaper.jsonEscape(value.getAgentResource().getSystem())); + b.writeByte('\"'); + + if (this.dateTypeIsNumber) { + b.writeAscii(TIMESTAMP_NUM_FILED_NAME); + b.writeAscii(value.getEpochMillis()); + } else { + writeKeyValue(b, TIMESTAMP_FILED_NAME, value, this.dateFormats, sb, false); + } + + writeKeyValue(b, LOG_LEVEL_FIELD_NAME, value, this.levelFormats, sb, false); + writeKeyValue(b, THREAD_ID_FIELD_NAME, value, this.threadIdFormats, sb, false); + + // instrumentInfo - loggerName + writeKeyValue(b, LOCATION_FIELD_NAME, value, this.locationFormats, sb, false); + + // attribute and custom + if (!this.customFields.isEmpty()) { + for (Map.Entry> c : this.customFields.entrySet()) { + writeKeyValue(b, c.getKey(), value, c.getValue(), sb, true); + } + } + + writeKeyValue(b, MESSAGE_FIELD_NAME, value, this.messageFormats, sb, true); + + b.writeByte(125); + + } + + /** + * count size written by @{link writeKeyValue} + * @return size + */ + private int kvLength(String key, AgentLogData value, + List formatters, StringBuilder sb, boolean escape) { + String d = value.getPatternMap().get(key); + if (d == null) { + sb.setLength(0); + d = toSerializable(value, formatters, sb); + value.getPatternMap().put(key, d); + } + + if (StringUtils.isEmpty(d)) { + return 0; + } else { + if (escape) { + return key.length() + JsonEscaper.jsonEscapedSizeInBytes(d) + 1; + } else { + return key.length() + d.length() + 1; + } + } + } + + private void writeKeyValue(WriteBuffer b, String key, AgentLogData value, + List formatters, + StringBuilder sb, boolean escape) { + String d = value.getPatternMap().get(key); + if (d == null) { + sb.setLength(0); + d = toSerializable(value, formatters, sb); + } + + if (!StringUtils.isEmpty(d)) { + b.writeAscii(key); + if (escape) { + b.writeUtf8(JsonEscaper.jsonEscape(d)); + } else { + b.writeAscii(d); + } + b.writeByte('\"'); + } + } + + private void initFormatters() { + this.config.getConfigs().forEach((k, v) -> { + List logDataFormatters = LogDataPatternFormatter.transform(v, this.parser); + + switch (k) { + case LOG_LEVEL: + this.levelFormats = logDataFormatters; + break; + case THREAD_ID: + this.threadIdFormats = logDataFormatters; + break; + case LOCATION: + this.locationFormats = logDataFormatters; + break; + case TIMESTAMP: + this.dateFormats = logDataFormatters; + this.dateTypeIsNumber = v.equals("%d{UNIX_MILLIS}") || v.equals("%d{UNIX}") + || v.equals("%date{UNIX_MILLIS}") || v.equals("%date{UNIX}"); + break; + case MESSAGE: + this.messageFormats = logDataFormatters; + break; + default: + // custom attribute encoder + String key = ",\"" + k + "\":\""; + this.customFields.put(key, logDataFormatters); + break; + } + }); + } + + /** + * Returns a {@code StringBuilder} that this Layout implementation can use to write the formatted log event to. + * + * @return a {@code StringBuilder} + */ + protected static StringBuilder getStringBuilder() { + StringBuilder result = threadLocal.get(); + if (result == null) { + result = new StringBuilder(DEFAULT_STRING_BUILDER_SIZE); + threadLocal.set(result); + } + trimToMaxSize(result, MAX_STRING_BUILDER_SIZE); + result.setLength(0); + return result; + } + + private String toSerializable(final AgentLogData value, + List formatters, + final StringBuilder sb) { + sb.setLength(0); + for (LogDataPatternFormatter formatter : formatters) { + formatter.format(value, sb); + } + return sb.toString(); + } + + public static void trimToMaxSize(final StringBuilder stringBuilder, final int maxSize) { + if (stringBuilder != null && stringBuilder.capacity() > maxSize) { + stringBuilder.setLength(maxSize); + stringBuilder.trimToSize(); + } + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataDatePatternConverterDelegate.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataDatePatternConverterDelegate.java new file mode 100644 index 000000000..cd4c462ab --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataDatePatternConverterDelegate.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import org.apache.logging.log4j.core.pattern.DatePatternConverter; + +import java.util.concurrent.TimeUnit; + +public class LogDataDatePatternConverterDelegate extends LogDataPatternConverter { + DatePatternConverter converter; + + /** + * Create a new pattern converter. + * + * @param options options. + */ + protected LogDataDatePatternConverterDelegate(String[] options) { + super("Date", "date"); + this.converter = DatePatternConverter.newInstance(options); + } + + public LogDataDatePatternConverterDelegate(DatePatternConverter dateConverter) { + super("Date", "date"); + this.converter = dateConverter; + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + this.converter.format(event.getEpochMillis(), toAppendTo); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLevelPatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLevelPatternConverter.java new file mode 100644 index 000000000..6c116ce31 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLevelPatternConverter.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; + +public class LogDataLevelPatternConverter extends LogDataPatternConverter { + /** + * Create a new pattern converter. + */ + protected LogDataLevelPatternConverter() { + super("Level", "level"); + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + toAppendTo.append(event.getSeverityText()); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLineSeparatorPatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLineSeparatorPatternConverter.java new file mode 100644 index 000000000..2299c8ecd --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLineSeparatorPatternConverter.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; + +public class LogDataLineSeparatorPatternConverter extends LogDataPatternConverter { + public static final LogDataLineSeparatorPatternConverter INSTANCE = new LogDataLineSeparatorPatternConverter(); + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + toAppendTo.append(System.lineSeparator()); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLoggerPatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLoggerPatternConverter.java new file mode 100644 index 000000000..6a6693e80 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataLoggerPatternConverter.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; + +public class LogDataLoggerPatternConverter extends NamePatternConverter { + /** + * Create a new pattern converter. + * + * @param options options, may be null. + */ + public LogDataLoggerPatternConverter(String[] options) { + super("Logger", "logger", options); + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + abbreviate(event.getLocation(), toAppendTo); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataMdcPatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataMdcPatternConverter.java new file mode 100644 index 000000000..6e1b18de1 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataMdcPatternConverter.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.api.otlp.common.SemanticKey; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import org.apache.logging.log4j.util.StringBuilders; +import org.apache.logging.log4j.util.TriConsumer; + +import java.util.Arrays; + +/** + * port from log4j2.MdcPatternConverter + */ +public class LogDataMdcPatternConverter extends LogDataPatternConverter { + /** + * Name of property to output. + */ + private final String key; + private final String[] keys; + private final boolean full; + + // reference to log4j2's MdcPatternConverter + public LogDataMdcPatternConverter(String[] options) { + super(options != null && options.length > 0 ? "MDC{" + options[0] + '}' : "MDC", "mdc"); + if (options != null && options.length > 0) { + full = false; + if (options[0].indexOf(',') > -1) { + String oKey; + String[] oKeys = options[0].split(","); + int idx = 0; + for (int i = 0; i < oKeys.length; i++) { + oKey = oKeys[i].trim(); + if (oKey.length() <= 0) { + continue; + } + oKeys[idx++] = oKey; + } + if (idx == 0) { + keys = null; + key = options[0]; + } else { + keys = Arrays.copyOf(oKeys, idx); + key = null; + } + } else { + keys = null; + key = options[0]; + } + } else { + full = true; + key = null; + keys = null; + } + } + + private static final TriConsumer, Object, StringBuilder> WRITE_KEY_VALUES_INTO = (k, value, sb) -> { + sb.append(k.getKey()).append('='); + StringBuilders.appendValue(sb, value); + sb.append(", "); + }; + + /** + * {@inheritDoc} + */ + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + final Attributes contextData = event.getAttributes(); + // if there is no additional options, we output every single + // Key/Value pair for the MDC in a similar format to Hashtable.toString() + if (full) { + if (contextData == null || contextData.isEmpty()) { + toAppendTo.append("{}"); + return; + } + appendFully(contextData, toAppendTo); + } else if (keys != null) { + if (contextData == null || contextData.isEmpty()) { + toAppendTo.append("{}"); + return; + } + appendSelectedKeys(keys, contextData, toAppendTo); + } else if (contextData != null){ + // otherwise they just want a single key output + final Object value = contextData.get(SemanticKey.stringKey(key)); + if (value != null) { + StringBuilders.appendValue(toAppendTo, value); + } + } + } + + private static void appendFully(final Attributes contextData, final StringBuilder toAppendTo) { + toAppendTo.append("{"); + final int start = toAppendTo.length(); + contextData.forEach((k, v) -> WRITE_KEY_VALUES_INTO.accept(k, v, toAppendTo)); + final int end = toAppendTo.length(); + if (end > start) { + toAppendTo.setCharAt(end - 2, '}'); + toAppendTo.deleteCharAt(end - 1); + } else { + toAppendTo.append('}'); + } + } + + private static void appendSelectedKeys(final String[] keys, final Attributes contextData, final StringBuilder sb) { + // Print all the keys in the array that have a value. + final int start = sb.length(); + sb.append('{'); + for (final String theKey : keys) { + final Object value = contextData.get(SemanticKey.stringKey(theKey)); + if (value != null) { + if (sb.length() - start > 1) { + sb.append(", "); + } + sb.append(theKey).append('='); + StringBuilders.appendValue(sb, value); + } + } + sb.append('}'); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataPatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataPatternConverter.java new file mode 100644 index 000000000..e05225328 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataPatternConverter.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; + +import java.util.ArrayList; +import java.util.List; + +public abstract class LogDataPatternConverter extends LogEventPatternConverter { + protected LogDataPatternConverter() { + super("", ""); + } + /** + * Create a new pattern converter. + * + * @param name name for pattern converter. + * @param style CSS style for formatted output. + */ + protected LogDataPatternConverter(String name, String style) { + super(name, style); + } + + /** + * Formats an event into a string buffer. + * + * @param event event to format, may not be null. + * @param toAppendTo string buffer to which the formatted event will be appended. May not be null. + */ + public abstract void format(final AgentLogData event, final StringBuilder toAppendTo); + + /** + * {@inheritDoc} + */ + @Override + public void format(final Object obj, final StringBuilder output) { + if (obj instanceof AgentLogData) { + format((AgentLogData) obj, output); + } else { + super.format(obj, output); + } + } + + @Override + public void format(final LogEvent event, final StringBuilder toAppendTo) { + // ignored + } + + protected String[] getOptions(String pattern, int start) { + ArrayList options = new ArrayList<>(); + extractOptions(pattern, start, options); + return options.toArray(new String[0]); + } + + /** + * Extract options. + * borrow from log4j:PatternParser + * + * @param pattern + * conversion pattern. + * @param start + * start of options. + * @param options + * array to receive extracted options + * @return position in pattern after options. + */ + @SuppressWarnings("UnusedReturnValue") + private int extractOptions(final String pattern, final int start, final List options) { + int i = pattern.indexOf('{', start); + if (i < 0) { + return start; + } + while (i < pattern.length() && pattern.charAt(i) == '{') { + i++; // skip opening "{" + final int begin = i; // position of first real char + int depth = 1; // already inside one level + while (depth > 0 && i < pattern.length()) { + final char c = pattern.charAt(i); + if (c == '{') { + depth++; + } else if (c == '}') { + depth--; + // TODO(?) maybe escaping of { and } with \ or % + } + i++; + } // while + + if (depth > 0) { // option not closed, continue with pattern after closing bracket + i = pattern.lastIndexOf('}'); + if (i == -1 || i < start) { + // if no closing bracket could be found or there is no closing bracket behind the starting + // character of our parsing process continue parsing after the first opening bracket + return begin; + } + return i + 1; + } + + options.add(pattern.substring(begin, i - 1)); + } // while + + return i; + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataPatternFormatter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataPatternFormatter.java new file mode 100644 index 000000000..169ea4736 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataPatternFormatter.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import org.apache.logging.log4j.core.pattern.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Log4j pattern formats to LogData pattern formats + */ +public class LogDataPatternFormatter extends LogDataPatternConverter { + private final LogDataPatternConverter converter; + private final FormattingInfo field; + private final boolean skipFormattingInfo; + + public LogDataPatternFormatter(String pattern, int patternOffset, + org.apache.logging.log4j.core.pattern.PatternFormatter formatter) { + super("", ""); + this.field = formatter.getFormattingInfo(); + this.skipFormattingInfo = this.field == FormattingInfo.getDefault(); + this.converter = extractConvert(formatter.getConverter(), pattern, patternOffset); + } + + public void format(final AgentLogData event, final StringBuilder buf) { + if (skipFormattingInfo) { + converter.format(event, buf); + } else { + formatWithInfo(event, buf); + } + } + + private void formatWithInfo(final AgentLogData event, final StringBuilder buf) { + final int startField = buf.length(); + converter.format(event, buf); + field.format(startField, buf); + } + + private LogDataPatternConverter extractConvert(LogEventPatternConverter converter, String pattern, int patternOffset) { + if (converter == null) { + return NoOpPatternConverter.INSTANCE; + } + + // xxx: can convert to name-INSTANCE map + if (converter instanceof DatePatternConverter) { + return new LogDataDatePatternConverterDelegate((DatePatternConverter)converter); + } else if (converter instanceof LoggerPatternConverter) { + return new LogDataLoggerPatternConverter(getOptions(pattern, patternOffset)); + } else if (converter instanceof LevelPatternConverter) { + return new LogDataLevelPatternConverter(); + } else if (converter.getName().equals("SimpleLiteral")) { + return new LogDataSimpleLiteralPatternConverter(converter); + } else if (converter instanceof MessagePatternConverter) { + return SimpleMessageConverter.INSTANCE; + } else if (converter instanceof ThreadNamePatternConverter) { + return LogDataThreadNamePatternConverter.INSTANCE; + } else if (converter instanceof LineSeparatorPatternConverter) { + return LogDataLineSeparatorPatternConverter.INSTANCE; + } else if (converter instanceof MdcPatternConverter) { + return new LogDataMdcPatternConverter(getOptions(pattern, patternOffset)); + } else if (converter instanceof ThrowablePatternConverter) { + return new LogDataThrowablePatternConverter(getOptions(pattern, patternOffset)); + } else { + return LogDataSimpleLiteralPatternConverter.UNKNOWN; + } + } + + public static List transform(String pattern, PatternParser parser) { + final List formatters = parser.parse(pattern, + false, false, false); + + final List logDataFormatters = new ArrayList<>(); + final AtomicInteger patternOffset = new AtomicInteger(0); + + formatters.forEach(f -> { + if (!f.getConverter().getName().equals("SimpleLiteral")) { + patternOffset.set(pattern.indexOf('%', patternOffset.get()) + 1); + } + logDataFormatters.add(new LogDataPatternFormatter(pattern, patternOffset.get(), f)); + }); + + return logDataFormatters; + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataSimpleLiteralPatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataSimpleLiteralPatternConverter.java new file mode 100644 index 000000000..faa0b2d56 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataSimpleLiteralPatternConverter.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; + +public class LogDataSimpleLiteralPatternConverter extends LogDataPatternConverter { + public static final LogDataPatternConverter UNKNOWN = new SimpleLiteralConverter("-Unknown Pattern-"); + + LogEventPatternConverter converter; + + /** + * Create a new pattern converter. + */ + public LogDataSimpleLiteralPatternConverter(LogEventPatternConverter converter) { + super("SimpleLiteral", "literal"); + this.converter = converter; + } + + @Override + public void format(final Object obj, final StringBuilder output) { + this.converter.format(obj, output); + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + this.format((Object) event, toAppendTo); + } + + public static class SimpleLiteralConverter extends LogDataPatternConverter { + String literal; + + /** + * Constructs an instance of LoggingEventPatternConverter. + */ + protected SimpleLiteralConverter(String literal) { + super("SimpleLiteral", "literal"); + this.literal = literal; + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + toAppendTo.append(this.literal); + } + + @Override + public void format(LogEvent event, StringBuilder toAppendTo) { + toAppendTo.append(this.literal); + } + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataThreadNamePatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataThreadNamePatternConverter.java new file mode 100644 index 000000000..174ff5e98 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataThreadNamePatternConverter.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; + +public class LogDataThreadNamePatternConverter extends LogDataPatternConverter { + public static final LogDataThreadNamePatternConverter INSTANCE = new LogDataThreadNamePatternConverter(); + /** + * Create a new pattern converter. + */ + protected LogDataThreadNamePatternConverter() { + super("Thread", "thread"); + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + toAppendTo.append(event.getThreadName()); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataThrowablePatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataThrowablePatternConverter.java new file mode 100644 index 000000000..8da50a6cb --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/LogDataThrowablePatternConverter.java @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import org.apache.logging.log4j.core.impl.ThrowableFormatOptions; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.apache.logging.log4j.core.pattern.PatternParser; +import org.apache.logging.log4j.core.util.StringBuilderWriter; +import org.apache.logging.log4j.util.Strings; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * ported from log4j2.ThrowablePatternConverter + */ +public class LogDataThrowablePatternConverter extends LogDataPatternConverter { + /** + * Lists {@link LogDataPatternFormatter}s for the suffix attribute. + */ + protected final List formatters; + private String rawOption; + private final boolean subShortOption; + private final boolean nonStandardLineSeparator; + + /** + * Options. + */ + protected final ThrowableFormatOptions options; + + public LogDataThrowablePatternConverter(String[] options) { + super("Throwable", "throwable"); + this.options = ThrowableFormatOptions.newInstance(options); + if (options != null && options.length > 0) { + rawOption = options[0]; + } + if (this.options.getSuffix() != null) { + final PatternParser parser = PatternLayout.createPatternParser(null); + final String suffixPattern = this.options.getSuffix(); + + final List parsedSuffixFormatters = LogDataPatternFormatter.transform(suffixPattern, parser); + + // filter out nested formatters that will handle throwable + boolean hasThrowableSuffixFormatter = false; + for (final LogDataPatternFormatter suffixFormatter : parsedSuffixFormatters) { + if (suffixFormatter.handlesThrowable()) { + hasThrowableSuffixFormatter = true; + } + } + if (!hasThrowableSuffixFormatter) { + this.formatters = parsedSuffixFormatters; + } else { + final List suffixFormatters = new ArrayList<>(); + for (final LogDataPatternFormatter suffixFormatter : parsedSuffixFormatters) { + if (!suffixFormatter.handlesThrowable()) { + suffixFormatters.add(suffixFormatter); + } + } + this.formatters = suffixFormatters; + } + } else { + this.formatters = Collections.emptyList(); + } + subShortOption = ThrowableFormatOptions.MESSAGE.equalsIgnoreCase(rawOption) || + ThrowableFormatOptions.LOCALIZED_MESSAGE.equalsIgnoreCase(rawOption) || + ThrowableFormatOptions.FILE_NAME.equalsIgnoreCase(rawOption) || + ThrowableFormatOptions.LINE_NUMBER.equalsIgnoreCase(rawOption) || + ThrowableFormatOptions.METHOD_NAME.equalsIgnoreCase(rawOption) || + ThrowableFormatOptions.CLASS_NAME.equalsIgnoreCase(rawOption); + nonStandardLineSeparator = !Strings.LINE_SEPARATOR.equals(this.options.getSeparator()); + } + + @Override + public void format(AgentLogData event, StringBuilder buffer) { + final Throwable t = event.getThrowable(); + + if (subShortOption) { + formatSubShortOption(t, getSuffix(event), buffer); + } + else if (t != null && options.anyLines()) { + formatOption(t, getSuffix(event), buffer); + } + } + + private void formatSubShortOption(final Throwable t, final String suffix, final StringBuilder buffer) { + StackTraceElement[] trace; + StackTraceElement throwingMethod = null; + int len; + + if (t != null) { + trace = t.getStackTrace(); + if (trace !=null && trace.length > 0) { + throwingMethod = trace[0]; + } + } + + if (t != null && throwingMethod != null) { + String toAppend = Strings.EMPTY; + + if (ThrowableFormatOptions.CLASS_NAME.equalsIgnoreCase(rawOption)) { + toAppend = throwingMethod.getClassName(); + } + else if (ThrowableFormatOptions.METHOD_NAME.equalsIgnoreCase(rawOption)) { + toAppend = throwingMethod.getMethodName(); + } + else if (ThrowableFormatOptions.LINE_NUMBER.equalsIgnoreCase(rawOption)) { + toAppend = String.valueOf(throwingMethod.getLineNumber()); + } + else if (ThrowableFormatOptions.MESSAGE.equalsIgnoreCase(rawOption)) { + toAppend = t.getMessage(); + } + else if (ThrowableFormatOptions.LOCALIZED_MESSAGE.equalsIgnoreCase(rawOption)) { + toAppend = t.getLocalizedMessage(); + } + else if (ThrowableFormatOptions.FILE_NAME.equalsIgnoreCase(rawOption)) { + toAppend = throwingMethod.getFileName(); + } + + len = buffer.length(); + if (len > 0 && !Character.isWhitespace(buffer.charAt(len - 1))) { + buffer.append(' '); + } + buffer.append(toAppend); + + if (Strings.isNotBlank(suffix)) { + buffer.append(' '); + buffer.append(suffix); + } + } + } + + private void formatOption(final Throwable throwable, final String suffix, final StringBuilder buffer) { + final int len = buffer.length(); + if (len > 0 && !Character.isWhitespace(buffer.charAt(len - 1))) { + buffer.append(' '); + } + if (!options.allLines() || nonStandardLineSeparator || Strings.isNotBlank(suffix)) { + final StringWriter w = new StringWriter(); + throwable.printStackTrace(new PrintWriter(w)); + + final String[] array = w.toString().split(Strings.LINE_SEPARATOR); + final int limit = options.minLines(array.length) - 1; + final boolean suffixNotBlank = Strings.isNotBlank(suffix); + for (int i = 0; i <= limit; ++i) { + buffer.append(array[i]); + if (suffixNotBlank) { + buffer.append(' '); + buffer.append(suffix); + } + if (i < limit) { + buffer.append(options.getSeparator()); + } + } + } else { + throwable.printStackTrace(new PrintWriter(new StringBuilderWriter(buffer))); + } + } + + /** + * This converter obviously handles throwables. + * + * @return true. + */ + @Override + public boolean handlesThrowable() { + return true; + } + + protected String getSuffix(final AgentLogData event) { + if (formatters.isEmpty()) { + return Strings.EMPTY; + } + + //noinspection ForLoopReplaceableByForEach + final StringBuilder toAppendTo = new StringBuilder(); + for (int i = 0, size = formatters.size(); i < size; i++) { + formatters.get(i).format(event, toAppendTo); + } + + return toAppendTo.toString(); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/NamePatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/NamePatternConverter.java new file mode 100644 index 000000000..8b7995622 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/NamePatternConverter.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import org.apache.logging.log4j.core.pattern.NameAbbreviator; + +public abstract class NamePatternConverter extends LogDataPatternConverter { + private final NameAbbreviator abbreviator; + + /** + * Constructor. + * + * @param name name of converter. + * @param style style name for associated output. + * @param options options, may be null, first element will be interpreted as an abbreviation pattern. + */ + protected NamePatternConverter(final String name, final String style, final String[] options) { + super(name, style); + + if (options != null && options.length > 0) { + abbreviator = NameAbbreviator.getAbbreviator(options[0]); + } else { + abbreviator = NameAbbreviator.getDefaultAbbreviator(); + } + } + + /** + * Abbreviate name in string buffer. + * + * @param original string containing name. + * @param destination the StringBuilder to write to + */ + protected final void abbreviate(final String original, final StringBuilder destination) { + abbreviator.abbreviate(original, destination); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/NoOpPatternConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/NoOpPatternConverter.java new file mode 100644 index 000000000..fc915ef12 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/NoOpPatternConverter.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; + +public class NoOpPatternConverter extends LogDataPatternConverter { + public final static NoOpPatternConverter INSTANCE = new NoOpPatternConverter("", ""); + + /** + * Create a new pattern converter. + * + * @param name name for pattern converter. + * @param style CSS style for formatted output. + */ + protected NoOpPatternConverter(String name, String style) { + super(name, style); + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/SimpleMessageConverter.java b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/SimpleMessageConverter.java new file mode 100644 index 000000000..d39b20452 --- /dev/null +++ b/report/src/main/java/com/megaease/easeagent/report/encoder/log/pattern/SimpleMessageConverter.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log.pattern; + +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; + +public class SimpleMessageConverter extends LogDataPatternConverter { + public static final SimpleMessageConverter INSTANCE = new SimpleMessageConverter(); + /** + * Create a new pattern converter. + */ + protected SimpleMessageConverter() { + super("msg", "msg"); + } + + @Override + public void format(AgentLogData event, StringBuilder toAppendTo) { + toAppendTo.append(event.getBody().asString()); + } +} diff --git a/report/src/main/java/com/megaease/easeagent/report/sender/SenderConfigDecorator.java b/report/src/main/java/com/megaease/easeagent/report/sender/SenderConfigDecorator.java index 928e1413d..c0c2a62fb 100644 --- a/report/src/main/java/com/megaease/easeagent/report/sender/SenderConfigDecorator.java +++ b/report/src/main/java/com/megaease/easeagent/report/sender/SenderConfigDecorator.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import static com.megaease.easeagent.config.ConfigUtils.extractByPrefix; import static com.megaease.easeagent.config.report.ReportConfigConst.*; @@ -39,7 +40,8 @@ public class SenderConfigDecorator protected Sender sender; String prefix; - Config config; + Config senderConfig; + Config packerConfig; String encoderKey; Encoder packer; @@ -48,7 +50,8 @@ public SenderConfigDecorator(String prefix, Sender sender, Config config) { this.prefix = prefix; this.encoderKey = getEncoderKey(prefix); config.addChangeListener(this); - this.config = new Configs(extractSenderConfig(this.prefix, config)); + this.senderConfig = new Configs(extractSenderConfig(this.prefix, config)); + this.packerConfig = new Configs(extractSenderConfig(encoderKey, config)); } @Override @@ -64,8 +67,8 @@ public String getPrefix() { @Override public void init(Config config, String prefix) { this.packer = ReporterRegistry.getEncoder(config.getString(this.encoderKey)); - this.packer.init(this.config); - this.sender.init(this.config, prefix); + this.packer.init(this.packerConfig); + this.sender.init(this.senderConfig, prefix); } @Override @@ -99,7 +102,6 @@ public void updateConfigs(Map changes) { log.warn("Sender update fail, can not close sender:{}", this.sender.name()); } } - updateEncoder(changes); } // checkEncoder update @@ -108,8 +110,8 @@ protected void updateEncoder(Map changes) { if (name == null || name.equals(this.packer.name())) { return; } - this.packer = ReporterRegistry.getEncoder(config.getString(this.encoderKey)); - this.packer.init(config); + this.packer = ReporterRegistry.getEncoder(packerConfig.getString(this.encoderKey)); + this.packer.init(packerConfig); } @Override @@ -123,8 +125,26 @@ public void onChange(List list) { if (changes.isEmpty()) { return; } - this.config.updateConfigs(changes); - this.updateConfigs(changes); + Map senderChanges = new TreeMap<>(); + Map packerChanges = new TreeMap<>(); + + changes.forEach((key, value) -> { + if (key.startsWith(encoderKey)) { + packerChanges.put(key, value); + } else { + senderChanges.put(key, value); + } + }); + + if (!packerChanges.isEmpty()) { + this.packerConfig.updateConfigs(packerChanges); + this.updateEncoder(packerChanges); + } + + if (!senderChanges.isEmpty()) { + this.senderConfig.updateConfigs(senderChanges); + this.updateConfigs(senderChanges); + } } private static String getEncoderKey(String cfgPrefix) { @@ -152,7 +172,7 @@ private Map filterChanges(List list) { .filter(one -> { String name = one.getFullName(); return name.startsWith(REPORT) - || name.startsWith(this.encoderKey) + || name.startsWith(encoderKey) || name.startsWith(prefix); }).forEach(one -> cfg.put(one.getFullName(), one.getNewValue())); diff --git a/report/src/test/java/com/megaease/easeagent/report/encoder/log/LogDataJsonEncoderTest.java b/report/src/test/java/com/megaease/easeagent/report/encoder/log/LogDataJsonEncoderTest.java new file mode 100644 index 000000000..033a31502 --- /dev/null +++ b/report/src/test/java/com/megaease/easeagent/report/encoder/log/LogDataJsonEncoderTest.java @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log; + +import com.google.common.base.CharMatcher; +import com.megaease.easeagent.config.Configs; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogData; +import com.megaease.easeagent.plugin.api.otlp.common.AgentLogDataImpl; +import com.megaease.easeagent.plugin.report.EncodedData; +import com.megaease.easeagent.plugin.utils.common.JsonUtil; +import io.opentelemetry.sdk.logs.data.Severity; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; + +import java.util.HashMap; +import java.util.Map; + +import static com.megaease.easeagent.config.report.ReportConfigConst.ENCODER_KEY; +import static com.megaease.easeagent.config.report.ReportConfigConst.join; +import static com.megaease.easeagent.report.encoder.log.LogDataWriter.LOCATION; +import static com.megaease.easeagent.report.encoder.log.LogDataWriter.LOG_LEVEL; + +public class LogDataJsonEncoderTest { + Logger log = LoggerFactory.getLogger(LogDataJsonEncoderTest.class); + LogDataJsonEncoder encoder; + Configs config; + + AgentLogData data; + + @Before + public void init() { + Map cfg = new HashMap<>(); + cfg.put(ENCODER_KEY, LogDataJsonEncoder.ENCODER_NAME); + cfg.put(join(ENCODER_KEY, "timestamp"), "%d{UNIX_MILLIS}"); + cfg.put(join(ENCODER_KEY, "logLevel"), "%-5level"); + cfg.put(join(ENCODER_KEY, "threadId"), "%thread"); + cfg.put(join(ENCODER_KEY, "location"), "%logger{3}"); + cfg.put(join(ENCODER_KEY, "message"), "%msg"); + cfg.put("name", "demo-service"); + cfg.put("system", "demo-system"); + this.config = new Configs(cfg); + + this.data = + AgentLogDataImpl.builder() + .epochMills(1648878722451L) + .logger(log.getName()) + .severity(Severity.INFO) + .severityText(Level.INFO.toString()) + .thread(Thread.currentThread()) + .throwable(new NullPointerException("test")) + .body("Hello") + .build(); + + encoder = new LogDataJsonEncoder(); + encoder.init(this.config); + } + + @Test + public void test_encoder() { + // size = 208 + int size = encoder.sizeInBytes(data); + Assert.assertEquals(203, size); + EncodedData encoded = encoder.encode(data); + Map jsonMap = JsonUtil.toMap(new String(encoded.getData())); + Assert.assertEquals("encoder.log.LogDataJsonEncoderTest", jsonMap.get(LOCATION)); + Assert.assertEquals(5, jsonMap.get(LOG_LEVEL).toString().length()); + } + + @Test + public void test_encoder_update() { + Map changes = new HashMap<>(); + String original = this.config.getString(join(ENCODER_KEY, "location")); + changes.put(join(ENCODER_KEY, "location"), "%logger{2}"); + this.config.updateConfigs(changes); + + int size = encoder.sizeInBytes(data); + Assert.assertEquals(195, size); + EncodedData encoded = encoder.encode(data); + Map jsonMap = JsonUtil.toMap(new String(encoded.getData())); + Assert.assertEquals("log.LogDataJsonEncoderTest", jsonMap.get(LOCATION)); + + changes.put(join(ENCODER_KEY, "location"), original); + this.config.updateConfigs(changes); + } + + @Test + public void test_throwable_encoder() { + Map changes = new HashMap<>(); + + String key = join(ENCODER_KEY, "message"); + String original = this.config.getString(key); + changes.put(key, "%msg%n%xEx"); + this.config.updateConfigs(changes); + + AgentLogData exceptionData = AgentLogDataImpl.builder() + .epochMills(1648878722451L) + .logger(log.getName()) + .severity(Severity.INFO) + .severityText(Level.INFO.toString()) + .thread(Thread.currentThread()) + .throwable(new NullPointerException("test")) + .body("Hello") + .build(); + + EncodedData encoded = encoder.encode(exceptionData); + Map jsonMap = JsonUtil.toMap(new String(encoded.getData())); + + Assert.assertTrue(jsonMap.get("message").toString().contains(NullPointerException.class.getCanonicalName())); + + changes.put(key, original); + this.config.updateConfigs(changes); + } + + @Test + public void test_throwable_short_encoder() { + Map changes = new HashMap<>(); + + String key = join(ENCODER_KEY, "message"); + String original = this.config.getString(key); + changes.put(key, "%msg%xEx{5,separator(|)}"); + this.config.updateConfigs(changes); + + AgentLogData exceptionData = AgentLogDataImpl.builder() + .epochMills(1648878722451L) + .logger(log.getName()) + .severity(Severity.INFO) + .severityText(Level.INFO.toString()) + .thread(Thread.currentThread()) + .throwable(new NullPointerException("test")) + .body("Hello") + .build(); + + // int size = encoder.sizeInBytes(exceptionData); + // Assert.assertEquals(668, size); + EncodedData encoded = encoder.encode(exceptionData); + Map jsonMap = JsonUtil.toMap(new String(encoded.getData())); + + String message = jsonMap.get("message").toString(); + Assert.assertTrue(message.contains(NullPointerException.class.getCanonicalName())); + int count = CharMatcher.is('|').countIn(message); + Assert.assertEquals(4, count); + + changes.put(key, original); + this.config.updateConfigs(changes); + } + + @Test + public void test_mdc_selected() { + Map changes = new HashMap<>(); + + String key = join(ENCODER_KEY, "message"); + String original = this.config.getString(key); + changes.put(key, "%X{name, number}-%msg%xEx{5,separator(|)}"); + this.config.updateConfigs(changes); + + AgentLogDataImpl.Builder builder = AgentLogDataImpl.builder() + .epochMills(1648878722451L) + .logger(log.getName()) + .severity(Severity.INFO) + .severityText(Level.INFO.toString()) + .thread(Thread.currentThread()) + .body("Hello"); + + AgentLogData noMdcData = builder.build(); + int size = encoder.sizeInBytes(noMdcData); + Assert.assertEquals(206, size); + EncodedData encoded = encoder.encode(noMdcData); + Map jsonMap = JsonUtil.toMap(new String(encoded.getData())); + String message = jsonMap.get("message").toString(); + Assert.assertTrue(message.startsWith("{}-Hello")); + + // test mdc + Map ctxData = new HashMap<>(); + ctxData.put("name", "easeagent"); + ctxData.put("number", "v2.2"); + builder.contextData(null, ctxData); + AgentLogData mdcData = builder.build(); + + size = encoder.sizeInBytes(mdcData); + Assert.assertEquals(233, size); + encoded = encoder.encode(mdcData); + jsonMap = JsonUtil.toMap(new String(encoded.getData())); + message = jsonMap.get("message").toString(); + Assert.assertTrue(message.startsWith("{name=easeagent, number=v2.2}-Hello")); + + changes.put(key, original); + this.config.updateConfigs(changes); + } + + @Test + public void test_custom() { + Map original = this.config.getConfigs(); + Map changes = new HashMap<>(); + changes.put("encoder.custom", "%X{custom}"); + + this.config.updateConfigs(changes); + + AgentLogDataImpl.Builder builder = AgentLogDataImpl.builder() + .epochMills(1648878722451L) + .logger(log.getName()) + .severity(Severity.INFO) + .severityText(Level.INFO.toString()) + .thread(Thread.currentThread()) + .body("Hello"); + + // test mdc + Map ctxData = new HashMap<>(); + ctxData.put("custom", "easeagent"); + builder.contextData(null, ctxData); + AgentLogData mdcData = builder.build(); + + int size = encoder.sizeInBytes(mdcData); + Assert.assertEquals(224, size); + EncodedData encoded = encoder.encode(mdcData); + Map jsonMap = JsonUtil.toMap(new String(encoded.getData())); + String custom = jsonMap.get("custom").toString(); + Assert.assertEquals("easeagent", custom); + + this.config = new Configs(original); + } +} diff --git a/report/src/test/java/com/megaease/easeagent/report/encoder/log/LogDataWriterTest.java b/report/src/test/java/com/megaease/easeagent/report/encoder/log/LogDataWriterTest.java new file mode 100644 index 000000000..09683e27b --- /dev/null +++ b/report/src/test/java/com/megaease/easeagent/report/encoder/log/LogDataWriterTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.megaease.easeagent.report.encoder.log; + +import com.megaease.easeagent.config.Configs; +import com.megaease.easeagent.plugin.api.config.Config; +import com.megaease.easeagent.plugin.api.config.IConfigFactory; +import com.megaease.easeagent.plugin.api.config.IPluginConfig; +import com.megaease.easeagent.plugin.bridge.EaseAgent; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +public class LogDataWriterTest { + Logger log = LoggerFactory.getLogger(LogDataWriterTest.class); + Configs config; + @Before + public void init() { + Map cfg = new HashMap<>(); + cfg.put("timestamp", "%d{UNIX_MILLIS}"); + cfg.put("logLevel", "%-5level"); + cfg.put("threadId", "%thread"); + cfg.put("location", "%logger{36}"); + cfg.put("message", "%msg"); + this.config = new Configs(cfg); + } + + @Test + public void test_location_pattern() { + LogDataWriter writer = new LogDataWriter(config); + } +}