From 68e916832b99805d8979f0d1b0537fc8dc7adc5e Mon Sep 17 00:00:00 2001 From: Iliyan Velichkov Date: Fri, 29 Nov 2024 09:21:02 +0200 Subject: [PATCH] OpenTelemetry for Eclipse Dirigible (#4451) * Add Spring Boot Admin (SBA) Signed-off-by: Iliyan Velichkov * add more comments Signed-off-by: Iliyan Velichkov * enable liveness and readiness Signed-off-by: Iliyan Velichkov * increase waiting time in CsvimIT.java Signed-off-by: Iliyan Velichkov * code formatting Signed-off-by: Iliyan Velichkov * OpenTelemetry spring boot setup Signed-off-by: Iliyan Velichkov * update test controller Signed-off-by: Iliyan Velichkov * update service name Signed-off-by: Iliyan Velichkov * enable actuator metrics report Signed-off-by: Iliyan Velichkov * otel profile and otel meter example Signed-off-by: Iliyan Velichkov * do not call GlobalOpenTelemetry Signed-off-by: Iliyan Velichkov * add micrometer counter Signed-off-by: Iliyan Velichkov * add custom span with remote call Signed-off-by: Iliyan Velichkov * add manual span via code not annotations Signed-off-by: Iliyan Velichkov * add query param as span attribute Signed-off-by: Iliyan Velichkov * add span with failure Signed-off-by: Iliyan Velichkov * add call with httpclient + create otel module Signed-off-by: Iliyan Velichkov * add repository calls to the example Signed-off-by: Iliyan Velichkov * add trace for scripts execution Signed-off-by: Iliyan Velichkov * add OpenTelmetry stack setup Signed-off-by: Iliyan Velichkov * adapt tests Signed-off-by: Iliyan Velichkov * loki cleanup, opensearch using docker profile only Signed-off-by: Iliyan Velichkov * add Dirigible sample project Signed-off-by: Iliyan Velichkov * export traces to debug as well Signed-off-by: Iliyan Velichkov * update README.md Signed-off-by: Iliyan Velichkov * less verbose debug exporter Signed-off-by: Iliyan Velichkov * update README.md, update sample project Signed-off-by: Iliyan Velichkov * add job details to the job execution span Signed-off-by: Iliyan Velichkov * enable flowable actuator Signed-off-by: Iliyan Velichkov * export flowable metrics based on data in flowable actuator endpoint Signed-off-by: Iliyan Velichkov * add traces for BPM tasks execution Signed-off-by: Iliyan Velichkov * more details in js and ts endpoints Signed-off-by: Iliyan Velichkov * remove span names for the endpoints Signed-off-by: Iliyan Velichkov * add traces for the synchronizers Signed-off-by: Iliyan Velichkov * rebase with master Signed-off-by: Iliyan Velichkov * fix flowable close Signed-off-by: Iliyan Velichkov * update headers Signed-off-by: Iliyan Velichkov * fix local tests execution Signed-off-by: Iliyan Velichkov * add custom quartz metrics Signed-off-by: Iliyan Velichkov * wip - camel metrics and traces Signed-off-by: Iliyan Velichkov * code formatting Signed-off-by: Iliyan Velichkov * delete CamelTelemetryConfigurator.java Signed-off-by: Iliyan Velichkov * add span for synchronizers span Signed-off-by: Iliyan Velichkov * properties cleanup Signed-off-by: Iliyan Velichkov * fix SecurityIT.java Signed-off-by: Iliyan Velichkov * cleanup and change attribute names - align with the others Signed-off-by: Iliyan Velichkov * cleanup Signed-off-by: Iliyan Velichkov * remove starter vs agent files Signed-off-by: Iliyan Velichkov * remove test REST Signed-off-by: Iliyan Velichkov * remove comment Signed-off-by: Iliyan Velichkov * update README Signed-off-by: Iliyan Velichkov * Update README.md --------- Signed-off-by: Iliyan Velichkov --- .../dirigible/DirigibleApplication.java | 2 + .../src/main/resources/logback.xml | 33 + .../dirigible/modules/build-source.sh | 3 +- .../components/api/test/APIAssertTest.java | 25 +- components/core/core-base/pom.xml | 8 +- .../topology/TopologicalDepleter.java | 7 + .../base/synchronizer/BaseSynchronizer.java | 94 +- .../ExtensionPointsSynchronizer.java | 2 +- .../synchronizer/ExtensionsSynchronizer.java | 2 +- .../synchronizer/SynchronizationJob.java | 3 +- .../csvim/synchronizer/CsvimSynchronizer.java | 2 +- .../synchronizer/DataSourcesSynchronizer.java | 2 +- .../synchronizer/EntitySynchronizer.java | 2 +- .../synchronizer/SchemasSynchronizer.java | 2 +- .../synchronizer/TablesSynchronizer.java | 2 +- .../synchronizer/ViewsSynchronizer.java | 2 +- components/engine/engine-bpm-flowable/pom.xml | 6 +- .../flowable/config/BpmFlowableConfig.java | 25 +- .../BpmProviderFlowableFactoryBean.java | 60 + .../delegate/DirigibleCallDelegate.java | 39 + .../FlowableMetricsConfigurator.java | 111 ++ .../provider/BpmProviderFlowable.java | 5 +- .../synchronizer/BpmnSynchronizer.java | 2 +- components/engine/engine-camel/pom.xml | 26 +- .../camel/synchronizer/CamelSynchronizer.java | 19 +- .../endpoint/JavascriptEndpoint.java | 329 +++-- .../components/jobs/config/QuartzConfig.java | 23 +- .../components/jobs/handler/JobHandler.java | 3 + .../jobs/synchronizer/JobSynchronizer.java | 2 +- .../telemetry/JobExecutionsCountListener.java | 32 + .../JobExecutionsDurationListener.java | 47 + .../telemetry/JobFailuresCountListener.java | 39 + .../jobs/telemetry/OpenTelemetryListener.java | 13 + .../jobs/telemetry/QuartzMetricConstants.java | 7 + .../RunningJobsCountMetricsConfigurator.java | 39 + .../synchronizer/ListenerSynchronizer.java | 2 +- .../odata/synchronizer/ODataSynchronizer.java | 98 +- .../engine/engine-open-telemetry/pom.xml | 68 + .../telemetry/OpenTelemetryConfiguration.java | 18 + .../open/telemetry/OpenTelemetryProvider.java | 46 + .../application-open-telemetry.properties | 21 + .../synchronizer/OpenAPISynchronizer.java | 2 +- .../synchronizer/AccessSynchronizer.java | 2 +- .../synchronizer/RolesSynchronizer.java | 2 +- .../engine/typescript/TypeScriptEndpoint.java | 79 +- .../synchronizer/WebsocketsSynchronizer.java | 2 +- .../synchronizer/ConfluenceSynchronizer.java | 2 +- .../synchronizer/MarkdownSynchronizer.java | 2 +- components/group/group-engines/pom.xml | 10 +- components/pom.xml | 1 + .../core/modules/DirigibleSourceProvider.java | 2 + .../engines/engine-graalium/execution/pom.xml | 8 +- .../core/javascript/GraalJSCodeRunner.java | 300 +++-- modules/pom.xml | 5 + open-telemetry/.env | 8 + open-telemetry/README.md | 93 ++ .../open-telemetry-sample-project.zip | Bin 0 -> 7451 bytes .../dirigible-image-otel-agent/Dockerfile | 9 + .../local-dirigible-otel-agent/Dockerfile | 12 + .../Dockerfile | 5 + open-telemetry/docker-compose.yaml | 167 +++ open-telemetry/grafana/grafana.ini | 1172 +++++++++++++++++ .../provisioning/dashboards/dirigible.yaml | 14 + .../dashboards/dirigible/Meters.json | 343 +++++ .../provisioning/datasources/jaeger.yaml | 29 + .../provisioning/datasources/loki.yaml | 11 + .../provisioning/datasources/opensearch.yaml | 18 + .../provisioning/datasources/prometheus.yaml | 17 + .../opensearch-dashboards/Dockerfile | 3 + .../opensearch_dashboards.yml | 5 + open-telemetry/otel-collector-config.yaml | 53 + open-telemetry/prometheus.yml | 7 + pom.xml | 15 +- .../dirigible/tests/DirigibleCleaner.java | 58 +- .../integration/tests/api/SecurityIT.java | 25 +- .../tests/ui/tests/OrderedTestSuite.java | 2 +- .../tests/ui/tests/SpringBootAdminIT.java | 9 + 77 files changed, 3261 insertions(+), 502 deletions(-) create mode 100644 components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmProviderFlowableFactoryBean.java create mode 100644 components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/open/telemetry/FlowableMetricsConfigurator.java create mode 100644 components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsCountListener.java create mode 100644 components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsDurationListener.java create mode 100644 components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobFailuresCountListener.java create mode 100644 components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/OpenTelemetryListener.java create mode 100644 components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/QuartzMetricConstants.java create mode 100644 components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/RunningJobsCountMetricsConfigurator.java create mode 100644 components/engine/engine-open-telemetry/pom.xml create mode 100644 components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryConfiguration.java create mode 100644 components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryProvider.java create mode 100644 components/engine/engine-open-telemetry/src/main/resources/application-open-telemetry.properties create mode 100644 open-telemetry/.env create mode 100644 open-telemetry/README.md create mode 100644 open-telemetry/dirigible-sample-project/open-telemetry-sample-project.zip create mode 100644 open-telemetry/dirigible/dirigible-image-otel-agent/Dockerfile create mode 100644 open-telemetry/dirigible/local-dirigible-otel-agent/Dockerfile create mode 100644 open-telemetry/dirigible/local-dirigible-otel-spring-starter/Dockerfile create mode 100644 open-telemetry/docker-compose.yaml create mode 100644 open-telemetry/grafana/grafana.ini create mode 100644 open-telemetry/grafana/provisioning/dashboards/dirigible.yaml create mode 100644 open-telemetry/grafana/provisioning/dashboards/dirigible/Meters.json create mode 100644 open-telemetry/grafana/provisioning/datasources/jaeger.yaml create mode 100644 open-telemetry/grafana/provisioning/datasources/loki.yaml create mode 100644 open-telemetry/grafana/provisioning/datasources/opensearch.yaml create mode 100644 open-telemetry/grafana/provisioning/datasources/prometheus.yaml create mode 100644 open-telemetry/opensearch-dashboards/Dockerfile create mode 100644 open-telemetry/opensearch-dashboards/opensearch_dashboards.yml create mode 100644 open-telemetry/otel-collector-config.yaml create mode 100644 open-telemetry/prometheus.yml diff --git a/build/application/src/main/java/org/eclipse/dirigible/DirigibleApplication.java b/build/application/src/main/java/org/eclipse/dirigible/DirigibleApplication.java index fa92eab1655..014c10b78e5 100644 --- a/build/application/src/main/java/org/eclipse/dirigible/DirigibleApplication.java +++ b/build/application/src/main/java/org/eclipse/dirigible/DirigibleApplication.java @@ -10,6 +10,7 @@ package org.eclipse.dirigible; import de.codecentric.boot.admin.server.config.EnableAdminServer; +import org.apache.camel.opentelemetry.starter.CamelOpenTelemetry; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; @@ -21,6 +22,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.client.RestTemplate; +@CamelOpenTelemetry @EnableAdminServer @SpringBootApplication(exclude = {DataSourceAutoConfiguration.class, DataSourceTransactionManagerAutoConfiguration.class, HibernateJpaAutoConfiguration.class, JdbcTemplateAutoConfiguration.class}) diff --git a/build/application/src/main/resources/logback.xml b/build/application/src/main/resources/logback.xml index 2ba7a4f5cab..e43643c4cc6 100644 --- a/build/application/src/main/resources/logback.xml +++ b/build/application/src/main/resources/logback.xml @@ -77,4 +77,37 @@ + + + + false + true + true + true + true + * + + + + + + + + + + + + + + + + + + + + + + + diff --git a/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/build-source.sh b/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/build-source.sh index 23579d289d2..7eb0d81dfd3 100755 --- a/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/build-source.sh +++ b/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/build-source.sh @@ -1,3 +1,4 @@ +#!/bin/sh # fail the whole script if any command bellow fails set -e @@ -11,4 +12,4 @@ esbuild $(find . -iname '*.ts' -not -iname '*.d.ts') '--out-extension:.js=.mjs' esbuild $(find . -iname '*.ts' -not -iname '*.d.ts') --sourcemap=inline --outdir=dist/cjs --format=cjs --target=es2022 # build dts -tsc --emitDeclarationOnly --outDir dist/dts \ No newline at end of file +tsc --emitDeclarationOnly --outDir dist/dts diff --git a/components/api/api-test/src/test/java/org/eclipse/dirigible/components/api/test/APIAssertTest.java b/components/api/api-test/src/test/java/org/eclipse/dirigible/components/api/test/APIAssertTest.java index 07e8dbd9f98..920befaa0ea 100644 --- a/components/api/api-test/src/test/java/org/eclipse/dirigible/components/api/test/APIAssertTest.java +++ b/components/api/api-test/src/test/java/org/eclipse/dirigible/components/api/test/APIAssertTest.java @@ -9,11 +9,6 @@ */ package org.eclipse.dirigible.components.api.test; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - import org.eclipse.dirigible.components.engine.javascript.service.JavascriptService; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -27,6 +22,11 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + @WithMockUser @ExtendWith(SpringExtension.class) @SpringBootTest @@ -34,21 +34,24 @@ @ComponentScan(basePackages = {"org.eclipse.dirigible.components.*"}) public class APIAssertTest { + @Autowired + protected WebApplicationContext wac; @Autowired private JavascriptService javascriptService; - @Autowired private MockMvc mockMvc; - @Autowired - protected WebApplicationContext wac; - // @Autowired // private FilterChainProxy springSecurityFilterChain; // // @Autowired // private IRepository repository; + + @SpringBootApplication + static class TestConfiguration { + } + @Test public void successful() throws Exception { // javascriptService.handleRequest("test", "successful.js", null, null, false); @@ -72,8 +75,4 @@ public void failed() throws Exception { .endsWith("Assertion 'assertTrue' failed")); } } - - @SpringBootApplication - static class TestConfiguration { - } } diff --git a/components/core/core-base/pom.xml b/components/core/core-base/pom.xml index 93af4ca511d..990e91d8c69 100644 --- a/components/core/core-base/pom.xml +++ b/components/core/core-base/pom.xml @@ -1,4 +1,4 @@ - 4.0.0 @@ -20,6 +20,10 @@ org.eclipse.dirigible dirigible-commons-helpers + + org.eclipse.dirigible + dirigible-components-engine-open-telemetry + @@ -27,4 +31,4 @@ ../../../ - \ No newline at end of file + diff --git a/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/artefact/topology/TopologicalDepleter.java b/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/artefact/topology/TopologicalDepleter.java index 8e0af3ba85d..e3393b764fa 100644 --- a/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/artefact/topology/TopologicalDepleter.java +++ b/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/artefact/topology/TopologicalDepleter.java @@ -9,6 +9,8 @@ */ package org.eclipse.dirigible.components.base.artefact.topology; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.instrumentation.annotations.WithSpan; import org.eclipse.dirigible.components.base.artefact.ArtefactPhase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +36,12 @@ public class TopologicalDepleter { * @param flow the flow * @return the list */ + @WithSpan public Set deplete(Set list, ArtefactPhase flow) { + Span span = Span.current(); + span.setAttribute("flow", flow.getValue()); + span.setAttribute("artifacts.count", list.size()); + Set depletables = new HashSet<>(); depletables.addAll(list); int count = depletables.size(); diff --git a/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/synchronizer/BaseSynchronizer.java b/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/synchronizer/BaseSynchronizer.java index 5a55c82cb1a..26b7ae17586 100644 --- a/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/synchronizer/BaseSynchronizer.java +++ b/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/synchronizer/BaseSynchronizer.java @@ -9,6 +9,9 @@ */ package org.eclipse.dirigible.components.base.synchronizer; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; import org.eclipse.dirigible.components.base.artefact.Artefact; import org.eclipse.dirigible.components.base.artefact.ArtefactLifecycle; import org.eclipse.dirigible.components.base.artefact.ArtefactPhase; @@ -16,22 +19,54 @@ import org.eclipse.dirigible.components.base.spring.BeanProvider; import org.eclipse.dirigible.components.base.tenant.TenantContext; import org.eclipse.dirigible.components.base.tenant.TenantResult; +import org.eclipse.dirigible.components.open.telemetry.OpenTelemetryProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; import java.util.List; /** * The Class BaseSynchronizer. - * */ public abstract class BaseSynchronizer implements Synchronizer { /** The Constant logger. */ private static final Logger logger = LoggerFactory.getLogger(BaseSynchronizer.class); + @Override + public final List parse(String location, byte[] content) throws ParseException { + Tracer tracer = OpenTelemetryProvider.get() + .getTracer("eclipse-dirigible"); + + Span span = tracer.spanBuilder(getSynchronizerSpanPrefix() + "parse_execution") + .startSpan(); + + try (Scope scope = span.makeCurrent()) { + span.setAttribute("location", location); + + return parseImpl(location, content); + + } catch (RuntimeException e) { + span.recordException(e); + span.setStatus(io.opentelemetry.api.trace.StatusCode.ERROR, "Exception occurred during synchronization"); + + throw e; + } finally { + span.end(); + } + } + + private String getSynchronizerSpanPrefix() { + String synchronizerClassName = this.getClass() + .getSimpleName(); + return "synchronizer_" + synchronizerClassName + "_"; + } + + protected abstract List parseImpl(String location, byte[] content) throws ParseException; + /** * Complete. * @@ -41,6 +76,27 @@ public abstract class BaseSynchronizer implements Synchr */ @Override public final boolean complete(TopologyWrapper wrapper, ArtefactPhase flow) { + Tracer tracer = OpenTelemetryProvider.get() + .getTracer("eclipse-dirigible"); + + Span span = tracer.spanBuilder(getSynchronizerSpanPrefix() + "complete_execution") + .startSpan(); + + try (Scope scope = span.makeCurrent()) { + addSpanAttributes(span, wrapper, flow); + return completeInternal(wrapper, flow); + + } catch (RuntimeException e) { + span.recordException(e); + span.setStatus(io.opentelemetry.api.trace.StatusCode.ERROR, "Exception occurred during synchronization"); + + throw e; + } finally { + span.end(); + } + } + + private boolean completeInternal(TopologyWrapper wrapper, ArtefactPhase flow) { A artefact = wrapper.getArtefact(); ArtefactLifecycle lifecycle = artefact.getLifecycle(); @@ -60,7 +116,7 @@ public final boolean complete(TopologyWrapper wrapper, ArtefactPhase flow) { return results.stream() .map(TenantResult::getResult) - .allMatch(r -> Boolean.TRUE.equals(r)); + .allMatch(Boolean.TRUE::equals); } /** @@ -92,12 +148,46 @@ protected boolean isMultitenantArtefact(A artefact) { */ protected abstract boolean completeImpl(TopologyWrapper wrapper, ArtefactPhase flow); + private void addSpanAttributes(Span span, TopologyWrapper wrapper, ArtefactPhase phase) { + addSpanAttributes(wrapper.getArtefact(), span); + + span.setAttribute("phase", phase.getValue()); + } + + private void addSpanAttributes(A artefact, Span span) { + span.setAttribute("synchronizer", this.getClass() + .getName()); + span.setAttribute("artefact.key", artefact.getKey()); + } + /** * Cleanup. * * @param artefact the artefact */ public final void cleanup(A artefact) { + Tracer tracer = OpenTelemetryProvider.get() + .getTracer("eclipse-dirigible"); + + Span span = tracer.spanBuilder(getSynchronizerSpanPrefix() + "cleanup_execution") + .startSpan(); + + try (Scope scope = span.makeCurrent()) { + addSpanAttributes(artefact, span); + + cleanupInternal(artefact); + + } catch (RuntimeException e) { + span.recordException(e); + span.setStatus(io.opentelemetry.api.trace.StatusCode.ERROR, "Exception occurred during synchronization"); + + throw e; + } finally { + span.end(); + } + } + + private void cleanupInternal(A artefact) { if (!multitenantExecution() || !isMultitenantArtefact(artefact)) { logger.debug("[{} will cleanup artefact [{}]", this, artefact); cleanupImpl(artefact); diff --git a/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionPointsSynchronizer.java b/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionPointsSynchronizer.java index 117e6b39cfe..64b2d8fd648 100644 --- a/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionPointsSynchronizer.java +++ b/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionPointsSynchronizer.java @@ -79,7 +79,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { ExtensionPoint extensionPoint = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), ExtensionPoint.class); Configuration.configureObject(extensionPoint); extensionPoint.setLocation(location); diff --git a/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionsSynchronizer.java b/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionsSynchronizer.java index e6cf843ae79..739fbce4b17 100644 --- a/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionsSynchronizer.java +++ b/components/core/core-extensions/src/main/java/org/eclipse/dirigible/components/extensions/synchronizer/ExtensionsSynchronizer.java @@ -78,7 +78,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Extension extension = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Extension.class); Configuration.configureObject(extension); extension.setLocation(location); diff --git a/components/core/core-initializers/src/main/java/org/eclipse/dirigible/components/initializers/synchronizer/SynchronizationJob.java b/components/core/core-initializers/src/main/java/org/eclipse/dirigible/components/initializers/synchronizer/SynchronizationJob.java index 160abb970dd..fa11be94d64 100644 --- a/components/core/core-initializers/src/main/java/org/eclipse/dirigible/components/initializers/synchronizer/SynchronizationJob.java +++ b/components/core/core-initializers/src/main/java/org/eclipse/dirigible/components/initializers/synchronizer/SynchronizationJob.java @@ -75,7 +75,8 @@ protected SimpleScheduleBuilder getSchedule() { */ @Override protected String getJobKey() { - return "SynchronizationJobDetail"; + return this.getClass() + .getSimpleName(); } /** diff --git a/components/data/data-csvim/src/main/java/org/eclipse/dirigible/components/data/csvim/synchronizer/CsvimSynchronizer.java b/components/data/data-csvim/src/main/java/org/eclipse/dirigible/components/data/csvim/synchronizer/CsvimSynchronizer.java index f3d2c439e3d..1d354869463 100644 --- a/components/data/data-csvim/src/main/java/org/eclipse/dirigible/components/data/csvim/synchronizer/CsvimSynchronizer.java +++ b/components/data/data-csvim/src/main/java/org/eclipse/dirigible/components/data/csvim/synchronizer/CsvimSynchronizer.java @@ -112,7 +112,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Csvim csvim = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Csvim.class); Configuration.configureObject(csvim); csvim.setLocation(location); diff --git a/components/data/data-sources/src/main/java/org/eclipse/dirigible/components/data/sources/synchronizer/DataSourcesSynchronizer.java b/components/data/data-sources/src/main/java/org/eclipse/dirigible/components/data/sources/synchronizer/DataSourcesSynchronizer.java index e370f14e2f4..ccee572e3d0 100644 --- a/components/data/data-sources/src/main/java/org/eclipse/dirigible/components/data/sources/synchronizer/DataSourcesSynchronizer.java +++ b/components/data/data-sources/src/main/java/org/eclipse/dirigible/components/data/sources/synchronizer/DataSourcesSynchronizer.java @@ -82,7 +82,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { DataSource datasource = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), DataSource.class); Configuration.configureObject(datasource); datasource.setLocation(location); diff --git a/components/data/data-store/src/main/java/org/eclipse/dirigible/components/data/store/synchronizer/EntitySynchronizer.java b/components/data/data-store/src/main/java/org/eclipse/dirigible/components/data/store/synchronizer/EntitySynchronizer.java index c6700644bad..c4d6aca25ef 100644 --- a/components/data/data-store/src/main/java/org/eclipse/dirigible/components/data/store/synchronizer/EntitySynchronizer.java +++ b/components/data/data-store/src/main/java/org/eclipse/dirigible/components/data/store/synchronizer/EntitySynchronizer.java @@ -91,7 +91,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Entity entity = new Entity(); entity.setLocation(location); entity.setName(Paths.get(location) diff --git a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java index ed02a3387da..1a4097fb1c6 100644 --- a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java +++ b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java @@ -124,7 +124,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { final Schema schema = parseSchema(location, new String(content, StandardCharsets.UTF_8)); Configuration.configureObject(schema); schema.setLocation(location); diff --git a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/TablesSynchronizer.java b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/TablesSynchronizer.java index 4ca638f8f0a..7f61641b591 100644 --- a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/TablesSynchronizer.java +++ b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/TablesSynchronizer.java @@ -89,7 +89,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List
parseImpl(String location, byte[] content) throws ParseException { Table table = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Table.class); Configuration.configureObject(table); table.setLocation(location); diff --git a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/ViewsSynchronizer.java b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/ViewsSynchronizer.java index 8281920a214..26cbcf61311 100644 --- a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/ViewsSynchronizer.java +++ b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/ViewsSynchronizer.java @@ -90,7 +90,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { View view = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), View.class); Configuration.configureObject(view); view.setLocation(location); diff --git a/components/engine/engine-bpm-flowable/pom.xml b/components/engine/engine-bpm-flowable/pom.xml index 95b507d82c9..665618f470d 100644 --- a/components/engine/engine-bpm-flowable/pom.xml +++ b/components/engine/engine-bpm-flowable/pom.xml @@ -46,7 +46,11 @@ - + + org.flowable + flowable-spring-boot-starter-actuator + ${flowable.version} + org.flowable flowable-spring diff --git a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmFlowableConfig.java b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmFlowableConfig.java index 34e075c33d3..5890509c84d 100644 --- a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmFlowableConfig.java +++ b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmFlowableConfig.java @@ -11,12 +11,13 @@ import org.eclipse.dirigible.components.engine.bpm.BpmProvider; import org.eclipse.dirigible.components.engine.bpm.flowable.provider.BpmProviderFlowable; -import org.eclipse.dirigible.repository.api.IRepository; +import org.flowable.engine.ProcessEngine; +import org.flowable.spring.boot.actuate.endpoint.ProcessEngineEndpoint; +import org.flowable.spring.boot.actuate.info.FlowableInfoContributor; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration; import org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.jdbc.datasource.DataSourceTransactionManager; @@ -36,15 +37,23 @@ public BpmProvider getBpmProvider(BpmProviderFlowable bpmProviderFlowable) { return bpmProviderFlowable; } - @Bean(destroyMethod = "cleanup") - BpmProviderFlowable provideBpmProviderFlowable(@Qualifier("SystemDB") DataSource datasource, IRepository repository, - ApplicationContext applicationContext, DataSourceTransactionManager dataSourceTransactionManager) { - return new BpmProviderFlowable(datasource, repository, dataSourceTransactionManager, applicationContext); - } - @Bean DataSourceTransactionManager provideTransactionManager(@Qualifier("SystemDB") DataSource dataSource) { return new DataSourceTransactionManager(dataSource); } + /** + * Enable actuator flowable endpoint + */ + @Bean + ProcessEngineEndpoint processEngineEndpoint(BpmProviderFlowable bpmProviderFlowable) { + ProcessEngine engine = bpmProviderFlowable.getProcessEngine(); + return new ProcessEngineEndpoint(engine); + } + + @Bean + FlowableInfoContributor flowableInfoContributor() { + return new FlowableInfoContributor(); + } + } diff --git a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmProviderFlowableFactoryBean.java b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmProviderFlowableFactoryBean.java new file mode 100644 index 00000000000..8a9dcbe55f3 --- /dev/null +++ b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/config/BpmProviderFlowableFactoryBean.java @@ -0,0 +1,60 @@ +package org.eclipse.dirigible.components.engine.bpm.flowable.config; + +import org.eclipse.dirigible.components.engine.bpm.flowable.provider.BpmProviderFlowable; +import org.eclipse.dirigible.repository.api.IRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.stereotype.Component; + +import javax.sql.DataSource; + +@Component +class BpmProviderFlowableFactoryBean implements FactoryBean, DisposableBean, ApplicationContextAware { + + private static final Logger LOGGER = LoggerFactory.getLogger(BpmProviderFlowableFactoryBean.class); + + private BpmProviderFlowable bpmProviderFlowable; + private ApplicationContext applicationContext; + + @Override + public void destroy() { + LOGGER.info("Destroying bean..."); + if (bpmProviderFlowable != null) { + bpmProviderFlowable.cleanup(); + this.bpmProviderFlowable = null; + } + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + this.applicationContext = applicationContext; + } + + @Override + public boolean isSingleton() { + return true; + } + + @Override + public synchronized BpmProviderFlowable getObject() { + if (null == bpmProviderFlowable) { + DataSource datasource = applicationContext.getBean("SystemDB", DataSource.class); + IRepository repository = applicationContext.getBean(IRepository.class); + DataSourceTransactionManager dataSourceTransactionManager = applicationContext.getBean(DataSourceTransactionManager.class); + + bpmProviderFlowable = new BpmProviderFlowable(datasource, repository, dataSourceTransactionManager, applicationContext); + } + return bpmProviderFlowable; + } + + @Override + public Class getObjectType() { + return BpmProviderFlowable.class; + } + +} diff --git a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/delegate/DirigibleCallDelegate.java b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/delegate/DirigibleCallDelegate.java index ea2a901c9f2..bb2e3d42434 100644 --- a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/delegate/DirigibleCallDelegate.java +++ b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/delegate/DirigibleCallDelegate.java @@ -9,9 +9,13 @@ */ package org.eclipse.dirigible.components.engine.bpm.flowable.delegate; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; import jakarta.annotation.Nullable; import org.eclipse.dirigible.commons.api.helpers.GsonHelper; import org.eclipse.dirigible.components.engine.bpm.flowable.dto.ExecutionData; +import org.eclipse.dirigible.components.open.telemetry.OpenTelemetryProvider; import org.eclipse.dirigible.graalium.core.DirigibleJavascriptCodeRunner; import org.eclipse.dirigible.repository.api.RepositoryPath; import org.flowable.engine.delegate.BpmnError; @@ -205,7 +209,39 @@ public void setType(FixedValue type) { */ @Override public void execute(DelegateExecution execution) { + Tracer tracer = OpenTelemetryProvider.get() + .getTracer("eclipse-dirigible"); + Span span = tracer.spanBuilder("flowable_task_execution") + .startSpan(); + try (Scope scope = span.makeCurrent()) { + addSpanAttributes(execution, span); + + executeInternal(execution); + } catch (RuntimeException e) { + span.recordException(e); + span.setStatus(io.opentelemetry.api.trace.StatusCode.ERROR, "Exception occurred during task execution"); + + throw e; + } finally { + span.end(); + } + } + + private void addSpanAttributes(DelegateExecution execution, Span span) { + String executionId = execution.getId(); + span.setAttribute("execution.id", executionId); + + String processInstanceId = execution.getProcessInstanceId(); + span.setAttribute("process.instance.id", processInstanceId); + String processInstanceBusinessKey = execution.getProcessInstanceBusinessKey(); + span.setAttribute("process.instance.business.key", processInstanceBusinessKey); + + String processDefinitionId = execution.getProcessDefinitionId(); + span.setAttribute("process.definition.id", processDefinitionId); + } + + private void executeInternal(DelegateExecution execution) { String action = (String) execution.getVariable(DIRIGIBLE_BPM_INTERNAL_SKIP_STEP); if (SKIP.getActionName() .equals(action)) { @@ -235,6 +271,9 @@ private void executeJSHandler(Map context) { RepositoryPath path = new RepositoryPath(handler.getExpressionText()); JSTask task = JSTask.fromRepositoryPath(path); + Span.current() + .setAttribute("handler", path.toString()); + try (DirigibleJavascriptCodeRunner runner = new DirigibleJavascriptCodeRunner(context, false)) { Source source = runner.prepareSource(task.getSourceFilePath()); Value value = runner.run(source); diff --git a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/open/telemetry/FlowableMetricsConfigurator.java b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/open/telemetry/FlowableMetricsConfigurator.java new file mode 100644 index 00000000000..3ced1b1f671 --- /dev/null +++ b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/open/telemetry/FlowableMetricsConfigurator.java @@ -0,0 +1,111 @@ +package org.eclipse.dirigible.components.engine.bpm.flowable.open.telemetry; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.Meter; +import org.eclipse.dirigible.components.engine.bpm.flowable.provider.BpmProviderFlowable; +import org.flowable.spring.boot.actuate.endpoint.ProcessEngineEndpoint; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +import java.util.Date; + +@Component +class FlowableMetricsConfigurator implements ApplicationListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlowableMetricsConfigurator.class); + + private static final String METRIC_PREFIX = "flowable_"; + private static final String METER_SCOPE_NAME = "dirigible_flowable"; + + private final BpmProviderFlowable bpmProviderFlowable; + private final OpenTelemetry openTelemetry; + + FlowableMetricsConfigurator(BpmProviderFlowable bpmProviderFlowable, OpenTelemetry openTelemetry) { + this.bpmProviderFlowable = bpmProviderFlowable; + this.openTelemetry = openTelemetry; + } + + /** + * Create meters based on the logic in the Flowable actuator endpoint in + * {@link ProcessEngineEndpoint} class + * + * @param event app ready event + */ + @Override + public void onApplicationEvent(ApplicationReadyEvent event) { + LOGGER.debug("Configuring flowable metrics..."); + + Meter meter = openTelemetry.getMeter(METER_SCOPE_NAME); + + meter.gaugeBuilder(METRIC_PREFIX + "processDefinitionCount") + .setDescription("Total number of flowable process definitions") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(bpmProviderFlowable.getProcessEngine() + .getRepositoryService() + .createProcessDefinitionQuery() + .count())); + + meter.gaugeBuilder(METRIC_PREFIX + "runningProcessInstanceCount") + .setDescription("Total number of flowable running process instances") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(bpmProviderFlowable.getProcessEngine() + .getRuntimeService() + .createProcessInstanceQuery() + .count())); + + meter.gaugeBuilder(METRIC_PREFIX + "completedProcessInstanceCount") + .setDescription("Total number of completed flowable process instances") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(bpmProviderFlowable.getProcessEngine() + .getHistoryService() + .createHistoricProcessInstanceQuery() + .finished() + .count())); + + meter.gaugeBuilder(METRIC_PREFIX + "openTaskCount") + .setDescription("Total number of flowable open tasks") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(bpmProviderFlowable.getProcessEngine() + .getTaskService() + .createTaskQuery() + .count())); + + meter.gaugeBuilder(METRIC_PREFIX + "completedTaskCount") + .setDescription("Total number of flowable completed tasks") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(bpmProviderFlowable.getProcessEngine() + .getHistoryService() + .createHistoricTaskInstanceQuery() + .finished() + .count())); + meter.gaugeBuilder(METRIC_PREFIX + "completedTaskCountToday") + .setDescription("Total number of flowable completed tasks for today") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(bpmProviderFlowable.getProcessEngine() + .getHistoryService() + .createHistoricTaskInstanceQuery() + .finished() + .taskCompletedAfter(new Date(System.currentTimeMillis() + - secondsForDays(1))) + .count())); + + meter.gaugeBuilder(METRIC_PREFIX + "completedActivities") + .setDescription("Total number of flowable completed activities") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(bpmProviderFlowable.getProcessEngine() + .getHistoryService() + .createHistoricActivityInstanceQuery() + .finished() + .count())); + } + + private long secondsForDays(int days) { + int hour = 60 * 60 * 1000; + int day = 24 * hour; + return (long) days * day; + } + +} diff --git a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/provider/BpmProviderFlowable.java b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/provider/BpmProviderFlowable.java index 10268a216f1..fa3508dee98 100644 --- a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/provider/BpmProviderFlowable.java +++ b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/provider/BpmProviderFlowable.java @@ -366,7 +366,10 @@ public void removeVariable(String executionId, String variableName) { public void cleanup() { logger.info("Cleaning [{}]...", this.getClass()); - processEngine.close(); + if (null != processEngine) { + ProcessEngines.destroy(); + processEngine = null; + } } } diff --git a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/synchronizer/BpmnSynchronizer.java b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/synchronizer/BpmnSynchronizer.java index ee733344ce4..5374f5c6bdd 100644 --- a/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/synchronizer/BpmnSynchronizer.java +++ b/components/engine/engine-bpm-flowable/src/main/java/org/eclipse/dirigible/components/engine/bpm/flowable/synchronizer/BpmnSynchronizer.java @@ -87,7 +87,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Bpmn bpmn = new Bpmn(); bpmn.setLocation(location); bpmn.setName(Paths.get(location) diff --git a/components/engine/engine-camel/pom.xml b/components/engine/engine-camel/pom.xml index 26eda2c2787..977d8d61bbb 100644 --- a/components/engine/engine-camel/pom.xml +++ b/components/engine/engine-camel/pom.xml @@ -72,8 +72,8 @@ - + org.eclipse.dirigible @@ -86,20 +86,25 @@ + + org.apache.camel.springboot + camel-opentelemetry-starter + + + org.apache.camel.springboot + camel-micrometer-starter + org.apache.camel.springboot camel-spring-boot-starter - ${camel.version} org.apache.camel.springboot camel-platform-http-starter - ${camel.version} org.apache.camel.springboot camel-yaml-dsl-starter - ${camel.version} org.apache.camel @@ -152,6 +157,19 @@ + + + + org.apache.camel.springboot + camel-spring-boot-bom + ${camel.version} + pom + import + + + + + diff --git a/components/engine/engine-camel/src/main/java/org/eclipse/dirigible/components/engine/camel/synchronizer/CamelSynchronizer.java b/components/engine/engine-camel/src/main/java/org/eclipse/dirigible/components/engine/camel/synchronizer/CamelSynchronizer.java index 6ce55f62c2e..077b03efda2 100644 --- a/components/engine/engine-camel/src/main/java/org/eclipse/dirigible/components/engine/camel/synchronizer/CamelSynchronizer.java +++ b/components/engine/engine-camel/src/main/java/org/eclipse/dirigible/components/engine/camel/synchronizer/CamelSynchronizer.java @@ -9,6 +9,8 @@ */ package org.eclipse.dirigible.components.engine.camel.synchronizer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; import org.eclipse.dirigible.components.base.artefact.ArtefactLifecycle; import org.eclipse.dirigible.components.base.artefact.ArtefactPhase; import org.eclipse.dirigible.components.base.artefact.ArtefactService; @@ -77,7 +79,7 @@ public boolean isAccepted(String type) { * @return the list */ @Override - public List parse(String location, byte[] content) { + protected List parseImpl(String location, byte[] content) { Camel camel = new Camel(); camel.setLocation(location); camel.setName(Paths.get(location) @@ -207,7 +209,11 @@ protected boolean completeImpl(TopologyWrapper wrapper, ArtefactPhase flo * @param camel the camel */ private void addToProcessor(Camel camel) { - camelProcessor.onCreateOrUpdate(camel); + // remove trace context to decouple async routes from the synchronizer job + Context rootContext = Context.root(); + try (Scope scope = rootContext.makeCurrent()) { + camelProcessor.onCreateOrUpdate(camel); + } } /** @@ -216,8 +222,13 @@ private void addToProcessor(Camel camel) { * @param camel the camel */ private void removeFromProcessor(Camel camel) { - getService().delete(camel); - camelProcessor.onRemove(camel); + // remove trace context to decouple async routes from the synchronizer job + Context rootContext = Context.root(); + try (Scope scope = rootContext.makeCurrent()) { + getService().delete(camel); + camelProcessor.onRemove(camel); + } + } /** diff --git a/components/engine/engine-javascript/src/main/java/org/eclipse/dirigible/components/engine/javascript/endpoint/JavascriptEndpoint.java b/components/engine/engine-javascript/src/main/java/org/eclipse/dirigible/components/engine/javascript/endpoint/JavascriptEndpoint.java index 805ba3a11b2..73c6dc243d9 100644 --- a/components/engine/engine-javascript/src/main/java/org/eclipse/dirigible/components/engine/javascript/endpoint/JavascriptEndpoint.java +++ b/components/engine/engine-javascript/src/main/java/org/eclipse/dirigible/components/engine/javascript/endpoint/JavascriptEndpoint.java @@ -9,15 +9,8 @@ */ package org.eclipse.dirigible.components.engine.javascript.endpoint; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import org.eclipse.dirigible.components.base.endpoint.BaseEndpoint; import org.eclipse.dirigible.components.engine.javascript.service.JavascriptService; import org.eclipse.dirigible.graalium.core.JavascriptSourceProvider; @@ -33,18 +26,17 @@ import org.springframework.lang.Nullable; import org.springframework.util.MultiValueMap; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PatchMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.server.ResponseStatusException; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + /** * The Class JavascriptEndpoint. */ @@ -57,14 +49,16 @@ public class JavascriptEndpoint extends BaseEndpoint { /** The Constant HTTP_PATH_MATCHER. */ private static final String HTTP_PATH_MATCHER = "/{projectName}/{*projectFilePath}"; - - + /** The Constant CJS. */ + private static final String CJS = ".cjs/"; + /** The Constant MJS. */ + private static final String MJS = ".mjs/"; + /** The Constant JS. */ + private static final String JS = ".js/"; /** The javascript service. */ private final JavascriptService javascriptService; - /** The repository. */ private final IRepository repository; - /** The source provider. */ private final JavascriptSourceProvider sourceProvider = new DirigibleSourceProvider(); @@ -80,6 +74,43 @@ public JavascriptEndpoint(JavascriptService javascriptService, IRepository repos this.repository = repository; } + /** + * The Dts. + */ + record Dts(String content, String moduleName, String filePath) { + + /** + * From dts path. + * + * @param dtsDirRoot the dts dir root + * @param dtsPath the dts path + * @return the dts + */ + static Dts fromDtsPath(Path dtsDirRoot, Path dtsPath) { + String content = readAllText(dtsPath); + Path relativePath = dtsDirRoot.relativize(dtsPath); + String filePath = "file:///node_modules/sdk/" + relativePath; + String moduleName = ("sdk/" + relativePath).replace("index.d.ts", "") + .replace(".d.ts", ""); + return new Dts(content, moduleName, filePath); + } + + /** + * Read all text. + * + * @param path the path + * @return the string + */ + private static String readAllText(Path path) { + try { + byte[] bytes = Files.readAllBytes(path); + return new String(bytes, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + /** * Gets the dts. * @@ -114,99 +145,10 @@ public List getDTS() throws IOException { * @param params the params * @return the response */ + @WithSpan @GetMapping(HTTP_PATH_MATCHER) - public ResponseEntity get(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, - @Nullable @RequestParam(required = false) MultiValueMap params) { - return executeJavaScript(projectName, projectFilePath, params, null); - } - - /** - * Post. - * - * @param projectName the project name - * @param projectFilePath the project file path - * @param params the params - * @return the response - */ - @PostMapping(HTTP_PATH_MATCHER) - public ResponseEntity post(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, - @Nullable @RequestParam(required = false) MultiValueMap params) { - return executeJavaScript(projectName, projectFilePath, params, null); - } - - /** - * Post. - * - * @param projectName the project name - * @param projectFilePath the project file path - * @param params the params - * @param file the file - * @return the response - */ - @PostMapping(value = HTTP_PATH_MATCHER, consumes = "multipart/form-data") - public ResponseEntity postFile(@PathVariable("projectName") String projectName, - @PathVariable("projectFilePath") String projectFilePath, - @Nullable @RequestParam(required = false) MultiValueMap params, - @Validated @RequestParam("file") MultipartFile[] file) { - return executeJavaScript(projectName, projectFilePath, params, file); - } - - /** - * Put. - * - * @param projectName the project name - * @param projectFilePath the project file path - * @param params the params - * @return the response - */ - @PutMapping(HTTP_PATH_MATCHER) - public ResponseEntity put(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, - @Nullable @RequestParam(required = false) MultiValueMap params) { - return executeJavaScript(projectName, projectFilePath, params, null); - } - - /** - * Put. - * - * @param projectName the project name - * @param projectFilePath the project file path - * @param params the params - * @param file the file - * @return the response - */ - @PutMapping(value = HTTP_PATH_MATCHER, consumes = "multipart/form-data") - public ResponseEntity putFile(@PathVariable("projectName") String projectName, - @PathVariable("projectFilePath") String projectFilePath, - @Nullable @RequestParam(required = false) MultiValueMap params, - @Validated @RequestParam("file") MultipartFile file) { - return executeJavaScript(projectName, projectFilePath, params, new MultipartFile[] {file}); - } - - /** - * Patch. - * - * @param projectName the project name - * @param projectFilePath the project file path - * @param params the params - * @return the response - */ - @PatchMapping(HTTP_PATH_MATCHER) - public ResponseEntity patch(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, - @Nullable @RequestParam(required = false) MultiValueMap params) { - return executeJavaScript(projectName, projectFilePath, params, null); - } - - /** - * Delete. - * - * @param projectName the project name - * @param projectFilePath the project file path - * @param params the params - * @return the response - */ - @DeleteMapping(HTTP_PATH_MATCHER) - public ResponseEntity delete(@PathVariable("projectName") String projectName, - @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity get(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params) { return executeJavaScript(projectName, projectFilePath, params, null); } @@ -227,16 +169,6 @@ private ResponseEntity executeJavaScript(String projectName, String projectFi return executeJavaScript(projectName, projectFilePath, projectFilePathParam, params, files); } - - /** The Constant CJS. */ - private static final String CJS = ".cjs/"; - - /** The Constant MJS. */ - private static final String MJS = ".mjs/"; - - /** The Constant JS. */ - private static final String JS = ".js/"; - /** * Extract project file path. * @@ -294,9 +226,7 @@ protected ResponseEntity executeJavaScript(String projectName, String project } List filesList = new ArrayList(); if (files != null) { - for (MultipartFile file : files) { - filesList.add(file); - } + Collections.addAll(filesList, files); context.put("files", filesList); } @@ -319,16 +249,6 @@ protected JavascriptService getJavascriptHandler() { return javascriptService; } - /** - * Gets the dirigible working directory. - * - * @return the dirigible working directory - */ - protected java.nio.file.Path getDirigibleWorkingDirectory() { - String publicRegistryPath = repository.getInternalResourcePath(IRepositoryStructure.PATH_REGISTRY_PUBLIC); - return java.nio.file.Path.of(publicRegistryPath); - } - /** * Checks if is valid. * @@ -349,6 +269,16 @@ public boolean isValid(String inputPath) { } } + /** + * Gets the dirigible working directory. + * + * @return the dirigible working directory + */ + protected java.nio.file.Path getDirigibleWorkingDirectory() { + String publicRegistryPath = repository.getInternalResourcePath(IRepositoryStructure.PATH_REGISTRY_PUBLIC); + return java.nio.file.Path.of(publicRegistryPath); + } + /** * Normalize path. * @@ -365,39 +295,102 @@ protected String normalizePath(String path) { } /** - * The Dts. + * Post. + * + * @param projectName the project name + * @param projectFilePath the project file path + * @param params the params + * @return the response */ - record Dts(String content, String moduleName, String filePath) { + @WithSpan + @PostMapping(HTTP_PATH_MATCHER) + public ResponseEntity post(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, + @Nullable @RequestParam(required = false) MultiValueMap params) { + return executeJavaScript(projectName, projectFilePath, params, null); + } - /** - * From dts path. - * - * @param dtsDirRoot the dts dir root - * @param dtsPath the dts path - * @return the dts - */ - static Dts fromDtsPath(Path dtsDirRoot, Path dtsPath) { - String content = readAllText(dtsPath); - Path relativePath = dtsDirRoot.relativize(dtsPath); - String filePath = "file:///node_modules/sdk/" + relativePath; - String moduleName = ("sdk/" + relativePath).replace("index.d.ts", "") - .replace(".d.ts", ""); - return new Dts(content, moduleName, filePath); - } + /** + * Post. + * + * @param projectName the project name + * @param projectFilePath the project file path + * @param params the params + * @param file the file + * @return the response + */ + @WithSpan + @PostMapping(value = HTTP_PATH_MATCHER, consumes = "multipart/form-data") + public ResponseEntity postFile(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, + @Nullable @RequestParam(required = false) MultiValueMap params, + @Validated @RequestParam("file") MultipartFile[] file) { + return executeJavaScript(projectName, projectFilePath, params, file); + } - /** - * Read all text. - * - * @param path the path - * @return the string - */ - private static String readAllText(Path path) { - try { - byte[] bytes = Files.readAllBytes(path); - return new String(bytes, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + /** + * Put. + * + * @param projectName the project name + * @param projectFilePath the project file path + * @param params the params + * @return the response + */ + @WithSpan + @PutMapping(HTTP_PATH_MATCHER) + public ResponseEntity put(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, + @Nullable @RequestParam(required = false) MultiValueMap params) { + return executeJavaScript(projectName, projectFilePath, params, null); + } + + /** + * Put. + * + * @param projectName the project name + * @param projectFilePath the project file path + * @param params the params + * @param file the file + * @return the response + */ + @WithSpan + @PutMapping(value = HTTP_PATH_MATCHER, consumes = "multipart/form-data") + public ResponseEntity putFile(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, + @Nullable @RequestParam(required = false) MultiValueMap params, + @Validated @RequestParam("file") MultipartFile file) { + return executeJavaScript(projectName, projectFilePath, params, new MultipartFile[] {file}); + } + + /** + * Patch. + * + * @param projectName the project name + * @param projectFilePath the project file path + * @param params the params + * @return the response + */ + @PatchMapping(HTTP_PATH_MATCHER) + @WithSpan("java_script_endpoint_patch") + public ResponseEntity patch(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, + @Nullable @RequestParam(required = false) MultiValueMap params) { + return executeJavaScript(projectName, projectFilePath, params, null); + } + + /** + * Delete. + * + * @param projectName the project name + * @param projectFilePath the project file path + * @param params the params + * @return the response + */ + @WithSpan + @DeleteMapping(HTTP_PATH_MATCHER) + public ResponseEntity delete(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, + @Nullable @RequestParam(required = false) MultiValueMap params) { + return executeJavaScript(projectName, projectFilePath, params, null); } } diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/config/QuartzConfig.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/config/QuartzConfig.java index 3e883ff4033..9fe9ba3018d 100644 --- a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/config/QuartzConfig.java +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/config/QuartzConfig.java @@ -10,6 +10,9 @@ package org.eclipse.dirigible.components.jobs.config; import org.eclipse.dirigible.components.data.sources.config.SystemDataSourceName; +import org.eclipse.dirigible.components.jobs.telemetry.JobExecutionsCountListener; +import org.eclipse.dirigible.components.jobs.telemetry.JobExecutionsDurationListener; +import org.eclipse.dirigible.components.jobs.telemetry.JobFailuresCountListener; import org.quartz.Scheduler; import org.quartz.SchedulerException; import org.quartz.impl.jdbcjobstore.JobStoreTX; @@ -37,18 +40,11 @@ class QuartzConfig { /** The Constant logger. */ private static final Logger logger = LoggerFactory.getLogger(QuartzConfig.class); - /** - * Scheduler. - * - * @param factory the factory - * @param systemDataSource the system data source - * @param systemDataSourceName the system data source name - * @return the scheduler - * @throws SchedulerException the scheduler exception - */ @Bean Scheduler scheduler(SchedulerFactoryBean factory, @Qualifier("SystemDB") DataSource systemDataSource, - @SystemDataSourceName String systemDataSourceName) throws SchedulerException { + @SystemDataSourceName String systemDataSourceName, JobExecutionsCountListener jobExecutionsCountListener, + JobExecutionsDurationListener jobExecutionsDurationListener, JobFailuresCountListener jobFailuresCountListener) + throws SchedulerException { factory.setDataSource(systemDataSource); DBConnectionManager.getInstance() .addConnectionProvider(systemDataSourceName, new CustomConnectionProvider(systemDataSource)); @@ -59,6 +55,13 @@ Scheduler scheduler(SchedulerFactoryBean factory, @Qualifier("SystemDB") DataSou // give some time for spring auto configurations to pass before starting scheduler.startDelayed(STARTUP_DELAY_SECONDS); + scheduler.getListenerManager() + .addJobListener(jobExecutionsCountListener); + scheduler.getListenerManager() + .addJobListener(jobExecutionsDurationListener); + scheduler.getListenerManager() + .addJobListener(jobFailuresCountListener); + return scheduler; } diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/handler/JobHandler.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/handler/JobHandler.java index 674fd276deb..13cb9a52ba8 100644 --- a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/handler/JobHandler.java +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/handler/JobHandler.java @@ -9,6 +9,7 @@ */ package org.eclipse.dirigible.components.jobs.handler; +import io.opentelemetry.api.trace.Span; import org.eclipse.dirigible.components.base.tenant.Tenant; import org.eclipse.dirigible.components.base.tenant.TenantContext; import org.eclipse.dirigible.components.jobs.domain.JobLog; @@ -96,6 +97,8 @@ private void executeJob(JobExecutionContext context) throws JobExecutionExceptio JobDataMap params = context.getJobDetail() .getJobDataMap(); String handler = params.getString(JOB_PARAMETER_HANDLER); + Span.current() + .setAttribute("handler", handler); JobLog triggered = registerTriggered(name, handler); if (triggered != null) { diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/synchronizer/JobSynchronizer.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/synchronizer/JobSynchronizer.java index b0d9f3391e4..c2dc17cd2aa 100644 --- a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/synchronizer/JobSynchronizer.java +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/synchronizer/JobSynchronizer.java @@ -105,7 +105,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Job job = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Job.class); Configuration.configureObject(job); job.setLocation(location); diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsCountListener.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsCountListener.java new file mode 100644 index 00000000000..e668028fbd1 --- /dev/null +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsCountListener.java @@ -0,0 +1,32 @@ +package org.eclipse.dirigible.components.jobs.telemetry; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.JobKey; +import org.springframework.stereotype.Component; + +@Component +public class JobExecutionsCountListener extends OpenTelemetryListener { + + private final LongCounter counter; + + JobExecutionsCountListener(OpenTelemetry openTelemetry) { + this.counter = openTelemetry.getMeter(QuartzMetricConstants.METER_SCOPE_NAME) + .counterBuilder("quartz_job_executed_count") + .setDescription("Total number of executed jobs") + .build(); + } + + @Override + public void jobWasExecuted(JobExecutionContext context, JobExecutionException jobException) { + JobKey jobKey = context.getJobDetail() + .getKey(); + + counter.add(1, Attributes.of(AttributeKey.stringKey("job_name"), jobKey.getName(), AttributeKey.stringKey("job_group"), + jobKey.getGroup())); + } +} diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsDurationListener.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsDurationListener.java new file mode 100644 index 00000000000..f4638e3a963 --- /dev/null +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobExecutionsDurationListener.java @@ -0,0 +1,47 @@ +package org.eclipse.dirigible.components.jobs.telemetry; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongHistogram; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.JobKey; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; + +@Component +public class JobExecutionsDurationListener extends OpenTelemetryListener { + + private static final String START_TIME_KEY = "startTime"; + + private final LongHistogram histogram; + + JobExecutionsDurationListener(OpenTelemetry openTelemetry) { + this.histogram = openTelemetry.getMeter(QuartzMetricConstants.METER_SCOPE_NAME) + .histogramBuilder("quartz_job_execution_time") + .setDescription("Job execution duration in milliseconds") + .ofLongs() + .build(); + } + + @Override + public void jobToBeExecuted(JobExecutionContext context) { + context.put(START_TIME_KEY, Instant.now()); + } + + @Override + public void jobWasExecuted(JobExecutionContext context, JobExecutionException jobException) { + Instant startTime = (Instant) context.get(START_TIME_KEY); + Instant endTime = Instant.now(); + Duration duration = Duration.between(startTime, endTime); + + JobKey jobKey = context.getJobDetail() + .getKey(); + + histogram.record(duration.toMillis(), Attributes.of(AttributeKey.stringKey("job_name"), jobKey.getName(), + AttributeKey.stringKey("job_group"), jobKey.getGroup())); + } +} diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobFailuresCountListener.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobFailuresCountListener.java new file mode 100644 index 00000000000..4308c068631 --- /dev/null +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/JobFailuresCountListener.java @@ -0,0 +1,39 @@ +package org.eclipse.dirigible.components.jobs.telemetry; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.JobKey; +import org.springframework.stereotype.Component; + +@Component +public class JobFailuresCountListener extends OpenTelemetryListener { + + private final LongCounter counter; + + JobFailuresCountListener(OpenTelemetry openTelemetry) { + this.counter = openTelemetry.getMeter(QuartzMetricConstants.METER_SCOPE_NAME) + .counterBuilder("quartz_job_failed_count") + .setDescription("Total number of failed jobs") + .build(); + } + + @Override + public void jobWasExecuted(JobExecutionContext context, JobExecutionException jobException) { + JobKey jobKey = context.getJobDetail() + .getKey(); + if (jobException == null) { + counter.add(0, Attributes.of(AttributeKey.stringKey("job_name"), jobKey.getName(), AttributeKey.stringKey("job_group"), + jobKey.getGroup())); + } else { + String exceptionType = jobException.getClass() + .getName(); + + counter.add(1, Attributes.of(AttributeKey.stringKey("job_name"), jobKey.getName(), AttributeKey.stringKey("job_group"), + jobKey.getGroup(), AttributeKey.stringKey("exception_type"), exceptionType)); + } + } +} diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/OpenTelemetryListener.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/OpenTelemetryListener.java new file mode 100644 index 00000000000..c40bc4f7666 --- /dev/null +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/OpenTelemetryListener.java @@ -0,0 +1,13 @@ +package org.eclipse.dirigible.components.jobs.telemetry; + +import org.quartz.listeners.JobListenerSupport; + +abstract class OpenTelemetryListener extends JobListenerSupport { + + @Override + public final String getName() { + return this.getClass() + .getSimpleName(); + } + +} diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/QuartzMetricConstants.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/QuartzMetricConstants.java new file mode 100644 index 00000000000..a65ff4bcbe4 --- /dev/null +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/QuartzMetricConstants.java @@ -0,0 +1,7 @@ +package org.eclipse.dirigible.components.jobs.telemetry; + +public final class QuartzMetricConstants { + + static final String METER_SCOPE_NAME = "dirigible_quartz"; + +} diff --git a/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/RunningJobsCountMetricsConfigurator.java b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/RunningJobsCountMetricsConfigurator.java new file mode 100644 index 00000000000..6d607a20032 --- /dev/null +++ b/components/engine/engine-jobs/src/main/java/org/eclipse/dirigible/components/jobs/telemetry/RunningJobsCountMetricsConfigurator.java @@ -0,0 +1,39 @@ +package org.eclipse.dirigible.components.jobs.telemetry; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.Meter; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +@Component +class RunningJobsCountMetricsConfigurator implements ApplicationListener { + + private final OpenTelemetry openTelemetry; + private final Scheduler scheduler; + + RunningJobsCountMetricsConfigurator(OpenTelemetry openTelemetry, Scheduler scheduler) { + this.openTelemetry = openTelemetry; + this.scheduler = scheduler; + } + + @Override + public void onApplicationEvent(ApplicationReadyEvent event) { + Meter meter = openTelemetry.getMeter(QuartzMetricConstants.METER_SCOPE_NAME); + + meter.gaugeBuilder("quartz_scheduler_running_jobs") + .setDescription("Current number of running jobs") + .ofLongs() + .buildWithCallback(observation -> { + try { + observation.record(scheduler.getCurrentlyExecutingJobs() + .size()); + } catch (SchedulerException e) { + observation.record(0); + } + }); + } + +} diff --git a/components/engine/engine-listeners/src/main/java/org/eclipse/dirigible/components/listeners/synchronizer/ListenerSynchronizer.java b/components/engine/engine-listeners/src/main/java/org/eclipse/dirigible/components/listeners/synchronizer/ListenerSynchronizer.java index 29aefc59eb5..772d0cb69b1 100644 --- a/components/engine/engine-listeners/src/main/java/org/eclipse/dirigible/components/listeners/synchronizer/ListenerSynchronizer.java +++ b/components/engine/engine-listeners/src/main/java/org/eclipse/dirigible/components/listeners/synchronizer/ListenerSynchronizer.java @@ -75,7 +75,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Listener listener = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Listener.class); Configuration.configureObject(listener); listener.setLocation(location); diff --git a/components/engine/engine-odata/src/main/java/org/eclipse/dirigible/components/odata/synchronizer/ODataSynchronizer.java b/components/engine/engine-odata/src/main/java/org/eclipse/dirigible/components/odata/synchronizer/ODataSynchronizer.java index 4b3e419cee8..e8f60c5f6f9 100644 --- a/components/engine/engine-odata/src/main/java/org/eclipse/dirigible/components/odata/synchronizer/ODataSynchronizer.java +++ b/components/engine/engine-odata/src/main/java/org/eclipse/dirigible/components/odata/synchronizer/ODataSynchronizer.java @@ -83,6 +83,41 @@ public static OData parseOData(String location, byte[] content) { return parseOData(location, new String(content, StandardCharsets.UTF_8)); } + /** + * Parses the O data. + * + * @param location the location + * @param content the content + * @return the o data + */ + public static OData parseOData(String location, String content) { + OData odata = JsonHelper.fromJson(content, OData.class); + Configuration.configureObject(odata); + odata.setLocation(location); + odata.setType(OData.ARTEFACT_TYPE); + odata.setName(FilenameUtils.getBaseName(location)); + odata.setContent(content); + odata.updateKey(); + odata.getAssociations() + .forEach(association -> { + if (association.getFrom() + .getProperty() != null) { + association.getFrom() + .getProperties() + .add(association.getFrom() + .getProperty()); + } + if (association.getTo() + .getProperty() != null) { + association.getTo() + .getProperties() + .add(association.getTo() + .getProperty()); + } + }); + return odata; + } + /** * Checks if is accepted. * @@ -103,7 +138,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { OData odata = parseOData(location, new String(content, StandardCharsets.UTF_8)); try { OData maybe = getService().findByKey(odata.getKey()); @@ -126,41 +161,6 @@ public List parse(String location, byte[] content) throws ParseException return List.of(odata); } - /** - * Parses the O data. - * - * @param location the location - * @param content the content - * @return the o data - */ - public static OData parseOData(String location, String content) { - OData odata = JsonHelper.fromJson(content, OData.class); - Configuration.configureObject(odata); - odata.setLocation(location); - odata.setType(OData.ARTEFACT_TYPE); - odata.setName(FilenameUtils.getBaseName(location)); - odata.setContent(content); - odata.updateKey(); - odata.getAssociations() - .forEach(association -> { - if (association.getFrom() - .getProperty() != null) { - association.getFrom() - .getProperties() - .add(association.getFrom() - .getProperty()); - } - if (association.getTo() - .getProperty() != null) { - association.getTo() - .getProperties() - .add(association.getTo() - .getProperty()); - } - }); - return odata; - } - /** * Gets the service. * @@ -277,19 +277,6 @@ public void generateOData(OData odata) throws SQLException { } } - /** - * Cleanup O data. - * - * @param odata the odata - */ - public void cleanupOData(OData odata) { - // CLEAN UP LOGIC - odataSchemaService.removeSchema(odata.getLocation()); - odataContainerService.removeContainer(odata.getLocation()); - odataMappingService.removeMappings(odata.getLocation()); - odataHandlerService.removeHandlers(odata.getLocation()); - } - /** * Generate OData Schema. * @@ -323,6 +310,19 @@ private List generateODataHandlers(OData model) throws SQLExceptio return odata2ODataHTransformer.transform(model); } + /** + * Cleanup O data. + * + * @param odata the odata + */ + public void cleanupOData(OData odata) { + // CLEAN UP LOGIC + odataSchemaService.removeSchema(odata.getLocation()); + odataContainerService.removeContainer(odata.getLocation()); + odataMappingService.removeMappings(odata.getLocation()); + odataHandlerService.removeHandlers(odata.getLocation()); + } + /** * Cleanup. * diff --git a/components/engine/engine-open-telemetry/pom.xml b/components/engine/engine-open-telemetry/pom.xml new file mode 100644 index 00000000000..c5b4afb7cf9 --- /dev/null +++ b/components/engine/engine-open-telemetry/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + org.eclipse.dirigible + dirigible-components-parent + 11.0.0-SNAPSHOT + ../../pom.xml + + + Components - Engine - OpenTelemetry + dirigible-components-engine-open-telemetry + jar + + + ../../../licensing-header.txt + ../../../ + + + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-annotations + + + + + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom + 2.8.0 + pom + import + + + + + + + default + + true + + + + open-telemetry + + + io.opentelemetry.instrumentation + opentelemetry-spring-boot-starter + + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + + + + + diff --git a/components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryConfiguration.java b/components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryConfiguration.java new file mode 100644 index 00000000000..7dbda105d8c --- /dev/null +++ b/components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryConfiguration.java @@ -0,0 +1,18 @@ +package org.eclipse.dirigible.components.open.telemetry; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +@Profile("!open-telemetry") +@Configuration +public class OpenTelemetryConfiguration { + + @Bean + OpenTelemetry provideOpenTelemetry() { + return GlobalOpenTelemetry.get(); + } + +} diff --git a/components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryProvider.java b/components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryProvider.java new file mode 100644 index 00000000000..2cf953045fc --- /dev/null +++ b/components/engine/engine-open-telemetry/src/main/java/org/eclipse/dirigible/components/open/telemetry/OpenTelemetryProvider.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.components.open.telemetry; + +import io.opentelemetry.api.OpenTelemetry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +@Component +public class OpenTelemetryProvider implements ApplicationContextAware { + + private static final Logger LOGGER = LoggerFactory.getLogger(OpenTelemetryProvider.class); + + private static ApplicationContext context; + + @Override + public void setApplicationContext(ApplicationContext argApplicationContext) throws BeansException { + context = argApplicationContext; + } + + public static OpenTelemetry get() { + assertSpringInitialized(); + return context.getBean(OpenTelemetry.class); + } + + private static void assertSpringInitialized() { + if (!isInitialzed()) { + throw new IllegalStateException("Spring is not initialized yet."); + } + } + + private static boolean isInitialzed() { + return context != null; + } +} diff --git a/components/engine/engine-open-telemetry/src/main/resources/application-open-telemetry.properties b/components/engine/engine-open-telemetry/src/main/resources/application-open-telemetry.properties new file mode 100644 index 00000000000..f2836cbbc65 --- /dev/null +++ b/components/engine/engine-open-telemetry/src/main/resources/application-open-telemetry.properties @@ -0,0 +1,21 @@ +# OpenTelemetry properties +otel.sdk.disabled=false + +otel.exporter.otlp.endpoint=http://otel-collector:4318 + +#otel.propagators=tracecontext +otel.resource.attributes.deployment.environment=production +otel.resource.attributes.service.name=eclipse-dirigible-starter +otel.resource.attributes.service.namespace=eclipse-namespace + +# disable what is not needed for Dirigible +otel.instrumentation.common.default-enabled=true +otel.instrumentation.kafka.enabled=false +otel.instrumentation.spring-webflux.enabled=false +otel.instrumentation.r2dbc.enabled=false +otel.instrumentation.mongo.enabled=false + +# enable micrometer/actuator metrics +otel.instrumentation.micrometer.enabled=true + +management.metrics.distribution.percentiles-histogram.http.server.requests=true diff --git a/components/engine/engine-openapi/src/main/java/org/eclipse/dirigible/components/openapi/synchronizer/OpenAPISynchronizer.java b/components/engine/engine-openapi/src/main/java/org/eclipse/dirigible/components/openapi/synchronizer/OpenAPISynchronizer.java index 457c9b425d6..d9db5ae1e58 100644 --- a/components/engine/engine-openapi/src/main/java/org/eclipse/dirigible/components/openapi/synchronizer/OpenAPISynchronizer.java +++ b/components/engine/engine-openapi/src/main/java/org/eclipse/dirigible/components/openapi/synchronizer/OpenAPISynchronizer.java @@ -87,7 +87,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { OpenAPI openAPI = new OpenAPI(); // JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), OpenAPI.class); Configuration.configureObject(openAPI); diff --git a/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/AccessSynchronizer.java b/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/AccessSynchronizer.java index e7b78d1d3bb..1a5c7d65123 100644 --- a/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/AccessSynchronizer.java +++ b/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/AccessSynchronizer.java @@ -90,7 +90,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + public List parseImpl(String location, byte[] content) throws ParseException { Constraints constraints = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Constraints.class); Configuration.configureObject(constraints); diff --git a/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/RolesSynchronizer.java b/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/RolesSynchronizer.java index f63ce013b00..e273e96dc26 100644 --- a/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/RolesSynchronizer.java +++ b/components/engine/engine-security/src/main/java/org/eclipse/dirigible/components/security/synchronizer/RolesSynchronizer.java @@ -90,7 +90,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Role[] roles = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Role[].class); Integer roleIndex = 1; for (Role role : roles) { diff --git a/components/engine/engine-typescript/src/main/java/org/eclipse/dirigible/components/engine/typescript/TypeScriptEndpoint.java b/components/engine/engine-typescript/src/main/java/org/eclipse/dirigible/components/engine/typescript/TypeScriptEndpoint.java index cc4c44dce2c..cc6c58ae352 100644 --- a/components/engine/engine-typescript/src/main/java/org/eclipse/dirigible/components/engine/typescript/TypeScriptEndpoint.java +++ b/components/engine/engine-typescript/src/main/java/org/eclipse/dirigible/components/engine/typescript/TypeScriptEndpoint.java @@ -9,6 +9,8 @@ */ package org.eclipse.dirigible.components.engine.typescript; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import org.eclipse.dirigible.components.base.endpoint.BaseEndpoint; import org.eclipse.dirigible.components.engine.javascript.endpoint.JavascriptEndpoint; import org.springframework.beans.factory.annotation.Autowired; @@ -16,15 +18,7 @@ import org.springframework.lang.Nullable; import org.springframework.util.MultiValueMap; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PatchMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; /** @@ -58,12 +52,32 @@ public TypeScriptEndpoint(JavascriptEndpoint javascriptEndpoint) { * @param params the params * @return the response entity */ + @WithSpan @GetMapping(HTTP_PATH_MATCHER) - public ResponseEntity get(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity get(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params) { return javascriptEndpoint.get(projectName, replaceTSWithJSExtension(projectFilePath), params); } + /** + * Replace TS with MJS extension. + * + * @param projectFilePath the project file path + * @return the string + */ + private String replaceTSWithJSExtension(String projectFilePath) { + int indexOfExtensionStart = projectFilePath.lastIndexOf(".ts"); + if (indexOfExtensionStart == -1) { + throw new RuntimeException("Could not find .ts extension"); + } + + String projectFilePathWithoutExtension = projectFilePath.substring(0, indexOfExtensionStart); + String maybePathParameters = projectFilePath.substring(indexOfExtensionStart) + .replace(".ts", ""); // for decorators and rs api + return projectFilePathWithoutExtension + ".js" + maybePathParameters; + } + /** * Post. * @@ -72,8 +86,10 @@ public ResponseEntity get(@PathVariable("projectName") String projectName, @P * @param params the params * @return the response entity */ + @WithSpan @PostMapping(HTTP_PATH_MATCHER) - public ResponseEntity post(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity post(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params) { return javascriptEndpoint.post(projectName, replaceTSWithJSExtension(projectFilePath), params); } @@ -87,9 +103,10 @@ public ResponseEntity post(@PathVariable("projectName") String projectName, @ * @param file the file * @return the response entity */ + @WithSpan @PostMapping(value = HTTP_PATH_MATCHER, consumes = "multipart/form-data") - public ResponseEntity postFile(@PathVariable("projectName") String projectName, - @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity postFile(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params, @Validated @RequestParam("file") MultipartFile[] file) { return javascriptEndpoint.postFile(projectName, replaceTSWithJSExtension(projectFilePath), params, file); @@ -103,8 +120,10 @@ public ResponseEntity postFile(@PathVariable("projectName") String projectNam * @param params the params * @return the response entity */ + @WithSpan @PutMapping(HTTP_PATH_MATCHER) - public ResponseEntity put(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity put(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params) { return javascriptEndpoint.put(projectName, replaceTSWithJSExtension(projectFilePath), params); } @@ -118,9 +137,10 @@ public ResponseEntity put(@PathVariable("projectName") String projectName, @P * @param file the file * @return the response entity */ + @WithSpan @PutMapping(value = HTTP_PATH_MATCHER, consumes = "multipart/form-data") - public ResponseEntity putFile(@PathVariable("projectName") String projectName, - @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity putFile(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params, @Validated @RequestParam("file") MultipartFile file) { return javascriptEndpoint.putFile(projectName, replaceTSWithJSExtension(projectFilePath), params, file); @@ -134,8 +154,10 @@ public ResponseEntity putFile(@PathVariable("projectName") String projectName * @param params the params * @return the response entity */ + @WithSpan @PatchMapping(HTTP_PATH_MATCHER) - public ResponseEntity patch(@PathVariable("projectName") String projectName, @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity patch(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("projectfile.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params) { return javascriptEndpoint.patch(projectName, replaceTSWithJSExtension(projectFilePath), params); } @@ -148,28 +170,11 @@ public ResponseEntity patch(@PathVariable("projectName") String projectName, * @param params the params * @return the response entity */ + @WithSpan @DeleteMapping(HTTP_PATH_MATCHER) - public ResponseEntity delete(@PathVariable("projectName") String projectName, - @PathVariable("projectFilePath") String projectFilePath, + public ResponseEntity delete(@SpanAttribute("project.name") @PathVariable("projectName") String projectName, + @SpanAttribute("project.file.path") @PathVariable("projectFilePath") String projectFilePath, @Nullable @RequestParam(required = false) MultiValueMap params) { return javascriptEndpoint.delete(projectName, replaceTSWithJSExtension(projectFilePath), params); } - - /** - * Replace TS with MJS extension. - * - * @param projectFilePath the project file path - * @return the string - */ - private String replaceTSWithJSExtension(String projectFilePath) { - int indexOfExtensionStart = projectFilePath.lastIndexOf(".ts"); - if (indexOfExtensionStart == -1) { - throw new RuntimeException("Could not find .ts extension"); - } - - String projectFilePathWithoutExtension = projectFilePath.substring(0, indexOfExtensionStart); - String maybePathParameters = projectFilePath.substring(indexOfExtensionStart) - .replace(".ts", ""); // for decorators and rs api - return projectFilePathWithoutExtension + ".js" + maybePathParameters; - } } diff --git a/components/engine/engine-websockets/src/main/java/org/eclipse/dirigible/components/websockets/synchronizer/WebsocketsSynchronizer.java b/components/engine/engine-websockets/src/main/java/org/eclipse/dirigible/components/websockets/synchronizer/WebsocketsSynchronizer.java index fba0feaad20..3f1aca405ee 100644 --- a/components/engine/engine-websockets/src/main/java/org/eclipse/dirigible/components/websockets/synchronizer/WebsocketsSynchronizer.java +++ b/components/engine/engine-websockets/src/main/java/org/eclipse/dirigible/components/websockets/synchronizer/WebsocketsSynchronizer.java @@ -71,7 +71,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Websocket websocket = JsonHelper.fromJson(new String(content, StandardCharsets.UTF_8), Websocket.class); Configuration.configureObject(websocket); websocket.setLocation(location); diff --git a/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/ConfluenceSynchronizer.java b/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/ConfluenceSynchronizer.java index 7d29e04978e..70519433679 100644 --- a/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/ConfluenceSynchronizer.java +++ b/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/ConfluenceSynchronizer.java @@ -115,7 +115,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Confluence wiki = new Confluence(); Configuration.configureObject(wiki); wiki.setLocation(location); diff --git a/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/MarkdownSynchronizer.java b/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/MarkdownSynchronizer.java index 9fac83d13d6..ce311cbc7c1 100644 --- a/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/MarkdownSynchronizer.java +++ b/components/engine/engine-wiki/src/main/java/org/eclipse/dirigible/components/engine/wiki/synchronizer/MarkdownSynchronizer.java @@ -115,7 +115,7 @@ public boolean isAccepted(String type) { * @throws ParseException the parse exception */ @Override - public List parse(String location, byte[] content) throws ParseException { + protected List parseImpl(String location, byte[] content) throws ParseException { Markdown wiki = new Markdown(); Configuration.configureObject(wiki); wiki.setLocation(location); diff --git a/components/group/group-engines/pom.xml b/components/group/group-engines/pom.xml index 705eb169ead..d4e974ec73a 100644 --- a/components/group/group-engines/pom.xml +++ b/components/group/group-engines/pom.xml @@ -1,5 +1,5 @@ - 4.0.0 @@ -98,6 +98,10 @@ org.eclipse.dirigible dirigible-components-engine-camel + + org.eclipse.dirigible + dirigible-components-engine-open-telemetry + - 3.4.0 - 3.3.6 + 3.3.4 + ${spring.boot.version} 2.6.0 @@ -731,13 +731,13 @@ com.squareup.okhttp3 okhttp test - ${okhttp.version} + ${okhttp.version} com.squareup.okhttp3 okhttp-urlconnection - ${okhttp.version} test + ${okhttp.version} org.mockito @@ -749,6 +749,13 @@ + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom + 2.8.0 + pom + import + org.springframework.boot spring-boot-dependencies diff --git a/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/DirigibleCleaner.java b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/DirigibleCleaner.java index 3cf96041477..75d895a96ff 100644 --- a/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/DirigibleCleaner.java +++ b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/DirigibleCleaner.java @@ -30,6 +30,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; +import java.util.stream.Collectors; @Component public class DirigibleCleaner { @@ -74,34 +75,11 @@ private void deleteDirigibleDBData() { DirigibleDataSource systemDataSource = dataSourcesManager.getSystemDataSource(); deleteAllTablesDataInSchema(systemDataSource); + dropAllTablesInSchema(systemDataSource, "QRTZ_"); deleteSchemas(defaultDataSource); } - private void deleteAllTablesDataInSchema(DirigibleDataSource dataSource) { - Set tables = getAllTables(dataSource); - - for (int idx = 0; idx < 4; idx++) { // execute it a few times due to constraint violations - Iterator iterator = tables.iterator(); - while (iterator.hasNext()) { - String table = iterator.next(); - try (Connection connection = dataSource.getConnection()) { - String sql = SqlDialectFactory.getDialect(dataSource) - .delete() - .from(table) - .build(); - try (PreparedStatement prepareStatement = connection.prepareStatement(sql)) { - int rowsAffected = prepareStatement.executeUpdate(); - LOGGER.info("Deleted [{}] from table [{}]", rowsAffected, table); - iterator.remove(); - } - } catch (SQLException ex) { - LOGGER.warn("Failed to delete data from table [{}] in data source [{}]", table, dataSource, ex); - } - } - } - } - private void dropAllSequencesInSchema(DirigibleDataSource dataSource) { List sequences = getAllSequences(dataSource); LOGGER.info("Will drop [{}] sequences from data source [{}]. Sequences: {}", sequences.size(), dataSource, sequences); @@ -142,8 +120,14 @@ private List getAllSequences(DataSource dataSource) { } } - private void dropAllTablesInSchema(DirigibleDataSource dataSource) { + private void dropAllTablesInSchema(DirigibleDataSource dataSource, String... skipTablePrefixes) { Set tables = getAllTables(dataSource); + for (String skipTablePrefix : skipTablePrefixes) { + tables = tables.stream() + .filter(t -> !t.startsWith(skipTablePrefix)) + .collect(Collectors.toSet()); + } + LOGGER.info("Will drop [{}] tables from data source [{}]. Tables: {}", tables.size(), dataSource, tables); for (int idx = 0; idx < 4; idx++) { // execute it a few times due to constraint violations @@ -240,6 +224,30 @@ private void deleteSchema(String schema, DirigibleDataSource dataSource) { } } + private void deleteAllTablesDataInSchema(DirigibleDataSource dataSource) { + Set tables = getAllTables(dataSource); + + for (int idx = 0; idx < 4; idx++) { // execute it a few times due to constraint violations + Iterator iterator = tables.iterator(); + while (iterator.hasNext()) { + String table = iterator.next(); + try (Connection connection = dataSource.getConnection()) { + String sql = SqlDialectFactory.getDialect(dataSource) + .delete() + .from(table) + .build(); + try (PreparedStatement prepareStatement = connection.prepareStatement(sql)) { + int rowsAffected = prepareStatement.executeUpdate(); + LOGGER.info("Deleted [{}] from table [{}]", rowsAffected, table); + iterator.remove(); + } + } catch (SQLException ex) { + LOGGER.warn("Failed to delete data from table [{}] in data source [{}]", table, dataSource, ex); + } + } + } + } + private void deleteH2Folder() { String h2Folder = getDirigibleSubfolder("h2"); FileUtil.deleteFolder(h2Folder); diff --git a/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/api/SecurityIT.java b/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/api/SecurityIT.java index 10e625d1f4a..6711e46ff12 100644 --- a/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/api/SecurityIT.java +++ b/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/api/SecurityIT.java @@ -1,3 +1,12 @@ +/* + * Copyright (c) 2010-2024 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ package org.eclipse.dirigible.integration.tests.api; import org.eclipse.dirigible.components.base.http.roles.Roles; @@ -11,7 +20,6 @@ import java.util.Map; import java.util.Set; -import static org.assertj.core.api.Assertions.fail; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -51,16 +59,13 @@ void testProtectedEndpointsWithUnauthorizedUser() throws Exception { @Test @WithMockUser(username = "operator", roles = {Roles.RoleNames.OPERATOR}) - void testOperatorEndpointIsAccessible() { + void testOperatorEndpointIsAccessible() throws Exception { Map paths = Map.of("/spring-admin", HttpStatus.NOT_FOUND, "/actuator/info", HttpStatus.OK); - paths.forEach((path, expectedStatus) -> { - try { - mvc.perform(get(path)) - .andExpect(status().is(expectedStatus.value())); - } catch (Exception e) { - fail(e); - } - }); + for (Map.Entry entry : paths.entrySet()) { + mvc.perform(get(entry.getKey())) + .andExpect(status().is(entry.getValue() + .value())); + } } @Test diff --git a/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/OrderedTestSuite.java b/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/OrderedTestSuite.java index e46d0de9065..a2dd65fdeef 100644 --- a/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/OrderedTestSuite.java +++ b/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/OrderedTestSuite.java @@ -15,7 +15,7 @@ @Suite @SuiteDisplayName("Ordered Test Suite") -@SelectClasses({BPMStarterTemplateIT.class, MultitenancyIT.class}) +@SelectClasses({HomepageRedirectIT.class, BPMStarterTemplateIT.class}) public class OrderedTestSuite { // use this suite class to run tests in specific order if needed // it is not configured to be executed automatically by the maven plugins diff --git a/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/SpringBootAdminIT.java b/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/SpringBootAdminIT.java index 398c626c45d..e17bdb6cca7 100644 --- a/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/SpringBootAdminIT.java +++ b/tests/tests-integrations/src/test/java/org/eclipse/dirigible/integration/tests/ui/tests/SpringBootAdminIT.java @@ -1,3 +1,12 @@ +/* + * Copyright (c) 2010-2024 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ package org.eclipse.dirigible.integration.tests.ui.tests; import org.eclipse.dirigible.tests.framework.HtmlElementType;