Add dropwiz metrics & StroomCache to proxy

Also fix data receipt bugs
gchq · Jan 23, 2025 · 62f6383 · 62f6383
1 parent 41f6e99
commit 62f6383
Show file tree

Hide file tree

Showing 98 changed files with 2,360 additions and 498 deletions.
diff --git a/build.gradle b/build.gradle
@@ -232,6 +232,7 @@ ext.libs = [
         dropwizard_jetty                    : "io.dropwizard:dropwizard-jetty", //was 1.0.6 now 1.1.0
         dropwizard_lifecycle                : "io.dropwizard:dropwizard-lifecycle", // version controlled by dropwizard-dependencies
         dropwizard_metrics_annotation       : "io.dropwizard.metrics:metrics-annotation", // version controlled by dropwizard-dependencies
+        dropwizard_metrics_caffeine3        : "io.dropwizard.metrics:metrics-caffeine3", // version controlled by dropwizard-dependencies
         dropwizard_metrics_core             : "io.dropwizard.metrics:metrics-core", // version controlled by dropwizard-dependencies
         dropwizard_metrics_healthchecks     : "io.dropwizard.metrics:metrics-healthchecks", // version controlled by dropwizard-dependencies
         dropwizard_servlets                 : "io.dropwizard:dropwizard-servlets", // version controlled by dropwizard-dependencies

diff --git a/settings.gradle b/settings.gradle
@@ -292,3 +292,4 @@ include 'stroom-view:stroom-view-impl'
 include 'stroom-ssl-test'
 include 'stroom-view:stroom-view-api'
 include 'stroom-index:stroom-index-api'
+include 'stroom-cache:stroom-cache-service-impl'
diff --git a/stroom-app/build.gradle b/stroom-app/build.gradle
@@ -61,6 +61,7 @@ dependencies {
     implementation project(':stroom-bytebuffer')
     implementation project(':stroom-cache:stroom-cache-api')
     implementation project(':stroom-cache:stroom-cache-impl')
+    implementation project(':stroom-cache:stroom-cache-service-impl')
     implementation project(':stroom-cluster:stroom-cluster-api')
     implementation project(':stroom-cluster:stroom-cluster-impl')
     implementation project(':stroom-cluster:stroom-cluster-lock-api')

diff --git a/stroom-app/src/main/java/stroom/app/guice/BootStrapModule.java b/stroom-app/src/main/java/stroom/app/guice/BootStrapModule.java
@@ -70,6 +70,7 @@ protected void configure() {
 
         bind(Config.class).toInstance(configuration);
         bind(Environment.class).toInstance(environment);
+        bind(MetricRegistry.class).toInstance(environment.metrics());
 
         final AppConfigModule appConfigModule = appConfigModuleFunc.apply(configHolder);
         install(appConfigModule);

diff --git a/stroom-app/src/main/java/stroom/app/guice/CoreModule.java b/stroom-app/src/main/java/stroom/app/guice/CoreModule.java
@@ -16,7 +16,8 @@ protected void configure() {
         install(new stroom.aws.s3.impl.S3ConfigHandlerModule());
         install(new stroom.aws.s3.impl.S3ConfigModule());
         install(new stroom.cache.impl.CacheModule());
-        install(new stroom.cache.impl.CacheResourceModule());
+        install(new stroom.cache.service.impl.CacheServiceModule());
+        install(new stroom.cache.service.impl.CacheResourceModule());
         install(new stroom.cluster.lock.impl.db.ClusterLockModule());
         install(new stroom.cluster.task.impl.ClusterTaskModule());
         install(new stroom.config.global.impl.ConfigProvidersModule());

diff --git a/stroom-app/src/test/java/stroom/data/store/impl/fs/TestFsPathHelperIntegration.java b/stroom-app/src/test/java/stroom/data/store/impl/fs/TestFsPathHelperIntegration.java
@@ -1,6 +1,7 @@
 package stroom.data.store.impl.fs;
 
 import stroom.cache.impl.CacheModule;
+import stroom.cache.service.impl.CacheServiceModule;
 import stroom.cluster.lock.mock.MockClusterLockModule;
 import stroom.data.shared.StreamTypeNames;
 import stroom.data.store.impl.fs.db.FsDataStoreDaoModule;
@@ -72,6 +73,7 @@ protected void configure() {
                         new MockTaskModule(),
                         new MockSecurityContextModule(),
                         new CacheModule(),
+                        new CacheServiceModule(),
                         new DbTestModule())
                 .injectMembers(this);
     }

diff --git a/stroom-app/src/test/java/stroom/job/impl/TestDistributedTaskFetcher.java b/stroom-app/src/test/java/stroom/job/impl/TestDistributedTaskFetcher.java
@@ -19,7 +19,7 @@
 import stroom.util.concurrent.ThreadUtil;
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
-import stroom.util.logging.Metrics;
+import stroom.util.logging.SimpleMetrics;
 import stroom.util.scheduler.FrequencyTrigger;
 import stroom.util.scheduler.SimpleScheduleExec;
 
@@ -89,7 +89,7 @@ public void triggerImmediateExecution(final JobNode jobNode) {
             final NodeInfo nodeInfo = new MockNodeInfo();
             final AtomicLong executionCount = new AtomicLong();
 
-            Metrics.setEnabled(true);
+            SimpleMetrics.setEnabled(true);
 
             final DistributedTaskFactory distributedTaskFactory = new DistributedTaskFactory() {
                 @Override
@@ -99,10 +99,10 @@ public List<DistributedTask> fetch(final String nodeName, final int count) {
                     }
 
                     final List<DistributedTask> list = new ArrayList<>(count);
-                    Metrics.measure("fetch", () -> {
+                    SimpleMetrics.measure("fetch", () -> {
                         for (int i = 0; i < count; i++) {
                             final Runnable runnable = () ->
-                                    Metrics.measure("exec task", executionCount::incrementAndGet);
+                                    SimpleMetrics.measure("exec task", executionCount::incrementAndGet);
                             final DistributedTask distributedTask =
                                     new DistributedTask(jobName, runnable, threadPool, "test");
                             list.add(distributedTask);
@@ -131,7 +131,7 @@ public Boolean abandon(final String nodeName, final List<DistributedTask> tasks)
             distributedTaskFetcher.execute();
 
             while (true) {
-                Metrics.report();
+                SimpleMetrics.report();
                 ThreadUtil.sleep(1000);
             }
         }

diff --git a/stroom-app/src/test/java/stroom/job/impl/TestJobNodeTrackerCache.java b/stroom-app/src/test/java/stroom/job/impl/TestJobNodeTrackerCache.java
@@ -13,7 +13,7 @@
 import stroom.util.concurrent.ThreadUtil;
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
-import stroom.util.logging.Metrics;
+import stroom.util.logging.SimpleMetrics;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -80,7 +80,7 @@ void test(final long delay, final long totalTime, final TimeUnit timeUnit) throw
             final JobNodeListResponse jobNodeListResponse = JobNodeListResponse
                     .createUnboundedJobNodeResponse(List.of(jobNode));
 
-            Metrics.setEnabled(true);
+            SimpleMetrics.setEnabled(true);
 
             final ExecutorProvider executorProvider = new ExecutorProvider() {
                 @Override
@@ -104,7 +104,7 @@ public Executor get() {
 
             when(jobNodeDao.find(Mockito.any(FindJobNodeCriteria.class)))
                     .then((Answer<JobNodeListResponse>) invocation -> {
-                        Metrics.measure("find", () -> {
+                        SimpleMetrics.measure("find", () -> {
                             // Add delay.
                             ThreadUtil.sleep(delay);
                         });
@@ -114,7 +114,7 @@ public Executor get() {
             final AtomicBoolean running = new AtomicBoolean(true);
             executors.execute(() -> {
                 while (running.get()) {
-                    Metrics.measure("getTrackers", () -> {
+                    SimpleMetrics.measure("getTrackers", () -> {
                         jobNodeTrackerCache.getTrackers();
                         calls.incrementAndGet();
                     });
@@ -130,7 +130,7 @@ public Executor get() {
             countDownLatch.await();
         }
 
-        Metrics.report();
+        SimpleMetrics.report();
 
         LOGGER.info(String.valueOf(calls.get()));
     }

diff --git a/stroom-app/src/test/java/stroom/receive/TestBaseModule.java b/stroom-app/src/test/java/stroom/receive/TestBaseModule.java
@@ -1,6 +1,7 @@
 package stroom.receive;
 
 import stroom.cache.impl.CacheModule;
+import stroom.cache.service.impl.CacheServiceModule;
 import stroom.collection.mock.MockCollectionModule;
 import stroom.core.receive.ReceiveDataModule;
 import stroom.data.store.mock.MockStreamStoreModule;
@@ -34,6 +35,7 @@ public class TestBaseModule extends AbstractModule {
     @Override
     protected void configure() {
         install(new CacheModule());
+        install(new CacheServiceModule());
         install(new DictionaryModule());
         install(new DocumentationModule());
         install(new DocStoreModule());

diff --git a/stroom-app/src/test/java/stroom/test/MockServiceModule.java b/stroom-app/src/test/java/stroom/test/MockServiceModule.java
@@ -2,6 +2,7 @@
 
 import stroom.activity.mock.MockActivityModule;
 import stroom.cache.impl.CacheModule;
+import stroom.cache.service.impl.CacheServiceModule;
 import stroom.core.dataprocess.PipelineStreamTaskModule;
 import stroom.data.store.mock.MockStreamStoreModule;
 import stroom.dictionary.mock.MockWordListProviderModule;
@@ -58,6 +59,7 @@ protected void configure() {
         install(new MockActivityModule());
         install(new MockDocRefInfoModule());
         install(new CacheModule());
+        install(new CacheServiceModule());
         install(new MockMetaModule());
         install(new MockStreamStoreModule());
         install(new MockWordListProviderModule());

diff --git a/stroom-cache/README.md b/stroom-cache/README.md
@@ -0,0 +1 @@
+# stroom-cache
diff --git a/stroom-cache/stroom-cache-api/src/main/java/stroom/cache/api/CacheManager.java b/stroom-cache/stroom-cache-api/src/main/java/stroom/cache/api/CacheManager.java
@@ -18,6 +18,7 @@
 
 import stroom.util.cache.CacheConfig;
 
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -120,4 +121,6 @@ default <K, V> LoadingStroomCache<K, V> createLoadingCache(
      * Clears down all caches registered with {@link CacheManager}.
      */
     void close();
+
+    Set<String> getCacheNames();
 }
diff --git a/stroom-cache/stroom-cache-api/src/main/java/stroom/cache/api/StroomCache.java b/stroom-cache/stroom-cache-api/src/main/java/stroom/cache/api/StroomCache.java
@@ -1,7 +1,7 @@
 package stroom.cache.api;
 
-import stroom.cache.shared.CacheInfo;
 import stroom.util.shared.PropertyPath;
+import stroom.util.shared.cache.CacheInfo;
 
 import java.util.Collection;
 import java.util.Map;

diff --git a/stroom-cache/stroom-cache-impl/build.gradle b/stroom-cache/stroom-cache-impl/build.gradle
@@ -2,27 +2,17 @@ ext.moduleName = 'stroom.cache.impl'
 
 dependencies {
     implementation project(':stroom-cache:stroom-cache-api')
-//    implementation project(':stroom-cluster:stroom-cluster-task-api')
-    implementation project(':stroom-core-shared')
-    implementation project(':stroom-docref')
-    // Seem to need this else it moans it can't see HasName in AbstractStroomCacheImpl
-    implementation project(':stroom-event-logging-rs:stroom-event-logging-rs-api')
-    implementation project(':stroom-job:stroom-job-api')
-    implementation project(':stroom-lifecycle:stroom-lifecycle-api')
-    implementation project(':stroom-node:stroom-node-api')
-    implementation project(':stroom-security:stroom-security-api')
-    implementation project(':stroom-task:stroom-task-api')
     implementation project(':stroom-util')
     implementation project(':stroom-util-shared')
 
     implementation libs.caffeine
 //    implementation libs.guava
+    implementation libs.dropwizard_metrics_caffeine3
+    implementation libs.dropwizard_metrics_core
     implementation libs.guice
     implementation libs.jackson_annotations
     implementation libs.jakarta_inject
-    implementation libs.restygwt
     implementation libs.slf4j_api
-    implementation libs.ws_rs_api
 
     testImplementation project(':stroom-test-common')
     testImplementation libs.assertj_core