From 3707f6ce406ad6be570c82ced72bc18fb81270be Mon Sep 17 00:00:00 2001 From: Anton Malinskiy Date: Sun, 25 Feb 2024 00:33:21 +1000 Subject: [PATCH] feat(analytics): configurable missing analytics values --- .../marathon/config/AnalyticsConfiguration.kt | 10 ++-- .../marathon/config/analytics/Defaults.kt | 5 ++ .../serialization/ConfigurationFactory.kt | 2 + .../yaml/GraphiteConfigurationDeserializer.kt | 6 ++- .../yaml/InfluxDbConfigurationDeserializer.kt | 7 ++- .../serialization/ConfigurationFactoryTest.kt | 54 +++++++++++-------- .../ConfigurationSerializationTest.kt | 1 + .../BaseMetricsProviderIntegrationTest.kt | 16 +++--- .../influx/InfluxDbProviderIntegrationTest.kt | 6 ++- .../InfluxDb2ProviderIntegrationTest.kt | 3 +- .../external/MetricsProviderFactory.kt | 6 +-- .../analytics/external/MetricsProviderImpl.kt | 24 +++++---- .../remote/graphite/GraphiteDataSourceTest.kt | 4 +- .../remote/influx/InfluxDBDataSourceTest.kt | 8 +-- .../remote/influx/MetricsProviderImplTest.kt | 26 ++++----- docs/runner/configuration/analytics.md | 16 ++++++ .../gradle/configuration/AnalyticsConfig.kt | 13 +++-- 17 files changed, 138 insertions(+), 69 deletions(-) create mode 100644 configuration/src/main/kotlin/com/malinskiy/marathon/config/analytics/Defaults.kt diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/AnalyticsConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/AnalyticsConfiguration.kt index 1f5df98a7..fa10baa2e 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/AnalyticsConfiguration.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/AnalyticsConfiguration.kt @@ -2,6 +2,7 @@ package com.malinskiy.marathon.config import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.malinskiy.marathon.config.analytics.Defaults @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -21,7 +22,8 @@ sealed class AnalyticsConfiguration { val user: String, val password: String, val dbName: String, - val retentionPolicyConfiguration: RetentionPolicyConfiguration + val retentionPolicyConfiguration: RetentionPolicyConfiguration, + val defaults: Defaults = Defaults(), ) : AnalyticsConfiguration() { data class RetentionPolicyConfiguration( val name: String, @@ -48,7 +50,8 @@ sealed class AnalyticsConfiguration { val token: String, val organization: String, val bucket: String, - val retentionPolicyConfiguration: RetentionPolicyConfiguration = RetentionPolicyConfiguration.default + val retentionPolicyConfiguration: RetentionPolicyConfiguration = RetentionPolicyConfiguration.default, + val defaults: Defaults = Defaults(), ) : AnalyticsConfiguration() { data class RetentionPolicyConfiguration( val everySeconds: Int, @@ -69,6 +72,7 @@ sealed class AnalyticsConfiguration { data class GraphiteConfiguration( val host: String, val port: Int?, - val prefix: String? + val prefix: String?, + val defaults: Defaults = Defaults(), ) : AnalyticsConfiguration() } diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/analytics/Defaults.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/analytics/Defaults.kt new file mode 100644 index 000000000..b06e67ec2 --- /dev/null +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/analytics/Defaults.kt @@ -0,0 +1,5 @@ +package com.malinskiy.marathon.config.analytics + +import java.time.Duration + +data class Defaults(val successRate: Double = .0, val duration: Duration = Duration.ofMinutes(5)) diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt index 37f00a210..fc84bff08 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt @@ -2,6 +2,7 @@ package com.malinskiy.marathon.config.serialization import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.dataformat.yaml.YAMLFactory @@ -43,6 +44,7 @@ class ConfigurationFactory( ) registerModule(JavaTimeModule()) configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + configure(SerializationFeature.WRITE_DURATIONS_AS_TIMESTAMPS, false) }, private val environmentVariableSubstitutor: StringSubstitutor = StringSubstitutor(StringLookupFactory.INSTANCE.environmentVariableStringLookup()), private val analyticsTracking: Boolean? = null, diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/GraphiteConfigurationDeserializer.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/GraphiteConfigurationDeserializer.kt index b3d6b46f8..705305a76 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/GraphiteConfigurationDeserializer.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/GraphiteConfigurationDeserializer.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.malinskiy.marathon.config.AnalyticsConfiguration +import com.malinskiy.marathon.config.analytics.Defaults import com.malinskiy.marathon.config.exceptions.ConfigurationException class GraphiteConfigurationDeserializer @@ -17,6 +18,9 @@ class GraphiteConfigurationDeserializer val host = node?.get("host")?.asText() val portString = node?.get("port")?.asText() val prefix = node?.get("prefix")?.asText() + val defaults = node?.get("defaults")?.let { + ctxt?.readTreeAsValue(it, Defaults::class.java) + } ?: Defaults() val port = portString?.toIntOrNull() @@ -24,6 +28,6 @@ class GraphiteConfigurationDeserializer if (portString != null && port == null) throw ConfigurationException("GraphiteConfigurationDeserializer: port should be a number, e.g. 2003") if (prefix?.isEmpty() == true) throw ConfigurationException("GraphiteConfigurationDeserializer: prefix cannot be empty") - return AnalyticsConfiguration.GraphiteConfiguration(host, port, prefix) + return AnalyticsConfiguration.GraphiteConfiguration(host, port, prefix, defaults) } } diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/InfluxDbConfigurationDeserializer.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/InfluxDbConfigurationDeserializer.kt index e57b67a37..5b00839d5 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/InfluxDbConfigurationDeserializer.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/yaml/InfluxDbConfigurationDeserializer.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.malinskiy.marathon.config.AnalyticsConfiguration +import com.malinskiy.marathon.config.analytics.Defaults import com.malinskiy.marathon.config.exceptions.ConfigurationException class InfluxDbConfigurationDeserializer @@ -23,11 +24,15 @@ class InfluxDbConfigurationDeserializer retentionPolicyNode?.let { ctxt?.readValue(retentionPolicyNode, policyClazz) } ?: AnalyticsConfiguration.InfluxDbConfiguration.RetentionPolicyConfiguration.default + val defaults = node?.get("defaults")?.let { + ctxt?.readTreeAsValue(it, Defaults::class.java) + } ?: Defaults() + if (url == null) throw ConfigurationException("InfluxDbConfigurationDeserializer: url should be specified") if (user == null) throw ConfigurationException("InfluxDbConfigurationDeserializer: user should be specified") if (password == null) throw ConfigurationException("InfluxDbConfigurationDeserializer: password should be specified") if (dbName == null) throw ConfigurationException("InfluxDbConfigurationDeserializer: dbName should be specified") - return AnalyticsConfiguration.InfluxDbConfiguration(url, user, password, dbName, retentionPolicyConfiguration) + return AnalyticsConfiguration.InfluxDbConfiguration(url, user, password, dbName, retentionPolicyConfiguration, defaults) } } diff --git a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt index 222488fcb..9641bd272 100644 --- a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt +++ b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt @@ -9,6 +9,7 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule import com.malinskiy.marathon.config.AnalyticsConfiguration import com.malinskiy.marathon.config.ScreenRecordingPolicy import com.malinskiy.marathon.config.TestFilterConfiguration +import com.malinskiy.marathon.config.analytics.Defaults import com.malinskiy.marathon.config.environment.EnvironmentConfiguration import com.malinskiy.marathon.config.environment.EnvironmentReader import com.malinskiy.marathon.config.exceptions.ConfigurationException @@ -72,14 +73,16 @@ class ConfigurationFactoryTest { mockMarathonFileDir, ) ) - .registerModule(KotlinModule.Builder() - .withReflectionCacheSize(512) - .configure(KotlinFeature.NullToEmptyCollection, false) - .configure(KotlinFeature.NullToEmptyMap, false) - .configure(KotlinFeature.NullIsSameAsDefault, false) - .configure(KotlinFeature.SingletonSupport, true) - .configure(KotlinFeature.StrictNullChecks, false) - .build()) + .registerModule( + KotlinModule.Builder() + .withReflectionCacheSize(512) + .configure(KotlinFeature.NullToEmptyCollection, false) + .configure(KotlinFeature.NullToEmptyMap, false) + .configure(KotlinFeature.NullIsSameAsDefault, false) + .configure(KotlinFeature.SingletonSupport, true) + .configure(KotlinFeature.StrictNullChecks, false) + .build() + ) .registerModule(JavaTimeModule()) parser = ConfigurationFactory( marathonfileDir = mockMarathonFileDir, @@ -99,7 +102,8 @@ class ConfigurationFactoryTest { user = "root", password = "root", dbName = "marathon", - retentionPolicyConfiguration = AnalyticsConfiguration.InfluxDbConfiguration.RetentionPolicyConfiguration.default + retentionPolicyConfiguration = AnalyticsConfiguration.InfluxDbConfiguration.RetentionPolicyConfiguration.default, + defaults = Defaults(), ) val poolingStrategyConfiguration = configuration.poolingStrategy as PoolingStrategyConfiguration.ComboPoolingStrategyConfiguration poolingStrategyConfiguration.list[0] shouldBeInstanceOf PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration::class @@ -208,7 +212,8 @@ class ConfigurationFactoryTest { "1h", 5, false - ) + ), + defaults = Defaults(), ) } @@ -256,7 +261,6 @@ class ConfigurationFactoryTest { } - @Test fun `on configuration without an explicit xctestrun path should throw an exception`() { val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/sample_5.yaml").file) @@ -276,14 +280,16 @@ class ConfigurationFactoryTest { mockMarathonFileDir, ) ) - .registerModule(KotlinModule.Builder() - .withReflectionCacheSize(512) - .configure(KotlinFeature.NullToEmptyCollection, false) - .configure(KotlinFeature.NullToEmptyMap, false) - .configure(KotlinFeature.NullIsSameAsDefault, false) - .configure(KotlinFeature.SingletonSupport, true) - .configure(KotlinFeature.StrictNullChecks, false) - .build()) + .registerModule( + KotlinModule.Builder() + .withReflectionCacheSize(512) + .configure(KotlinFeature.NullToEmptyCollection, false) + .configure(KotlinFeature.NullToEmptyMap, false) + .configure(KotlinFeature.NullIsSameAsDefault, false) + .configure(KotlinFeature.SingletonSupport, true) + .configure(KotlinFeature.StrictNullChecks, false) + .build() + ) .registerModule(JavaTimeModule()) parser = ConfigurationFactory( marathonfileDir = mockMarathonFileDir, @@ -391,7 +397,10 @@ class ConfigurationFactoryTest { outputs shouldBeEqualTo listOf( AndroidTestBundleConfiguration( application = File(mockMarathonFileDir, "kotlin-buildscript/build/outputs/apk/debug/first-app-debug.apk"), - testApplication = File(mockMarathonFileDir, "kotlin-buildscript/build/outputs/apk/androidTest/debug/first-app-debug-androidTest.apk"), + testApplication = File( + mockMarathonFileDir, + "kotlin-buildscript/build/outputs/apk/androidTest/debug/first-app-debug-androidTest.apk" + ), extraApplications = null, splitApks = listOf( File(mockMarathonFileDir, "kotlin-buildscript/build/outputs/apk/androidTest/debug/first-app-split-debug.apk") @@ -399,7 +408,10 @@ class ConfigurationFactoryTest { ), AndroidTestBundleConfiguration( application = File(mockMarathonFileDir, "kotlin-buildscript/build/outputs/apk/debug/second-app-debug.apk"), - testApplication = File(mockMarathonFileDir, "kotlin-buildscript/build/outputs/apk/androidTest/debug/second-app-debug-androidTest.apk"), + testApplication = File( + mockMarathonFileDir, + "kotlin-buildscript/build/outputs/apk/androidTest/debug/second-app-debug-androidTest.apk" + ), extraApplications = null, splitApks = listOf( File(mockMarathonFileDir, "kotlin-buildscript/build/outputs/apk/androidTest/debug/second-app-split-debug.apk") diff --git a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationSerializationTest.kt b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationSerializationTest.kt index b6ba17d39..1dd12ac4f 100644 --- a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationSerializationTest.kt +++ b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationSerializationTest.kt @@ -77,6 +77,7 @@ class ConfigurationSerializationTest { ) registerModule(JavaTimeModule()) configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + configure(SerializationFeature.WRITE_DURATIONS_AS_TIMESTAMPS, false) } parser = ConfigurationFactory( marathonfileDir = marathonfileDir, diff --git a/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/BaseMetricsProviderIntegrationTest.kt b/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/BaseMetricsProviderIntegrationTest.kt index b7375cd78..492036bc7 100644 --- a/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/BaseMetricsProviderIntegrationTest.kt +++ b/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/BaseMetricsProviderIntegrationTest.kt @@ -3,10 +3,12 @@ package com.malinskiy.marathon.analytics.metrics.remote import com.malinskiy.marathon.analytics.external.MetricsProvider import com.malinskiy.marathon.analytics.external.MetricsProviderImpl import com.malinskiy.marathon.generateTest +import org.amshove.kluent.shouldBeEqualTo import org.amshove.kluent.shouldBeInRange import org.amshove.kluent.shouldEqualTo import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.time.Duration import java.time.Instant import java.time.temporal.ChronoUnit @@ -19,19 +21,19 @@ abstract class BaseMetricsProviderIntegrationTest { @BeforeEach fun setUp() { - metricsProvider = MetricsProviderImpl(createRemoteDataSource()) + metricsProvider = MetricsProviderImpl(createRemoteDataSource(), .0, Duration.ofMinutes(5)) } @Test fun `on empty db success rate default value is 0`() { val result = metricsProvider.successRate(test, Instant.now()) - result shouldEqualTo 0.0 + result shouldBeEqualTo 0.0 } @Test fun `execution time default value is 300_000`() { val result = metricsProvider.executionTime(test, 90.0, Instant.now()) - result shouldEqualTo 300_000.0 + result shouldBeEqualTo 300_000.0 } @Test @@ -49,19 +51,19 @@ abstract class BaseMetricsProviderIntegrationTest { @Test fun `verify execution time 50 percentile for 25 minutes`() { val result = metricsProvider.executionTime(test, 50.0, Instant.now().minus(25, ChronoUnit.MINUTES)) - result shouldEqualTo 2000.0 + result shouldBeEqualTo 2000.0 } @Test fun `verify execution time 90 percentile for 25 minutes`() { val result = metricsProvider.executionTime(test, 90.0, Instant.now().minus(35, ChronoUnit.MINUTES)) - result shouldEqualTo 4000.0 + result shouldBeEqualTo 4000.0 } @Test fun `verify test success rate should return 1 for last 50 minutes`() { val result = metricsProvider.successRate(test, Instant.now().minus(50, ChronoUnit.MINUTES)) - result shouldEqualTo 1.0 + result shouldBeEqualTo 1.0 } @Test @@ -73,6 +75,6 @@ abstract class BaseMetricsProviderIntegrationTest { @Test fun `verify test success rate should return 0_5 for last 2 hours`() { val result = metricsProvider.successRate(test, Instant.now().minus(2, ChronoUnit.HOURS)) - result shouldEqualTo 0.5 + result shouldBeEqualTo 0.5 } } diff --git a/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDbProviderIntegrationTest.kt b/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDbProviderIntegrationTest.kt index d5d6d4f4b..993823fb9 100644 --- a/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDbProviderIntegrationTest.kt +++ b/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDbProviderIntegrationTest.kt @@ -6,11 +6,13 @@ import com.malinskiy.marathon.analytics.external.influx.InfluxDbTracker import com.malinskiy.marathon.analytics.metrics.remote.getTestEvents import com.malinskiy.marathon.config.AnalyticsConfiguration import com.malinskiy.marathon.generateTest +import org.amshove.kluent.shouldBeEqualTo import org.amshove.kluent.shouldEqualTo import org.influxdb.InfluxDB import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.time.Duration import java.time.Instant import java.time.temporal.ChronoUnit @@ -57,13 +59,13 @@ class InfluxDbProviderIntegrationTest { thirdDbInstance = provider.createDb() val metricsProvider = - MetricsProviderImpl(InfluxDBDataSource(thirdDbInstance!!, database, rpName)) + MetricsProviderImpl(InfluxDBDataSource(thirdDbInstance!!, database, rpName), .0, Duration.ofMinutes(5)) val result = metricsProvider.executionTime( test, 50.0, Instant.now().minus(2, ChronoUnit.DAYS) ) - result shouldEqualTo 5000.0 + result shouldBeEqualTo 5000.0 } } diff --git a/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx2/InfluxDb2ProviderIntegrationTest.kt b/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx2/InfluxDb2ProviderIntegrationTest.kt index 5c7d5ffbd..3b292b7eb 100644 --- a/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx2/InfluxDb2ProviderIntegrationTest.kt +++ b/core/src/integrationTest/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx2/InfluxDb2ProviderIntegrationTest.kt @@ -11,6 +11,7 @@ import org.amshove.kluent.shouldBeEqualTo import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.time.Duration import java.time.Instant import java.time.temporal.ChronoUnit @@ -65,7 +66,7 @@ class InfluxDb2ProviderIntegrationTest { thirdDbInstance = provider.createDb() val metricsProvider = - MetricsProviderImpl(InfluxDB2DataSource(thirdDbInstance!!, bucket)) + MetricsProviderImpl(InfluxDB2DataSource(thirdDbInstance!!, bucket), .0, Duration.ofMinutes(5)) val result1 = metricsProvider.executionTime( test, diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderFactory.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderFactory.kt index 98b09464c..f947d732f 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderFactory.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderFactory.kt @@ -28,7 +28,7 @@ internal class MetricsProviderFactory(configuration: Configuration) { return try { val db = InfluxDbProvider(configuration, debug).createDb() val dataSource = InfluxDBDataSource(db, configuration.dbName, configuration.retentionPolicyConfiguration.name) - MetricsProviderImpl(dataSource) + MetricsProviderImpl(dataSource, configuration.defaults.successRate, configuration.defaults.duration) } catch (e: Exception) { fallback } @@ -38,13 +38,13 @@ internal class MetricsProviderFactory(configuration: Configuration) { return try { val db = InfluxDb2Provider(configuration, debug).createDb() val dataSource = InfluxDB2DataSource(db, configuration.bucket) - MetricsProviderImpl(dataSource) + MetricsProviderImpl(dataSource, configuration.defaults.successRate, configuration.defaults.duration) } catch (e: Exception) { fallback } } private fun createGraphiteMetricsProvider(configuration: GraphiteConfiguration): MetricsProvider { - return MetricsProviderImpl(GraphiteDataSource(QueryableGraphiteClient(configuration.host), configuration.prefix)) + return MetricsProviderImpl(GraphiteDataSource(QueryableGraphiteClient(configuration.host), configuration.prefix), configuration.defaults.successRate, configuration.defaults.duration) } } diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderImpl.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderImpl.kt index 67b90846a..bd7682c4b 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderImpl.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/external/MetricsProviderImpl.kt @@ -6,6 +6,7 @@ import com.malinskiy.marathon.log.MarathonLogging import com.malinskiy.marathon.test.Test import com.malinskiy.marathon.test.toSafeTestName import kotlinx.coroutines.runBlocking +import java.time.Duration import java.time.Instant private data class MeasurementKey(val key: Double = Double.NaN, val limit: Instant) @@ -19,7 +20,11 @@ private class MeasurementValues(private val values: Map) { private const val RETRY_DELAY: Long = 50L -class MetricsProviderImpl(private val remoteDataStore: RemoteDataSource) : MetricsProvider { +class MetricsProviderImpl( + private val remoteDataStore: RemoteDataSource, + private val defaultSuccessRate: Double, + private val defaultDuration: Duration +) : MetricsProvider { private val logger = MarathonLogging.logger(MetricsProviderImpl::class.java.simpleName) @@ -44,13 +49,13 @@ class MetricsProviderImpl(private val remoteDataStore: RemoteDataSource) : Metri } val testName = test.toSafeTestName() - return successRateMeasurements[key]?.get(testName) ?: { + return successRateMeasurements[key]?.get(testName) ?: run { if (!warningSuccessRateTimeReported.contains(testName)) { - logger.warn { "No success rate found for $testName. Using 0 i.e. fails all the time" } + logger.warn { "No success rate found for $testName. Using $defaultSuccessRate" } warningSuccessRateTimeReported.add(testName) } - 0.0 - }() + defaultSuccessRate + } } private fun fetchSuccessRateData(limit: Instant) = runBlocking { @@ -79,13 +84,14 @@ class MetricsProviderImpl(private val remoteDataStore: RemoteDataSource) : Metri }) } val testName = test.toSafeTestName() - return executionTimeMeasurements[key]?.get(testName) ?: { + return executionTimeMeasurements[key]?.get(testName) ?: run { + val default = defaultDuration.toMillis() if (!warningExecutionTimeReported.contains(testName)) { - logger.warn { "No execution time found for $testName. Using 300_000 seconds i.e. long test" } + logger.warn { "No execution time found for $testName. Using $default seconds" } warningExecutionTimeReported.add(testName) } - 300_000.0 - }() + default.toDouble() + } } private val warningExecutionTimeReported = hashSetOf() diff --git a/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/graphite/GraphiteDataSourceTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/graphite/GraphiteDataSourceTest.kt index 7df92aa17..99d65e589 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/graphite/GraphiteDataSourceTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/graphite/GraphiteDataSourceTest.kt @@ -27,7 +27,7 @@ internal class GraphiteDataSourceTest { val rate = provider.requestAllSuccessRates(limit) rate.size shouldEqualTo 1 - rate[0].mean shouldEqualTo 0.5 + rate[0].mean shouldBeEqualTo 0.5 rate[0].testName shouldBeEqualTo "com.example.SingleTest.method" } @@ -47,6 +47,6 @@ internal class GraphiteDataSourceTest { rate.size shouldEqual 1 rate[0].testName shouldBeEqualTo "com.example.SingleTest.method" - rate[0].percentile shouldEqualTo 5000.0 + rate[0].percentile shouldBeEqualTo 5000.0 } } diff --git a/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDBDataSourceTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDBDataSourceTest.kt index 24ce3f659..dc0369e8b 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDBDataSourceTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/InfluxDBDataSourceTest.kt @@ -42,7 +42,7 @@ class InfluxDBDataSourceTest { val rate = provider.requestAllSuccessRates(Instant.now()) rate.size shouldEqualTo 1 - rate[0].mean shouldEqualTo 0.5 + rate[0].mean shouldBeEqualTo 0.5 rate[0].testName shouldBeEqualTo "com.example.SingleTest.method" } @@ -74,7 +74,7 @@ class InfluxDBDataSourceTest { rate.size shouldEqual 1 rate[0].testName shouldBeEqualTo "com.example.SingleTest.method" - rate[0].percentile shouldEqualTo 5000.0 + rate[0].percentile shouldBeEqualTo 5000.0 } @Test @@ -114,7 +114,7 @@ class InfluxDBDataSourceTest { val rate = provider.requestAllSuccessRates(Instant.parse("2019-08-24T08:25:09.219Z")) rate.size shouldEqualTo 1 - rate[0].mean shouldEqualTo 0.5 + rate[0].mean shouldBeEqualTo 0.5 rate[0].testName shouldBeEqualTo "com.example.SingleTest.method" } @@ -155,6 +155,6 @@ class InfluxDBDataSourceTest { rate.size shouldEqual 1 rate[0].testName shouldBeEqualTo "com.example.SingleTest.method" - rate[0].percentile shouldEqualTo 5000.0 + rate[0].percentile shouldBeEqualTo 5000.0 } } diff --git a/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/MetricsProviderImplTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/MetricsProviderImplTest.kt index 632a2fee1..6fed8cf8a 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/MetricsProviderImplTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/analytics/metrics/remote/influx/MetricsProviderImplTest.kt @@ -6,6 +6,7 @@ import com.malinskiy.marathon.analytics.metrics.remote.RemoteDataSource import com.malinskiy.marathon.analytics.metrics.remote.SuccessRate import com.malinskiy.marathon.generateTest import com.malinskiy.marathon.test.toSafeTestName +import org.amshove.kluent.shouldBeEqualTo import org.mockito.kotlin.mock import org.amshove.kluent.shouldEqualTo import org.junit.jupiter.api.Test @@ -13,13 +14,14 @@ import org.mockito.kotlin.eq import org.mockito.kotlin.verify import org.mockito.kotlin.verifyNoMoreInteractions import org.mockito.kotlin.whenever +import java.time.Duration import java.time.Instant class MetricsProviderImplTest { @Test fun testExecutionTimeFromCache() { val dataStore = mock() - val provider = MetricsProviderImpl(dataStore) + val provider = MetricsProviderImpl(dataStore, .0, Duration.ofMinutes(5)) val test = generateTest() val requestPercentile = 90.0 val resultTime = 100.5 @@ -32,16 +34,16 @@ class MetricsProviderImplTest { eq(limit) ) ).thenReturn(list) - provider.executionTime(test, requestPercentile, limit) shouldEqualTo resultTime + provider.executionTime(test, requestPercentile, limit) shouldBeEqualTo resultTime verify(dataStore).requestAllExecutionTimes(eq(requestPercentile), eq(limit)) verifyNoMoreInteractions(dataStore) - provider.executionTime(test, requestPercentile, limit) shouldEqualTo resultTime + provider.executionTime(test, requestPercentile, limit) shouldBeEqualTo resultTime } @Test fun testExecutionTimeFromMissingCache() { val dataStore = mock() - val provider = MetricsProviderImpl(dataStore) + val provider = MetricsProviderImpl(dataStore, .0, Duration.ofMinutes(5)) val test = generateTest() val firstPercent = 80.0 val firstTime = 900.0 @@ -54,7 +56,7 @@ class MetricsProviderImplTest { eq(firstLimit) ) ).thenReturn(firstList) - provider.executionTime(test, firstPercent, firstLimit) shouldEqualTo firstTime + provider.executionTime(test, firstPercent, firstLimit) shouldBeEqualTo firstTime verify(dataStore).requestAllExecutionTimes(eq(firstPercent), eq(firstLimit)) val secondLimit = Instant.now() @@ -68,36 +70,36 @@ class MetricsProviderImplTest { eq(secondLimit) ) ).thenReturn(secondList) - provider.executionTime(test, secondPercent, secondLimit) shouldEqualTo secondTime + provider.executionTime(test, secondPercent, secondLimit) shouldBeEqualTo secondTime verify(dataStore).requestAllExecutionTimes(eq(secondPercent), eq(secondLimit)) } @Test fun testSuccessRateFromCache() { val dataStore = mock() - val provider = MetricsProviderImpl(dataStore) + val provider = MetricsProviderImpl(dataStore, .0, Duration.ofMinutes(5)) val test = generateTest() val mean = 90.0 val limit = Instant.now() val list = listOf(SuccessRate(test.toSafeTestName(), mean), SuccessRate("test", 80.0)) whenever(dataStore.requestAllSuccessRates(limit)).thenReturn(list) - provider.successRate(test, limit) shouldEqualTo mean + provider.successRate(test, limit) shouldBeEqualTo mean verify(dataStore).requestAllSuccessRates(eq(limit)) verifyNoMoreInteractions(dataStore) - provider.successRate(test, limit) shouldEqualTo mean + provider.successRate(test, limit) shouldBeEqualTo mean } @Test fun testSuccessRateWithMissingCache() { val dataStore = mock() - val provider = MetricsProviderImpl(dataStore) + val provider = MetricsProviderImpl(dataStore, .0, Duration.ofMinutes(5)) val test = generateTest() val firstMean = 90.0 val firstLimit = Instant.now().minusSeconds(60) val firstList = listOf(SuccessRate(test.toSafeTestName(), firstMean), SuccessRate("test", 80.0)) whenever(dataStore.requestAllSuccessRates(firstLimit)).thenReturn(firstList) - provider.successRate(test, firstLimit) shouldEqualTo firstMean + provider.successRate(test, firstLimit) shouldBeEqualTo firstMean verify(dataStore).requestAllSuccessRates(eq(firstLimit)) val secondLimit = Instant.now() @@ -105,7 +107,7 @@ class MetricsProviderImplTest { val secondList = listOf(SuccessRate(test.toSafeTestName(), secondMean), SuccessRate("test", 90.0)) whenever(dataStore.requestAllSuccessRates(secondLimit)).thenReturn(secondList) - provider.successRate(test, secondLimit) shouldEqualTo secondMean + provider.successRate(test, secondLimit) shouldBeEqualTo secondMean verify(dataStore).requestAllSuccessRates(eq(secondLimit)) } } diff --git a/docs/runner/configuration/analytics.md b/docs/runner/configuration/analytics.md index bd93e3fb8..247409714 100644 --- a/docs/runner/configuration/analytics.md +++ b/docs/runner/configuration/analytics.md @@ -38,12 +38,17 @@ analyticsConfiguration: retentionPolicyConfiguration: everySeconds: 604800 # Duration in seconds for how long data will be kept in the database. 0 means infinite. minimum: 0 shardGroupDurationSeconds: 0 # Shard duration measured in seconds + defaults: + successRate: 0.1 + duration: "PT300S" ``` ```kotlin +import java.time.Duration + marathon { analytics { influx { @@ -51,6 +56,7 @@ marathon { token = "my-super-secret-token" organization = "starlabs" bucket = "marathon" + defaults = Defaults(0.0, Duration.ofMinutes(5)) } } } @@ -88,6 +94,9 @@ analyticsConfiguration: shardDuration: "1h" replicationFactor: 5 isDefault: false + defaults: + successRate: 0.1 + duration: "PT300S" ``` @@ -101,6 +110,7 @@ marathon { user = "root" password = "root" dbName = "marathon" + defaults = Defaults(0.0, Duration.ofMinutes(5)) } } } @@ -117,6 +127,7 @@ marathon { user = "root" password = "root" dbName = "marathon" + defaults = Defaults(0.0, Duration.ofMinutes(5)) } } } @@ -142,6 +153,9 @@ analyticsConfiguration: host: "influx.svc.cluster.local" port: "8080" prefix: "prf" + defaults: + successRate: 0.1 + duration: "PT300S" ``` @@ -154,6 +168,7 @@ marathon { host = "influx.svc.cluster.local" port = "8080" prefix = "prf" + defaults = Defaults(0.0, Duration.ofMinutes(5)) } } } @@ -169,6 +184,7 @@ marathon { host = "influx.svc.cluster.local" port = "8080" prefix = "prf" + defaults = Defaults(0.0, Duration.ofMinutes(5)) } } } diff --git a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/configuration/AnalyticsConfig.kt b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/configuration/AnalyticsConfig.kt index 143d25efa..b288818e7 100644 --- a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/configuration/AnalyticsConfig.kt +++ b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/configuration/AnalyticsConfig.kt @@ -1,6 +1,7 @@ package com.malinskiy.marathon.gradle import com.malinskiy.marathon.config.AnalyticsConfiguration +import com.malinskiy.marathon.config.analytics.Defaults import groovy.lang.Closure import org.gradle.api.Action import org.gradle.util.internal.ConfigureUtil @@ -35,6 +36,7 @@ class InfluxConfig { var password: String = "" var dbName: String = "" var retentionPolicy: RetentionPolicy? = null + var defaults: Defaults = Defaults() } class Influx2Config { @@ -43,6 +45,7 @@ class Influx2Config { var organization: String = "" var bucket: String = "" var retentionPolicy: Influx2RetentionPolicy? = null + var defaults: Defaults = Defaults() } class Influx2RetentionPolicy { @@ -62,6 +65,7 @@ class GraphiteConfig { var host: String = "" var port: String? = null var prefix: String? = null + var defaults: Defaults = Defaults() } fun AnalyticsConfig.toAnalyticsConfiguration(): AnalyticsConfiguration { @@ -75,7 +79,8 @@ fun AnalyticsConfig.toAnalyticsConfiguration(): AnalyticsConfiguration { organization = influx2.organization, bucket = influx2.bucket, retentionPolicyConfiguration = influx2.retentionPolicy?.toRetentionPolicy() - ?: AnalyticsConfiguration.InfluxDb2Configuration.RetentionPolicyConfiguration.default + ?: AnalyticsConfiguration.InfluxDb2Configuration.RetentionPolicyConfiguration.default, + defaults = influx2.defaults ) influx != null -> AnalyticsConfiguration.InfluxDbConfiguration( dbName = influx.dbName, @@ -83,12 +88,14 @@ fun AnalyticsConfig.toAnalyticsConfiguration(): AnalyticsConfiguration { password = influx.password, url = influx.url, retentionPolicyConfiguration = influx.retentionPolicy?.toRetentionPolicy() - ?: AnalyticsConfiguration.InfluxDbConfiguration.RetentionPolicyConfiguration.default + ?: AnalyticsConfiguration.InfluxDbConfiguration.RetentionPolicyConfiguration.default, + defaults = influx.defaults ) graphite != null -> AnalyticsConfiguration.GraphiteConfiguration( host = graphite.host, port = graphite.port?.toIntOrNull(), - prefix = graphite.prefix + prefix = graphite.prefix, + defaults = graphite.defaults ) else -> AnalyticsConfiguration.DisabledAnalytics }