From 182a775c9df5e32159052db81bb3438b04672ec7 Mon Sep 17 00:00:00 2001 From: Anton Malinskiy Date: Wed, 4 Dec 2024 17:33:40 +1000 Subject: [PATCH] feat(core): add progress api + rework reporters --- buildSrc/src/main/kotlin/Versions.kt | 11 + .../marathon/config/Configuration.kt | 12 +- .../config/ProgressReporterConfiguration.kt | 50 ++++ core/build.gradle.kts | 43 ++++ .../kotlin/com/malinskiy/marathon/Marathon.kt | 4 + .../marathon/analytics/TrackerFactory.kt | 35 +-- .../analytics/internal/sub/ExecutionReport.kt | 83 ++++++- .../internal/sub/ExecutionReportGenerator.kt | 15 +- .../analytics/internal/sub/PoolSummary.kt | 15 +- .../com/malinskiy/marathon/di/Modules.kt | 12 +- .../malinskiy/marathon/execution/Scheduler.kt | 4 +- .../progress/PoolProgressAccumulator.kt | 42 ++-- .../execution/progress/ProgressReporter.kt | 25 -- .../malinskiy/marathon/integrations/ci/CI.kt | 5 - .../integrations/ci/CIIntegrationFactory.kt | 18 -- .../marathon/integrations/ci/None.kt | 5 - .../marathon/integrations/ci/Teamcity.kt | 9 - .../report/CompositeProgressReporter.kt | 34 +++ .../marathon/report/NoopProgressReporter.kt | 3 + .../marathon/report/ProgressReporter.kt | 16 ++ .../report/ProgressReporterFactory.kt | 84 +++++++ .../com/malinskiy/marathon/report/Reporter.kt | 7 - .../marathon/report/allure/AllureReporter.kt | 6 +- .../report/api/ApiV1ProgressReporter.kt | 216 ++++++++++++++++++ .../marathon/report/bill/BillingReporter.kt | 69 +----- .../marathon/report/bill/DeviceBill.kt | 14 ++ .../report/device/DeviceInfoJsonReporter.kt | 6 +- .../report/html/HtmlSummaryReporter.kt | 6 +- .../marathon/report/junit/JUnitReporter.kt | 6 +- .../marathon/report/raw/RawJsonReporter.kt | 6 +- ...tReporter.kt => StdoutProgressReporter.kt} | 60 ++++- .../teamcity/TeamCityProgressReporter.kt | 15 ++ .../marathon/report/test/TestJsonReporter.kt | 6 +- .../report/timeline/TimelineReporter.kt | 6 +- .../timeline/TimelineSummaryProvider.kt | 3 +- core/src/main/proto/progress_api_v1.proto | 90 ++++++++ .../progress/PoolProgressAccumulatorTest.kt | 106 ++++++--- .../execution/queue/QueueActorTest.kt | 3 +- .../queue/TestResultReporterAllSuccessTest.kt | 7 +- .../impl/retry/NoRetryStrategyTest.kt | 4 +- .../fixedquota/FixedQuotaRetryStrategyTest.kt | 17 +- .../marathon/report/ExecutionReportTest.kt | 8 +- .../marathon/report/JUnitReporterTest.kt | 25 +- .../scenario/DeviceFilteringScenarioTest.kt | 5 + .../scenario/DisconnectingScenariosTest.kt | 3 + .../marathon/scenario/SuccessScenariosTest.kt | 3 + .../scenario/UncompletedScenariosTest.kt | 6 + .../executor/listeners/TestResultsListener.kt | 1 - .../test/factory/ConfigurationFactory.kt | 3 + .../marathon/test/factory/MarathonFactory.kt | 9 +- 50 files changed, 927 insertions(+), 314 deletions(-) create mode 100644 configuration/src/main/kotlin/com/malinskiy/marathon/config/ProgressReporterConfiguration.kt delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CI.kt delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CIIntegrationFactory.kt delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/None.kt delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/Teamcity.kt create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/CompositeProgressReporter.kt create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/NoopProgressReporter.kt create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporter.kt create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporterFactory.kt delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/Reporter.kt create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/api/ApiV1ProgressReporter.kt create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/bill/DeviceBill.kt rename core/src/main/kotlin/com/malinskiy/marathon/report/stdout/{StdoutReporter.kt => StdoutProgressReporter.kt} (50%) create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/report/teamcity/TeamCityProgressReporter.kt create mode 100644 core/src/main/proto/progress_api_v1.proto diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 370a2ea10..b366ca24c 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -51,6 +51,12 @@ object Versions { val jsonAssert = "1.5.3" val xmlUnit = "2.10.0" val assertk = "0.28.1" + + val grpc = "1.59.0" + val grpcKotlin = "1.4.0" + val grpcOkhttp = "1.59.0" + val protobufGradle = "0.9.4" + val protobuf = "3.25.0" } object BuildPlugins { @@ -97,6 +103,11 @@ object Libraries { val kotlinProcess = "com.github.pgreze:kotlin-process:${Versions.kotlinProcess}" val okhttp = "com.squareup.okhttp3:okhttp:${Versions.okhttp}" val ktorNetwork = "io.ktor:ktor-network:${Versions.ktor}" + val protobufLite = "com.google.protobuf:protobuf-javalite:${Versions.protobuf}" + val grpcStub = "io.grpc:grpc-stub:${Versions.grpc}" + val grpcKotlinStub = "io.grpc:grpc-kotlin-stub:${Versions.grpcKotlin}" + val grpcProtobufLite = "io.grpc:grpc-protobuf-lite:${Versions.grpc}" + val grpcOkhttp = "io.grpc:grpc-okhttp:${Versions.grpcOkhttp}" } object TestLibraries { diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt index 2bcb21dcf..6e0d33a85 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt @@ -50,7 +50,7 @@ data class Configuration private constructor( val analyticsTracking: Boolean, val bugsnagReporting: Boolean, val deviceInitializationTimeoutMillis: Long, - val ciConfiguration: CIConfiguration, + val progressConfiguration: ProgressConfiguration, ) { fun toMap() = mapOf( @@ -76,7 +76,7 @@ data class Configuration private constructor( "screenRecordingPolicy" to screenRecordingPolicy.toString(), "vendorConfiguration" to vendorConfiguration.toString(), "deviceInitializationTimeoutMillis" to deviceInitializationTimeoutMillis.toString(), - "ciConfiguration" to ciConfiguration.toString(), + "progressConfiguration" to progressConfiguration.toString(), ) override fun equals(other: Any?): Boolean { @@ -110,7 +110,7 @@ data class Configuration private constructor( if (analyticsTracking != other.analyticsTracking) return false if (bugsnagReporting != other.bugsnagReporting) return false if (deviceInitializationTimeoutMillis != other.deviceInitializationTimeoutMillis) return false - if (ciConfiguration != other.ciConfiguration) return false + if (progressConfiguration != other.progressConfiguration) return false return true } @@ -141,7 +141,7 @@ data class Configuration private constructor( result = 31 * result + analyticsTracking.hashCode() result = 31 * result + bugsnagReporting.hashCode() result = 31 * result + deviceInitializationTimeoutMillis.hashCode() - result = 31 * result + ciConfiguration.hashCode() + result = 31 * result + progressConfiguration.hashCode() return result } @@ -177,7 +177,7 @@ data class Configuration private constructor( var outputConfiguration: OutputConfiguration = OutputConfiguration(), var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(), - var ciConfiguration: CIConfiguration = CIConfiguration.None, + var progressConfiguration: ProgressConfiguration = ProgressConfiguration.Auto, ) { fun build(): Configuration { return Configuration( @@ -206,7 +206,7 @@ data class Configuration private constructor( analyticsTracking = analyticsTracking, bugsnagReporting = bugsnagReporting, deviceInitializationTimeoutMillis = deviceInitializationTimeoutMillis, - ciConfiguration = ciConfiguration, + progressConfiguration = progressConfiguration, ) } } diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/ProgressReporterConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/ProgressReporterConfiguration.kt new file mode 100644 index 000000000..ac90b0001 --- /dev/null +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/ProgressReporterConfiguration.kt @@ -0,0 +1,50 @@ +package com.malinskiy.marathon.config + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes( + JsonSubTypes.Type(value = ProgressConfiguration.Auto::class, name = "auto"), + JsonSubTypes.Type(value = ProgressConfiguration.Custom::class, name = "custom"), +) + +sealed class ProgressConfiguration { + data object Auto : ProgressConfiguration() + data class Custom(val values: List) : ProgressConfiguration() +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes( + JsonSubTypes.Type(value = ProgressReporterConfiguration.Stdout::class, name = "stdout"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.Teamcity::class, name = "teamcity"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.Device::class, name = "device"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.Billing::class, name = "billing"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.JUnit::class, name = "junit"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.Timeline::class, name = "timeline"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.Raw::class, name = "raw"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.Allure::class, name = "allure"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.Html::class, name = "html"), + JsonSubTypes.Type(value = ProgressReporterConfiguration.ApiV1::class, name = "api_v1"), +) +sealed class ProgressReporterConfiguration { + data object Stdout : ProgressReporterConfiguration() + data object Teamcity : ProgressReporterConfiguration() + data object Device : ProgressReporterConfiguration() + data object Billing : ProgressReporterConfiguration() + data object JUnit : ProgressReporterConfiguration() + data object Timeline : ProgressReporterConfiguration() + data object Raw : ProgressReporterConfiguration() + data object Test : ProgressReporterConfiguration() + data object Allure : ProgressReporterConfiguration() + data object Html : ProgressReporterConfiguration() + data class ApiV1(val target: String, val run_id: String) : ProgressReporterConfiguration() +} diff --git a/core/build.gradle.kts b/core/build.gradle.kts index d9346d815..fd70c5e83 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -1,4 +1,6 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet +import com.google.protobuf.gradle.id +import com.google.protobuf.gradle.remove plugins { idea @@ -7,6 +9,7 @@ plugins { id("org.jetbrains.dokka") jacoco id("com.github.gmazzo.buildconfig") version "5.5.0" + id("com.google.protobuf") version Versions.protobufGradle } sourceSets { @@ -62,6 +65,11 @@ dependencies { implementation(Libraries.scalr) api(Libraries.koin) api(Libraries.bugsnag) + api(Libraries.protobufLite) + api(Libraries.grpcProtobufLite) + api(Libraries.grpcKotlinStub) + api(Libraries.grpcOkhttp) + api(Libraries.grpcStub) testImplementation(project(":vendor:vendor-test")) testImplementation(TestLibraries.junit5) testImplementation(TestLibraries.kluent) @@ -90,3 +98,38 @@ val integrationTest = task("integrationTest") { setupDeployment() setupKotlinCompiler() setupTestTask() + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:${Versions.protobuf}" + } + plugins { + id("java") { + artifact = "io.grpc:protoc-gen-grpc-java:${Versions.grpc}" + } + id("grpc") { + artifact = "io.grpc:protoc-gen-grpc-java:${Versions.grpc}" + } + id("grpckt") { + artifact = "io.grpc:protoc-gen-grpc-kotlin:${Versions.grpcKotlin}:jdk8@jar" + } + } + generateProtoTasks { + all().forEach { + it.builtins { + remove("java") + } + it.plugins { + id("java") { + option("lite") + } + id("grpc") { + option("lite") + } + id("grpckt") { + option("lite") + } + } + } + } +} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt index cb54b159b..3f7f45e51 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt @@ -21,6 +21,7 @@ import com.malinskiy.marathon.execution.TestParser import com.malinskiy.marathon.execution.TestShard import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.execution.withRetry import com.malinskiy.marathon.extension.toFlakinessStrategy import com.malinskiy.marathon.extension.toShardingStrategy @@ -54,6 +55,7 @@ class Marathon( private val tracker: TrackerInternal, private val analytics: Analytics, private val track: Track, + private val progressReporter: ProgressReporter, private val timer: Timer, private val marathonTestParseCommand: MarathonTestParseCommand, private val usageTracker: UsageTracker, @@ -155,6 +157,7 @@ class Marathon( val shard = prepareTestShard(parsedFilteredTests, analytics) + progressReporter.begin(parsedFilteredTests) usageTracker.trackEvent(Event.TestsTotal(parsedAllTests.size)) usageTracker.trackEvent(Event.TestsRun(parsedFilteredTests.size)) @@ -167,6 +170,7 @@ class Marathon( configuration, shard, track, + progressReporter, timer, testBundleIdentifier, currentCoroutineContext diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/TrackerFactory.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/TrackerFactory.kt index 6dbdf4faa..5fa0d9be7 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/TrackerFactory.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/TrackerFactory.kt @@ -1,6 +1,5 @@ package com.malinskiy.marathon.analytics -import com.google.gson.Gson import com.malinskiy.marathon.analytics.external.graphite.BasicGraphiteClient import com.malinskiy.marathon.analytics.external.graphite.GraphiteTracker import com.malinskiy.marathon.analytics.external.influx.InfluxDbProvider @@ -15,29 +14,13 @@ import com.malinskiy.marathon.config.AnalyticsConfiguration.GraphiteConfiguratio import com.malinskiy.marathon.config.AnalyticsConfiguration.InfluxDb2Configuration import com.malinskiy.marathon.config.AnalyticsConfiguration.InfluxDbConfiguration import com.malinskiy.marathon.config.Configuration -import com.malinskiy.marathon.io.FileManager +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.log.MarathonLogging -import com.malinskiy.marathon.report.allure.AllureReporter -import com.malinskiy.marathon.report.bill.BillingReporter -import com.malinskiy.marathon.report.device.DeviceInfoJsonReporter -import com.malinskiy.marathon.report.html.HtmlSummaryReporter -import com.malinskiy.marathon.report.junit.JUnitReporter -import com.malinskiy.marathon.report.raw.RawJsonReporter -import com.malinskiy.marathon.report.stdout.StdoutReporter -import com.malinskiy.marathon.report.test.TestJsonReporter -import com.malinskiy.marathon.report.timeline.TimelineReporter -import com.malinskiy.marathon.report.timeline.TimelineSummaryProvider -import com.malinskiy.marathon.time.Timer -import com.malinskiy.marathon.usageanalytics.tracker.UsageTracker -import java.io.File internal class TrackerFactory( private val configuration: Configuration, - private val fileManager: FileManager, - private val gson: Gson, - private val timer: Timer, private val track: Track, - private val usageTracker: UsageTracker, + private val progressReporter: ProgressReporter, ) { val log = MarathonLogging.logger("TrackerFactory") @@ -88,18 +71,6 @@ internal class TrackerFactory( } private fun createExecutionReportGenerator(): ExecutionReportGenerator { - return ExecutionReportGenerator( - listOf( - DeviceInfoJsonReporter(fileManager, gson), - BillingReporter(fileManager, gson, usageTracker), - JUnitReporter(configuration.outputDir), - TimelineReporter(TimelineSummaryProvider(), gson, configuration.outputDir), - RawJsonReporter(fileManager, gson), - TestJsonReporter(fileManager, gson), - AllureReporter(configuration, File(configuration.outputDir, "allure-results")), - HtmlSummaryReporter(gson, fileManager, configuration.outputDir, configuration), - StdoutReporter(timer) - ) - ) + return ExecutionReportGenerator(progressReporter) } } diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReport.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReport.kt index 70ae0ade0..80d6567b8 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReport.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReport.kt @@ -4,8 +4,12 @@ import com.malinskiy.marathon.device.DeviceInfo import com.malinskiy.marathon.device.DevicePoolId import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.log.MarathonLogging +import com.malinskiy.marathon.report.bill.DeviceBill import com.malinskiy.marathon.test.toTestName +import java.time.Duration import java.time.Instant +import java.time.temporal.ChronoUnit /** * Events are sorted by timestamp @@ -15,14 +19,67 @@ data class ExecutionReport( val deviceDisconnectedEvents: List, val devicePreparingEvents: List, val deviceProviderPreparingEvent: List, - val testEvents: List + val testEvents: List, + val defaultStart: Instant, ) { + private val logger = MarathonLogging.logger {} + val summary: Summary by lazy { val pools = deviceConnectedEvents.map { it.poolId }.distinct() val poolsSummary: List = pools.map { compilePoolSummary(it) } Summary(poolsSummary) } + val result: Boolean by lazy { summary.pools.map { it.failed.isEmpty() }.reduceOrNull { acc, b -> acc && b } ?: true } + val flakiness: Duration by lazy { + summary.pools.mapNotNull { + val poolFlakiness = it.rawDurationMillis - it.durationMillis + if (poolFlakiness < 0) { + logger.warn { "Pool ${it.poolId.name} has negative flakiness" } + null + } else { + Duration.of(poolFlakiness, ChronoUnit.MILLIS) + } + }.fold(Duration.ZERO) { acc, duration -> acc + duration} + } + val duration: Duration by lazy { Duration.between(defaultStart, Instant.now()) } + + val bills: List by lazy { + val starts = mutableMapOf() + val devices = deviceConnectedEvents.associateBy { it.device.serialNumber }.mapValues { it.value.device } + val pools = deviceConnectedEvents.associateBy { it.device.serialNumber }.mapValues { it.value.poolId } + + devicePreparingEvents.forEach { + if (starts.contains(it.serialNumber)) { + //Only replace if event finish is before current one + if (starts[it.serialNumber]?.isAfter(it.finish) == true) { + starts[it.serialNumber] = it.finish + } + } else { + starts[it.serialNumber] = it.finish + } + } + + val testEventsByDeviceSerial = testEvents.groupBy { it.device.serialNumber } + val ends = testEventsByDeviceSerial.mapValues { deviceEvents -> + deviceEvents.value.maxByOrNull { it.instant }?.instant + } + + val serials = starts.keys + ends.keys + serials.mapNotNull { + val start = starts[it] ?: return@mapNotNull null + val end = ends[it] ?: return@mapNotNull null + val info = devices[it] + val pool = pools[it] + if (info != null && pool != null) { + DeviceBill(info, pool, start, end, Duration.between(start, end)) + } else { + logger.warn { "Failure to process device bill: missing timeline event" } + null + } + } + } + private fun compilePoolSummary(poolId: DevicePoolId): PoolSummary { val devices = deviceConnectedEvents.filter { it.poolId == poolId }.map { it.device }.distinctBy { it.serialNumber } @@ -35,13 +92,14 @@ data class ExecutionReport( val passed = tests .filter { it.status == TestStatus.PASSED } - .map { it.test.toTestName() } + .map { it.test } .toSet() val ignored = tests - .filter { it.status == TestStatus.IGNORED - || it.status == TestStatus.ASSUMPTION_FAILURE - }.map { it.test.toTestName() } + .filter { + it.status == TestStatus.IGNORED + || it.status == TestStatus.ASSUMPTION_FAILURE + }.map { it.test } .toSet() val failed = tests @@ -49,7 +107,7 @@ data class ExecutionReport( it.status != TestStatus.PASSED && it.status != TestStatus.IGNORED && it.status != TestStatus.ASSUMPTION_FAILURE - }.map { it.test.toTestName() } + }.map { it.test } .toSet() val duration = tests.filter { it.isTimeInfoAvailable } @@ -59,17 +117,18 @@ data class ExecutionReport( .map { it.testResult } val rawPassed = rawTests .filter { it.status == TestStatus.PASSED } - .map { it.test.toTestName() } + .map { it.test } val rawIgnored = rawTests - .filter { it.status == TestStatus.IGNORED - || it.status == TestStatus.ASSUMPTION_FAILURE - }.map { it.test.toTestName() } + .filter { + it.status == TestStatus.IGNORED + || it.status == TestStatus.ASSUMPTION_FAILURE + }.map { it.test } val rawFailed = rawTests .filter { it.status == TestStatus.FAILURE } - .map { it.test.toTestName() } + .map { it.test } val rawIncomplete = rawTests .filter { it.status == TestStatus.INCOMPLETE } - .map { it.test.toTestName() } + .map { it.test } val rawDuration = rawTests //Incomplete tests mess up the calculations of time since their end time is 0 and duration is, hence, years //We filter here for unavailable time just to be safe diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReportGenerator.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReportGenerator.kt index 159b469ca..7d5ea384d 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReportGenerator.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/ExecutionReportGenerator.kt @@ -1,14 +1,16 @@ package com.malinskiy.marathon.analytics.internal.sub -import com.malinskiy.marathon.report.Reporter +import com.malinskiy.marathon.report.ProgressReporter +import java.time.Instant import java.util.* -class ExecutionReportGenerator(private val reporters: List) : TrackerInternal { +class ExecutionReportGenerator(private val progressReporter: ProgressReporter) : TrackerInternal { private val devicePreparingEvents: MutableList = Collections.synchronizedList(LinkedList()) private val deviceConnectedEvents: MutableList = Collections.synchronizedList(LinkedList()) private val deviceDisconnectedEvents: MutableList = Collections.synchronizedList(LinkedList()) private val deviceProviderPreparingEvents: MutableList = Collections.synchronizedList(LinkedList()) private val testEvents: MutableList = Collections.synchronizedList(mutableListOf()) + private val defaultStart = Instant.now() override fun track(event: Event) { when (event) { @@ -26,10 +28,9 @@ class ExecutionReportGenerator(private val reporters: List) : TrackerI deviceDisconnectedEvents.sortedBy { it.instant }, devicePreparingEvents.sortedBy { it.start }, deviceProviderPreparingEvents.sortedBy { it.start }, - testEvents.sortedBy { it.instant }) - - for (reporter in reporters) { - reporter.generate(report) - } + testEvents.sortedBy { it.instant }, + defaultStart + ) + progressReporter.end(report) } } diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/PoolSummary.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/PoolSummary.kt index 419602771..ba469809c 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/PoolSummary.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/internal/sub/PoolSummary.kt @@ -3,20 +3,21 @@ package com.malinskiy.marathon.analytics.internal.sub import com.malinskiy.marathon.device.DeviceInfo import com.malinskiy.marathon.device.DevicePoolId import com.malinskiy.marathon.execution.TestResult +import com.malinskiy.marathon.test.Test data class PoolSummary( val poolId: DevicePoolId, val tests: List, val retries: Map>, - val passed: Set, - val ignored: Set, - val failed: Set, + val passed: Set, + val ignored: Set, + val failed: Set, val flaky: Int, val durationMillis: Long, val devices: List, - val rawPassed: List, - val rawIgnored: List, - val rawFailed: List, - val rawIncomplete: List, + val rawPassed: List, + val rawIgnored: List, + val rawFailed: List, + val rawIncomplete: List, val rawDurationMillis: Long ) diff --git a/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt b/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt index a863a3676..47124ed3b 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt @@ -7,6 +7,8 @@ import com.malinskiy.marathon.analytics.external.AnalyticsFactory import com.malinskiy.marathon.analytics.internal.pub.Track import com.malinskiy.marathon.config.Configuration import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand +import com.malinskiy.marathon.report.ProgressReporter +import com.malinskiy.marathon.report.ProgressReporterFactory import com.malinskiy.marathon.io.FileManager import com.malinskiy.marathon.json.FileSerializer import com.malinskiy.marathon.time.SystemTimer @@ -23,11 +25,11 @@ import java.time.Clock val analyticsModule = module { single { Track() } - single { TrackerFactory(get(), get(), get(), get(), get(), get()).create() } + single { TrackerFactory(get(), get(), get()).create() } single { AnalyticsFactory(get()).create() } single { val configuration = get() - if(configuration.analyticsTracking) { + if (configuration.analyticsTracking) { GrafanaCloud() } else { EmptyTracker() @@ -51,7 +53,11 @@ val coreModule = module { val configuration = get() MarathonTestParseCommand(configuration.outputDir) } - single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) } + single { + val factory = ProgressReporterFactory(get(), get(), get(), get(), get()) + factory.create() + } + single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) } } fun marathonStartKoin(configuration: Configuration, modules: List): KoinApplication { diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt index 6bf823235..d534a0ce6 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt @@ -12,6 +12,7 @@ import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler.AddDevic import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler.RemoveDevice import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.extension.toPoolingStrategy import com.malinskiy.marathon.log.MarathonLogging import com.malinskiy.marathon.time.Timer @@ -36,6 +37,7 @@ class Scheduler( private val configuration: Configuration, private val shard: TestShard, private val track: Track, + private val progressReporter: ProgressReporter, private val timer: Timer, private val testBundleIdentifier: TestBundleIdentifier?, override val coroutineContext: CoroutineContext @@ -112,7 +114,7 @@ class Scheduler( val poolId = poolingStrategy.associate(device) logger.debug { "device ${device.serialNumber} associated with poolId ${poolId.name}" } val accumulator = results.computeIfAbsent(poolId) { id -> - PoolProgressAccumulator(id, shard, configuration, track) + PoolProgressAccumulator(id, shard, configuration, track, progressReporter) } pools.computeIfAbsent(poolId) { id -> diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt index e84d7d1eb..0c7fe4052 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt @@ -12,22 +12,21 @@ import com.malinskiy.marathon.execution.TestStatus import com.malinskiy.marathon.execution.queue.TestAction import com.malinskiy.marathon.execution.queue.TestEvent import com.malinskiy.marathon.execution.queue.TestState -import com.malinskiy.marathon.integrations.ci.CIIntegrationFactory import com.malinskiy.marathon.log.MarathonLogging +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.test.Test import com.malinskiy.marathon.test.toTestName -import kotlin.math.roundToInt class PoolProgressAccumulator( private val poolId: DevicePoolId, shard: TestShard, configuration: Configuration, - private val track: Track + private val track: Track, + private val progressReporter: ProgressReporter, ) { private val tests: HashMap = HashMap() private val logger = MarathonLogging.logger {} private val executionStrategy = configuration.executionStrategy - private val ci = CIIntegrationFactory.get(configuration) private fun createState(initialCount: Int) = StateMachine.create { initialState(TestState.Added(initialCount)) @@ -267,33 +266,38 @@ class PoolProgressAccumulator( fun testStarted(device: DeviceInfo, test: Test) { transition(test, TestEvent.Started) - ci.setBuildProgress(progress().toInt()) - println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started") + progressReporter.testStarted(progress(), poolId.name, device.serialNumber, test.toTestName()) } /** + * Side effect should be actioned before reporting progress to account for the current test effect on the progress + * * @param final used for incomplete tests to signal no more retries left, hence a decision on the status has to be made */ fun testEnded(device: DeviceInfo, testResult: TestResult, final: Boolean = false): TestAction? { return when (testResult.status) { TestStatus.FAILURE -> { - println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} failed") - transition(testResult.test, TestEvent.Failed(device, testResult)).sideffect() + val sideffect = transition(testResult.test, TestEvent.Failed(device, testResult)).sideffect() + progressReporter.testFailed(progress(), poolId.name, device.serialNumber, testResult.test.toTestName()) + sideffect } TestStatus.PASSED -> { - println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} passed") - transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect() + val sideffect = transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect() + progressReporter.testPassed(progress(), poolId.name, device.serialNumber, testResult.test.toTestName()) + sideffect } TestStatus.IGNORED, TestStatus.ASSUMPTION_FAILURE -> { - println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ignored") - transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect() + val sideffect = transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect() + progressReporter.testIgnored(progress(), poolId.name, device.serialNumber, testResult.test.toTestName()) + sideffect } TestStatus.INCOMPLETE -> { - println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} incomplete") - transition(testResult.test, TestEvent.Incomplete(device, testResult, final)).sideffect() + val sideffect = transition(testResult.test, TestEvent.Incomplete(device, testResult, final)).sideffect() + progressReporter.testIncomplete(progress(), poolId.name, device.serialNumber, testResult.test.toTestName()) + sideffect } } } @@ -422,16 +426,6 @@ class PoolProgressAccumulator( } return testActionTransition } - - private fun toPercent(float: Float): String { - val percent = (float * HUNDRED_PERCENT_IN_FLOAT).roundToInt() - val format = "%02d%%" - return String.format(format, percent) - } - - companion object { - const val HUNDRED_PERCENT_IN_FLOAT: Float = 100.0f - } } private fun StateMachine.Transition?.sideffect(): SIDE_EFFECT? { diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt deleted file mode 100644 index 19be2590d..000000000 --- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt +++ /dev/null @@ -1,25 +0,0 @@ -package com.malinskiy.marathon.execution.progress - -import com.malinskiy.marathon.device.DeviceInfo -import com.malinskiy.marathon.device.DevicePoolId -import com.malinskiy.marathon.test.Test -import com.malinskiy.marathon.test.TestBatch -import com.malinskiy.marathon.test.toTestName - -class ProgressReporter(private val batch: TestBatch, private val poolId: DevicePoolId, private val device: DeviceInfo) { - fun testStarted(test: Test) { - println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started") - } - - fun testFailed(test: Test) { - println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} failed") - } - - fun testPassed(test: Test) { - println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} passed") - } - - fun testIgnored(test: Test) { - println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ignored") - } -} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CI.kt b/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CI.kt deleted file mode 100644 index 9688d74ae..000000000 --- a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CI.kt +++ /dev/null @@ -1,5 +0,0 @@ -package com.malinskiy.marathon.integrations.ci - -interface CI { - fun setBuildProgress(int: Int) -} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CIIntegrationFactory.kt b/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CIIntegrationFactory.kt deleted file mode 100644 index 9b77f27d8..000000000 --- a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/CIIntegrationFactory.kt +++ /dev/null @@ -1,18 +0,0 @@ -package com.malinskiy.marathon.integrations.ci - -import com.malinskiy.marathon.config.CIConfiguration -import com.malinskiy.marathon.config.Configuration - -object CIIntegrationFactory { - fun get(configuration: Configuration) = when (configuration.ciConfiguration) { - CIConfiguration.Auto -> auto() - CIConfiguration.None -> None - CIConfiguration.Teamcity -> Teamcity - } - - private fun auto() = when { - // Teamcity predefined variables: https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#84e0b866 - System.getenv().containsValue("TEAMCITY_VERSION") -> Teamcity - else -> None - } -} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/None.kt b/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/None.kt deleted file mode 100644 index ea79c04cd..000000000 --- a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/None.kt +++ /dev/null @@ -1,5 +0,0 @@ -package com.malinskiy.marathon.integrations.ci - -object None: CI { - override fun setBuildProgress(int: Int) = Unit -} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/Teamcity.kt b/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/Teamcity.kt deleted file mode 100644 index 34c1c2926..000000000 --- a/core/src/main/kotlin/com/malinskiy/marathon/integrations/ci/Teamcity.kt +++ /dev/null @@ -1,9 +0,0 @@ -package com.malinskiy.marathon.integrations.ci - -object Teamcity: CI { - private fun setBuildMessage(message: String) = println("##teamcity[buildStatus text='$message']") - - private fun setKeyValue(key: String, value: String) = println("##teamcity[buildStatisticValue key='$key' value='$value']") - - override fun setBuildProgress(int: Int) = setBuildMessage("Marathon run: $int %") -} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/CompositeProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/CompositeProgressReporter.kt new file mode 100644 index 000000000..cb0301431 --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/CompositeProgressReporter.kt @@ -0,0 +1,34 @@ +package com.malinskiy.marathon.report + +import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport +import com.malinskiy.marathon.test.Test + +class CompositeProgressReporter(private val delegates: Collection) : ProgressReporter { + override fun begin(parsedFilteredTests: List) { + delegates.forEach { it.begin(parsedFilteredTests) } + } + + override fun testStarted(progress: Float, poolId: String, deviceSerial: String, testName: String) { + delegates.forEach { it.testStarted(progress, poolId, deviceSerial, testName) } + } + + override fun testFailed(progress: Float, poolId: String, deviceSerial: String, testName: String) { + delegates.forEach { it.testFailed(progress, poolId, deviceSerial, testName) } + } + + override fun testPassed(progress: Float, poolId: String, deviceSerial: String, testName: String) { + delegates.forEach { it.testPassed(progress, poolId, deviceSerial, testName) } + } + + override fun testIgnored(progress: Float, poolId: String, deviceSerial: String, testName: String) { + delegates.forEach { it.testIgnored(progress, poolId, deviceSerial, testName) } + } + + override fun testIncomplete(progress: Float, poolId: String, deviceSerial: String, testName: String) { + delegates.forEach { it.testIncomplete(progress, poolId, deviceSerial, testName) } + } + + override fun end(executionReport: ExecutionReport) { + delegates.forEach { it.end(executionReport) } + } +} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/NoopProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/NoopProgressReporter.kt new file mode 100644 index 000000000..9eb77780e --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/NoopProgressReporter.kt @@ -0,0 +1,3 @@ +package com.malinskiy.marathon.report + +object NoopProgressReporter : ProgressReporter diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporter.kt new file mode 100644 index 000000000..98cce8671 --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporter.kt @@ -0,0 +1,16 @@ +package com.malinskiy.marathon.report + +import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport +import com.malinskiy.marathon.test.Test + +interface ProgressReporter { + fun begin(parsedFilteredTests: List) = Unit + + fun testStarted(progress: Float, poolId: String, deviceSerial: String, testName: String) = Unit + fun testFailed(progress: Float, poolId: String, deviceSerial: String, testName: String) = Unit + fun testPassed(progress: Float, poolId: String, deviceSerial: String, testName: String) = Unit + fun testIgnored(progress: Float, poolId: String, deviceSerial: String, testName: String) = Unit + fun testIncomplete(progress: Float, poolId: String, deviceSerial: String, testName: String) = Unit + + fun end(executionReport: ExecutionReport) = Unit +} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporterFactory.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporterFactory.kt new file mode 100644 index 000000000..2eb2ef076 --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/ProgressReporterFactory.kt @@ -0,0 +1,84 @@ +package com.malinskiy.marathon.report + +import com.google.gson.Gson +import com.malinskiy.marathon.config.Configuration +import com.malinskiy.marathon.config.ProgressConfiguration +import com.malinskiy.marathon.config.ProgressReporterConfiguration +import com.malinskiy.marathon.io.FileManager +import com.malinskiy.marathon.report.allure.AllureReporter +import com.malinskiy.marathon.report.api.ApiV1ProgressReporter +import com.malinskiy.marathon.report.bill.BillingReporter +import com.malinskiy.marathon.report.device.DeviceInfoJsonReporter +import com.malinskiy.marathon.report.html.HtmlSummaryReporter +import com.malinskiy.marathon.report.junit.JUnitReporter +import com.malinskiy.marathon.report.raw.RawJsonReporter +import com.malinskiy.marathon.report.stdout.StdoutProgressReporter +import com.malinskiy.marathon.report.teamcity.TeamCityProgressReporter +import com.malinskiy.marathon.report.test.TestJsonReporter +import com.malinskiy.marathon.report.timeline.TimelineReporter +import com.malinskiy.marathon.report.timeline.TimelineSummaryProvider +import com.malinskiy.marathon.time.Timer +import com.malinskiy.marathon.usageanalytics.tracker.UsageTracker +import java.io.File + +class ProgressReporterFactory( + private val configuration: Configuration, + private val fileManager: FileManager, + private val gson: Gson, + private val usageTracker: UsageTracker, + private val timer: Timer +) { + fun create(): CompositeProgressReporter { + when (configuration.progressConfiguration) { + ProgressConfiguration.Auto -> { + //Map to a default custom configuration + val reporters = mutableListOf( + ProgressReporterConfiguration.Device, + ProgressReporterConfiguration.Billing, + ProgressReporterConfiguration.JUnit, + ProgressReporterConfiguration.Timeline, + ProgressReporterConfiguration.Raw, + ProgressReporterConfiguration.Test, + ProgressReporterConfiguration.Allure, + ProgressReporterConfiguration.Html, + ProgressReporterConfiguration.Stdout, + ) + // Teamcity predefined variables: https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#84e0b866 + if (System.getenv().containsValue("TEAMCITY_VERSION")) { + reporters.add(ProgressReporterConfiguration.Teamcity) + } + + return create(ProgressConfiguration.Custom(reporters)) + } + + is ProgressConfiguration.Custom -> { + return create(configuration.progressConfiguration as ProgressConfiguration.Custom) + } + } + + } + + fun create(cfg: ProgressConfiguration.Custom): CompositeProgressReporter { + val reporters: List = cfg.values.map { + val reporter = when (it) { + is ProgressReporterConfiguration.ApiV1 -> ApiV1ProgressReporter(it.target, it.run_id) + ProgressReporterConfiguration.Stdout -> StdoutProgressReporter(timer) + ProgressReporterConfiguration.Teamcity -> TeamCityProgressReporter() + ProgressReporterConfiguration.Allure -> AllureReporter( + configuration, + File(configuration.outputDir, "allure-results") + ) + + ProgressReporterConfiguration.Billing -> BillingReporter(fileManager, gson, usageTracker) + ProgressReporterConfiguration.Device -> DeviceInfoJsonReporter(fileManager, gson) + ProgressReporterConfiguration.Html -> HtmlSummaryReporter(gson, fileManager, configuration.outputDir, configuration) + ProgressReporterConfiguration.JUnit -> JUnitReporter(configuration.outputDir) + ProgressReporterConfiguration.Raw -> RawJsonReporter(fileManager, gson) + ProgressReporterConfiguration.Test -> TestJsonReporter(fileManager, gson) + ProgressReporterConfiguration.Timeline -> TimelineReporter(TimelineSummaryProvider(), gson, configuration.outputDir) + } + reporter + }.toList() + return CompositeProgressReporter(reporters) + } +} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/Reporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/Reporter.kt deleted file mode 100644 index 08a31fa1d..000000000 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/Reporter.kt +++ /dev/null @@ -1,7 +0,0 @@ -package com.malinskiy.marathon.report - -import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport - -interface Reporter { - fun generate(executionReport: ExecutionReport) -} \ No newline at end of file diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt index 8d56e5420..d2d93a393 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt @@ -9,8 +9,8 @@ import com.malinskiy.marathon.device.DeviceInfo import com.malinskiy.marathon.execution.AttachmentType import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.extension.relativePathTo -import com.malinskiy.marathon.report.Reporter import com.malinskiy.marathon.test.Test import com.malinskiy.marathon.test.toClassName import com.malinskiy.marathon.test.toSafeTestName @@ -45,11 +45,11 @@ import io.qameta.allure.kotlin.SeverityLevel as KotlinSeverityLevel import io.qameta.allure.kotlin.Story as KotlinStory import io.qameta.allure.kotlin.TmsLink as KotlinTmsLink -class AllureReporter(val configuration: Configuration, private val outputDirectory: File) : Reporter { +class AllureReporter(val configuration: Configuration, private val outputDirectory: File) : ProgressReporter { private val lifecycle: AllureLifecycle by lazy { AllureLifecycle(FileSystemResultsWriter(outputDirectory.toPath())) } - override fun generate(executionReport: ExecutionReport) { + override fun end(executionReport: ExecutionReport) { executionReport.testEvents.forEach { testEvent -> val uuid = UUID.randomUUID().toString() val allureResults = createTestResult(uuid, testEvent.device, testEvent.testResult) diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/api/ApiV1ProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/api/ApiV1ProgressReporter.kt new file mode 100644 index 000000000..b217f3da0 --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/api/ApiV1ProgressReporter.kt @@ -0,0 +1,216 @@ +package com.malinskiy.marathon.report.api + +import com.google.protobuf.Duration +import com.google.protobuf.Timestamp +import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport +import com.malinskiy.marathon.coroutines.newCoroutineExceptionHandler +import com.malinskiy.marathon.log.MarathonLogging +import com.malinskiy.marathon.report.ProgressReporter +import com.malinskiy.marathon.test.Test +import com.malinskiy.marathon.test.toTestName +import io.grpc.ManagedChannelBuilder +import io.marathonlabs.testing.progress.DeviceBill +import io.marathonlabs.testing.progress.ProgressServiceGrpcKt +import io.marathonlabs.testing.progress.StartRequest +import io.marathonlabs.testing.progress.Stats +import io.marathonlabs.testing.progress.TestStat +import io.marathonlabs.testing.progress.TrackRequest +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.newFixedThreadPoolContext +import kotlinx.coroutines.runBlocking +import java.time.Instant + +class ApiV1ProgressReporter(private val target: String, private val runId: String) : ProgressReporter, CoroutineScope { + private val logger = MarathonLogging.logger(ApiV1ProgressReporter::class.java.simpleName) + private val dispatcher by lazy { + newFixedThreadPoolContext(1, "ApiV1ProgressReporter") + } + + override val coroutineContext = dispatcher + newCoroutineExceptionHandler(logger) + + private val client: ProgressServiceGrpcKt.ProgressServiceCoroutineStub by lazy { + val channel = ManagedChannelBuilder.forTarget(target) + .usePlaintext() + .build() + ProgressServiceGrpcKt.ProgressServiceCoroutineStub(channel) + } + + private var pendingUpdate: TrackRequest? = null + + override fun begin(parsedFilteredTests: List) { + val testPlan = StartRequest.newBuilder() + .addAllTests(parsedFilteredTests.into()) + .setRunId(runId) + .build() + runBlocking { + client.start(testPlan) + } + } + + override fun testStarted(progress: Float, poolId: String, deviceSerial: String, testName: String) { + synchronized(this) { + if (pendingUpdate != null) { + TrackRequest.newBuilder(pendingUpdate) + .build() + } else { + null + } + } + } + + override fun testFailed(progress: Float, poolId: String, deviceSerial: String, testName: String) { + synchronized(this) { + if (pendingUpdate != null) { + TrackRequest.newBuilder(pendingUpdate) + .build() + } else { + null + } + } + } + + override fun testPassed(progress: Float, poolId: String, deviceSerial: String, testName: String) { + synchronized(this) { + if (pendingUpdate != null) { + TrackRequest.newBuilder(pendingUpdate) + .build() + } else { + null + } + } + } + + override fun testIgnored(progress: Float, poolId: String, deviceSerial: String, testName: String) { + synchronized(this) { + if (pendingUpdate != null) { + TrackRequest.newBuilder(pendingUpdate) + .build() + } else { + null + } + } + } + + override fun testIncomplete(progress: Float, poolId: String, deviceSerial: String, testName: String) { + synchronized(this) { + if (pendingUpdate != null) { + TrackRequest.newBuilder(pendingUpdate) + .build() + } else { + null + } + } + } + + override fun end(executionReport: ExecutionReport) { + runBlocking { + client.end(executionReport.into()) + } + } + + private fun List.into(): List { + return map { test -> + io.marathonlabs.testing.progress.Test.newBuilder() + .setPkg(test.pkg) + .setClazz(test.clazz) + .setMethod(test.method) + .addAllMetaProperties(test.metaProperties.map { it.name }) + .build() + }.toList() + } + + private fun ExecutionReport.into(): io.marathonlabs.testing.progress.EndRequest { + val cost = bills.fold(java.time.Duration.ZERO) { acc, bill -> acc + bill.duration } + + val passed = summary.pools.sumOf { it.passed.size } + val failed = summary.pools.sumOf { it.failed.size } + val ignored = summary.pools.sumOf { it.ignored.size } + val finalStats = Stats.newBuilder() + .setPassed(passed) + .setFailed(failed) + .setIgnored(ignored) + .setIncomplete(0) //Final tests can't be incomplete - they terminate as failed + .build() + + val rawPassed = summary.pools.sumOf { it.rawPassed.size } + val rawFailed = summary.pools.sumOf { it.rawFailed.size } + val rawIgnored = summary.pools.sumOf { it.rawIgnored.size } + val rawIncomplete = summary.pools.sumOf { it.rawIncomplete.size } + val rawStats = Stats.newBuilder() + .setPassed(rawPassed) + .setFailed(rawFailed) + .setIgnored(rawIgnored) + .setIncomplete(rawIncomplete) + .build() + + val finalFailedTests = summary.pools + .flatMap { it.failed } + .map { + TestStat.newBuilder() + .setTest(it.into()) + .setCount(1) + .build() + } + val rawFailedTests = summary.pools + .flatMap { it.rawFailed } + .groupBy { it.toTestName() } + .toSortedMap() + .map { + TestStat.newBuilder() + .setTest(it.value.first().into()) + .setCount(it.value.size) + .build() + } + val rawIncompleteTests = summary.pools + .flatMap { it.rawIncomplete } + .groupBy { it.toTestName() } + .toSortedMap() + .map { + TestStat.newBuilder() + .setTest(it.value.first().into()) + .setCount(it.value.size) + .build() + } + + return io.marathonlabs.testing.progress.EndRequest.newBuilder() + .setRunId(runId) + .setSuccess(result) + .setFlakiness(convertDuration(flakiness)) + .setCost(convertDuration(cost)) + .setDuration(convertDuration(duration)) + .setRaw(rawStats) + .setFinal(finalStats) + .addAllFinalFailed(finalFailedTests) + .addAllRawFailed(rawFailedTests) + .addAllRawIncomplete(rawIncompleteTests) + .addAllDeviceBill(bills.map { it.into() }) + .build() + } + + private fun com.malinskiy.marathon.report.bill.DeviceBill.into(): DeviceBill { + return DeviceBill.newBuilder() + .setStart(convertTimestamp(start)) + .setEnd(convertTimestamp(end)) + .setDuration(convertDuration(duration)) + .build() + } + + private fun convertDuration(duration: java.time.Duration): Duration = Duration.newBuilder() + .setSeconds(duration.seconds) + .setNanos(duration.nano) + .build() + + private fun convertTimestamp(start: Instant): Timestamp = Timestamp.newBuilder() + .setSeconds(start.epochSecond) + .setNanos(start.nano) + .build() + + private fun Test.into(): io.marathonlabs.testing.progress.Test? { + return io.marathonlabs.testing.progress.Test.newBuilder() + .setPkg(pkg) + .setClazz(clazz) + .setMethod(method) + .addAllMetaProperties(metaProperties.map { it.name }) + .build() + } +} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/bill/BillingReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/bill/BillingReporter.kt index f2aebf63a..358b9b2b9 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/bill/BillingReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/bill/BillingReporter.kt @@ -2,79 +2,28 @@ package com.malinskiy.marathon.report.bill import com.google.gson.Gson import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport -import com.malinskiy.marathon.device.DeviceInfo -import com.malinskiy.marathon.device.DevicePoolId +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.io.FileManager import com.malinskiy.marathon.io.FileType -import com.malinskiy.marathon.log.MarathonLogging -import com.malinskiy.marathon.report.Reporter import com.malinskiy.marathon.usageanalytics.Event import com.malinskiy.marathon.usageanalytics.tracker.UsageTracker -import java.time.Duration -import java.time.Instant internal class BillingReporter( private val fileManager: FileManager, private val gson: Gson, private val usageTracker: UsageTracker, -) : Reporter { - private val defaultStart = Instant.now() - private val logger = MarathonLogging.logger {} - override fun generate(executionReport: ExecutionReport) { - val starts = mutableMapOf() - val devices = executionReport.deviceConnectedEvents.associateBy { it.device.serialNumber }.mapValues { it.value.device } - val pools = executionReport.deviceConnectedEvents.associateBy { it.device.serialNumber }.mapValues { it.value.poolId } - - executionReport.devicePreparingEvents.forEach { - if (starts.contains(it.serialNumber)) { - //Only replace if event finish is before current one - if (starts[it.serialNumber]?.isAfter(it.finish) == true) { - starts[it.serialNumber] = it.finish - } - } else { - starts[it.serialNumber] = it.finish - } - } - - val testEventsByDeviceSerial = executionReport.testEvents.groupBy { it.device.serialNumber } - val ends = testEventsByDeviceSerial.mapValues { deviceEvents -> - deviceEvents.value.maxByOrNull { it.instant }?.instant - } - - val serials = starts.keys + ends.keys - val bills: List = serials.mapNotNull { - val start = starts[it] ?: defaultStart - val end = ends[it] ?: return@mapNotNull null - val info = devices[it] - val pool = pools[it] - if (info != null && pool != null) { - DeviceBill(info, pool, start.toEpochMilli(), end.toEpochMilli(), Duration.between(start, end).toMillis()) - } else { - logger.warn { "Failure to process device bill: missing timeline event" } - null - } - } - - bills.forEach { +) : ProgressReporter { + override fun end(executionReport: ExecutionReport) { + executionReport.bills.forEach { val json = gson.toJson(it) fileManager.createFile(FileType.BILL, it.pool, device = it.device).writeText(json) } - usageTracker.trackEvent(Event.Devices(bills.size)) - val result = executionReport.summary.pools.map { it.failed.isEmpty() }.reduceOrNull { acc, b -> acc && b } ?: true - val flakiness = executionReport.summary.pools.sumOf { (it.rawDurationMillis - it.durationMillis) / 1000 } - val durationSeconds = ((Instant.now().toEpochMilli() - defaultStart.toEpochMilli()) / 1000) - usageTracker.trackEvent(Event.Executed(seconds = bills.sumOf { it.duration } / 1000, - success = result, - flakinessSeconds = flakiness, - durationSeconds = durationSeconds)) + usageTracker.trackEvent(Event.Devices(executionReport.bills.size)) + usageTracker.trackEvent(Event.Executed(seconds = executionReport.bills.sumOf { it.duration.seconds }, + success = executionReport.result, + flakinessSeconds = executionReport.flakiness.seconds, + durationSeconds = executionReport.duration.seconds)) } } -data class DeviceBill( - val device: DeviceInfo, - val pool: DevicePoolId, - val start: Long, - val end: Long, - val duration: Long, -) diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/bill/DeviceBill.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/bill/DeviceBill.kt new file mode 100644 index 000000000..d0fb7af96 --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/bill/DeviceBill.kt @@ -0,0 +1,14 @@ +package com.malinskiy.marathon.report.bill + +import com.malinskiy.marathon.device.DeviceInfo +import com.malinskiy.marathon.device.DevicePoolId +import java.time.Duration +import java.time.Instant + +data class DeviceBill( + val device: DeviceInfo, + val pool: DevicePoolId, + val start: Instant, + val end: Instant, + val duration: Duration, +) diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/device/DeviceInfoJsonReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/device/DeviceInfoJsonReporter.kt index 3e52f7b0c..4e463849e 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/device/DeviceInfoJsonReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/device/DeviceInfoJsonReporter.kt @@ -2,15 +2,15 @@ package com.malinskiy.marathon.report.device import com.google.gson.Gson import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.io.FileManager import com.malinskiy.marathon.io.FileType -import com.malinskiy.marathon.report.Reporter internal class DeviceInfoJsonReporter( private val fileManager: FileManager, private val gson: Gson -) : Reporter { - override fun generate(executionReport: ExecutionReport) { +) : ProgressReporter { + override fun end(executionReport: ExecutionReport) { executionReport.deviceConnectedEvents .forEach { event -> val json = gson.toJson(event.device) diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/html/HtmlSummaryReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/html/HtmlSummaryReporter.kt index 8c6f50fb0..4d559d316 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/html/HtmlSummaryReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/html/HtmlSummaryReporter.kt @@ -11,6 +11,7 @@ import com.malinskiy.marathon.device.DevicePoolId import com.malinskiy.marathon.execution.AttachmentType import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.extension.escape import com.malinskiy.marathon.extension.relativePathTo import com.malinskiy.marathon.io.FileManager @@ -22,7 +23,6 @@ import com.malinskiy.marathon.report.HtmlIndex import com.malinskiy.marathon.report.HtmlPoolSummary import com.malinskiy.marathon.report.HtmlShortTest import com.malinskiy.marathon.report.HtmlTestLogDetails -import com.malinskiy.marathon.report.Reporter import com.malinskiy.marathon.report.Status import com.malinskiy.marathon.test.toClassName import org.apache.commons.text.StringEscapeUtils @@ -38,7 +38,7 @@ class HtmlSummaryReporter( private val fileManager: FileManager, private val rootOutput: File, private val configuration: Configuration -) : Reporter { +) : ProgressReporter { /** * Following file tree structure will be created: @@ -46,7 +46,7 @@ class HtmlSummaryReporter( * - suites/suiteId.json * - suites/deviceId/testId.json */ - override fun generate(executionReport: ExecutionReport) { + override fun end(executionReport: ExecutionReport) { val summary = executionReport.summary if (summary.pools.isEmpty()) return diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitReporter.kt index 9fd473539..4d1538aad 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitReporter.kt @@ -5,8 +5,8 @@ import com.malinskiy.marathon.analytics.internal.sub.PoolSummary import com.malinskiy.marathon.device.DevicePoolId import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.io.FileType -import com.malinskiy.marathon.report.Reporter import com.malinskiy.marathon.report.junit.model.Failure import com.malinskiy.marathon.report.junit.model.JUnitReport import com.malinskiy.marathon.report.junit.model.Rerun @@ -25,11 +25,11 @@ private val FORMATTER = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).app timeZone = TimeZone.getTimeZone("UTC") } -internal class JUnitReporter(private val outputDir: File) : Reporter { +internal class JUnitReporter(private val outputDir: File) : ProgressReporter { private val reportName = "marathon_junit_report" private val serializer: JUnitReportSerializer = JUnitReportSerializer() - override fun generate(executionReport: ExecutionReport) { + override fun end(executionReport: ExecutionReport) { val junitReports = hashMapOf() executionReport.summary.pools.forEach { poolSummary -> val poolId = poolSummary.poolId diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/raw/RawJsonReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/raw/RawJsonReporter.kt index 66df01f58..1ba85074d 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/raw/RawJsonReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/raw/RawJsonReporter.kt @@ -4,15 +4,15 @@ import com.google.gson.Gson import com.google.gson.annotations.SerializedName import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.io.FileManager -import com.malinskiy.marathon.report.Reporter class RawJsonReporter( private val fileManager: FileManager, private val gson: Gson -) : Reporter { +) : ProgressReporter { - override fun generate(executionReport: ExecutionReport) { + override fun end(executionReport: ExecutionReport) { val testResults = executionReport.testEvents.map { val metaPropertiesList = it.testResult.test.metaProperties.map { metaProp -> val valuesAsStringMap = metaProp.values.mapValues { (_, value) -> diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/stdout/StdoutReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/stdout/StdoutProgressReporter.kt similarity index 50% rename from core/src/main/kotlin/com/malinskiy/marathon/report/stdout/StdoutReporter.kt rename to core/src/main/kotlin/com/malinskiy/marathon/report/stdout/StdoutProgressReporter.kt index 8fe673325..784576251 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/stdout/StdoutReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/stdout/StdoutProgressReporter.kt @@ -1,12 +1,49 @@ package com.malinskiy.marathon.report.stdout import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport -import com.malinskiy.marathon.report.Reporter +import com.malinskiy.marathon.report.ProgressReporter +import com.malinskiy.marathon.test.toTestName import com.malinskiy.marathon.time.Timer import org.apache.commons.lang3.time.DurationFormatUtils +import kotlin.math.roundToInt -class StdoutReporter(private val timer: Timer) : Reporter { - override fun generate(executionReport: ExecutionReport) { +class StdoutProgressReporter(private val timer: Timer) : ProgressReporter { + override fun testFailed(progress: Float, poolId: String, deviceSerial: String, testName: String) { + post("failed", progress, poolId, deviceSerial, testName) + } + + override fun testPassed(progress: Float, poolId: String, deviceSerial: String, testName: String) { + post("passed", progress, poolId, deviceSerial, testName) + } + + override fun testIgnored(progress: Float, poolId: String, deviceSerial: String, testName: String) { + post("ignored", progress, poolId, deviceSerial, testName) + } + + override fun testStarted(progress: Float, poolId: String, deviceSerial: String, testName: String) { + post("started", progress, poolId, deviceSerial, testName) + } + + override fun testIncomplete(progress: Float, poolId: String, deviceSerial: String, testName: String) { + post("incomplete", progress, poolId, deviceSerial, testName) + } + + private fun post(status: String, progress: Float, poolId: String, deviceSerial: String, testName: String) { + println("${toPercent(progress)} | [$poolId]-[$deviceSerial] $testName} $status") + } + + private fun toPercent(float: Float): String { + val percent = (float * HUNDRED_PERCENT_IN_FLOAT).roundToInt() + val format = "%02d%%" + return String.format(format, percent) + } + + + companion object { + const val HUNDRED_PERCENT_IN_FLOAT: Float = 100.0f + } + + override fun end(executionReport: ExecutionReport) { val summary = executionReport.summary if (summary.pools.isEmpty()) return @@ -18,6 +55,7 @@ class StdoutReporter(private val timer: Timer) : Reporter { if(poolSummary.failed.isNotEmpty()){ cliReportBuilder.appendLine("\tFailed tests:") poolSummary.failed + .map { it.toTestName() } .toSortedSet() .forEach { testName -> cliReportBuilder.appendLine("\t\t$testName") } } @@ -28,7 +66,7 @@ class StdoutReporter(private val timer: Timer) : Reporter { if(poolSummary.rawFailed.isNotEmpty()){ cliReportBuilder.appendLine("\tFailed tests:") poolSummary.rawFailed - .groupBy { it } + .groupBy { it.toTestName() } .toSortedMap() .mapValues { it.value.size } .forEach { (testName, count) -> @@ -38,13 +76,13 @@ class StdoutReporter(private val timer: Timer) : Reporter { if(poolSummary.rawIncomplete.isNotEmpty()){ cliReportBuilder.appendLine("\tIncomplete tests:") - poolSummary.rawIncomplete - .groupBy { it } - .toSortedMap() - .mapValues { it.value.size } - .forEach { (testName, count) -> - cliReportBuilder.appendLine("\t\t$testName incomplete $count time(s)") - } + poolSummary.rawIncomplete + .groupBy { it.toTestName() } + .toSortedMap() + .mapValues { it.value.size } + .forEach { (testName, count) -> + cliReportBuilder.appendLine("\t\t$testName incomplete $count time(s)") + } } } cliReportBuilder.appendLine("Total time: ${formatDuration(timer.elapsedTimeMillis)}") diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/teamcity/TeamCityProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/teamcity/TeamCityProgressReporter.kt new file mode 100644 index 000000000..ea582e220 --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/teamcity/TeamCityProgressReporter.kt @@ -0,0 +1,15 @@ +package com.malinskiy.marathon.report.teamcity + +import com.malinskiy.marathon.report.ProgressReporter + +class TeamCityProgressReporter : ProgressReporter { + private fun setBuildMessage(message: String) = println("##teamcity[buildStatus text='$message']") + + private fun setKeyValue(key: String, value: String) = println("##teamcity[buildStatisticValue key='$key' value='$value']") + + fun setBuildProgress(int: Int) = setBuildMessage("Marathon run: $int %") + + override fun testStarted(progress: Float, poolId: String, deviceSerial: String, testName: String) { + setBuildProgress(progress.toInt()) + } +} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/test/TestJsonReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/test/TestJsonReporter.kt index 05c9981db..8b820079b 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/test/TestJsonReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/test/TestJsonReporter.kt @@ -2,15 +2,15 @@ package com.malinskiy.marathon.report.test import com.google.gson.Gson import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.io.FileManager import com.malinskiy.marathon.io.FileType -import com.malinskiy.marathon.report.Reporter internal class TestJsonReporter( private val fileManager: FileManager, private val gson: Gson -) : Reporter { - override fun generate(executionReport: ExecutionReport) { +) : ProgressReporter { + override fun end(executionReport: ExecutionReport) { for (testEvent in executionReport.testEvents.filter { it.final }) { val file = fileManager.createFile(FileType.TEST_RESULT, testEvent.poolId, testEvent.device, testEvent.testResult.test) file.writeText(gson.toJson(testEvent.testResult)) diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineReporter.kt index 4e2f9f7ee..60e1889f6 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineReporter.kt @@ -2,8 +2,8 @@ package com.malinskiy.marathon.report.timeline import com.google.gson.Gson import com.malinskiy.marathon.analytics.internal.sub.ExecutionReport +import com.malinskiy.marathon.report.ProgressReporter import com.malinskiy.marathon.log.MarathonLogging -import com.malinskiy.marathon.report.Reporter import java.io.File import java.io.InputStream @@ -12,9 +12,9 @@ class TimelineReporter( private val provider: TimelineSummaryProvider, private val gson: Gson, private val rootOutput: File -) : Reporter { +) : ProgressReporter { - override fun generate(executionReport: ExecutionReport) { + override fun end(executionReport: ExecutionReport) { val htmlDir = File(rootOutput, "/html") htmlDir.mkdirs() val timelineDir = File(htmlDir, "/timeline") diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineSummaryProvider.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineSummaryProvider.kt index 31d6309ca..ba48b57a1 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineSummaryProvider.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/timeline/TimelineSummaryProvider.kt @@ -100,7 +100,8 @@ class TimelineSummaryProvider { deviceDisconnectedEvents = deviceDisconnectedEvents[key] ?: emptyList(), devicePreparingEvents = devicePreparingEvent[key] ?: emptyList(), deviceProviderPreparingEvent = deviceProviderPreparingEvents[key] ?: emptyList(), - testEvents = testEvents[key] ?: emptyList() + testEvents = testEvents[key] ?: emptyList(), + defaultStart = executionReport.defaultStart, ) }.toMap() diff --git a/core/src/main/proto/progress_api_v1.proto b/core/src/main/proto/progress_api_v1.proto new file mode 100644 index 000000000..a9c1d8095 --- /dev/null +++ b/core/src/main/proto/progress_api_v1.proto @@ -0,0 +1,90 @@ +syntax = "proto3"; +package io.marathonlabs.testing.progress.v1; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_package = "io.marathonlabs.testing.progress"; + +service ProgressService { + rpc Start(StartRequest) returns (StartResponse); + rpc Track(stream TrackRequest) returns (TrackResponse); + rpc End(EndRequest) returns (EndResponse); +} + +message StartRequest { + string run_id = 1; + repeated Test tests = 2; +} + +message StartResponse {} +message TrackResponse {} +message EndResponse {} + +message TrackRequest { + string run_id = 1; + repeated TestUpdate test_updates = 2; + Stats raw = 3; + Stats final = 4; +} + +message TestUpdate { + Event event = 1; + Test test = 2; + DeviceInfo device_info = 3; +} + +enum Event { + EVENT_UNSPECIFIED = 0; + EVENT_STARTED = 1; + EVENT_FAILED = 2; + EVENT_PASSED = 3; + EVENT_IGNORED = 4; + EVENT_INCOMPLETE = 5; +} + +message Stats { + uint32 passed = 1; + uint32 failed = 2; + uint32 ignored = 3; + uint32 incomplete = 4; +} + +message EndRequest { + string run_id = 1; + bool success = 2; + google.protobuf.Duration flakiness = 3; + google.protobuf.Duration cost = 4; + google.protobuf.Duration duration = 5; + Stats raw = 6; + Stats final = 7; + repeated TestStat final_failed = 8; + repeated TestStat raw_failed = 9; + repeated TestStat raw_incomplete = 10; + repeated DeviceBill device_bill = 11; +} + +message TestStat { + Test test = 1; + uint32 count = 2; +} + +message DeviceBill { + DeviceInfo device_info = 1; + google.protobuf.Timestamp start = 2; + google.protobuf.Timestamp end = 3; + google.protobuf.Duration duration = 4; +} + +message DeviceInfo { + string serial = 1; + string operating_system = 2; +} + +message Test { + string pkg = 1; + string clazz = 2; + string method = 3; + repeated string meta_properties = 4; +} diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt index 2f04ae12c..190eceec3 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt @@ -11,6 +11,7 @@ import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestShard import com.malinskiy.marathon.execution.TestStatus import com.malinskiy.marathon.generateTest +import com.malinskiy.marathon.report.NoopProgressReporter import com.malinskiy.marathon.report.getDevice import org.amshove.kluent.shouldBe import org.mockito.kotlin.mock @@ -55,7 +56,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -72,7 +74,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -89,7 +92,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -106,7 +110,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.aggregateResult().shouldBeEqualTo(false) } @@ -117,7 +122,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.retryTest(test) @@ -141,7 +147,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(testParameterized)), anySuccessConfig, - track + track, + NoopProgressReporter, ) val test0 = generateTest( pkg = "com.malinskiy.marathon", @@ -174,7 +181,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), successFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -191,7 +199,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), successFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -204,7 +213,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), successFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -217,7 +227,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -238,7 +249,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -260,7 +272,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -281,7 +294,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -302,7 +316,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), successFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -323,7 +338,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.retryTest(test) @@ -351,7 +367,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), successFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.INCOMPLETE, 0, 1), false) @@ -370,7 +387,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -391,7 +409,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -414,7 +433,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -437,7 +457,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), failFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -454,7 +475,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -471,7 +493,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), failFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -486,7 +509,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -503,7 +527,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -516,7 +541,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -529,7 +555,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -542,7 +569,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), failFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -555,7 +583,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test)), failFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.retryTest(test) @@ -573,7 +602,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test, test)), failFastConfig, - track + track, + NoopProgressReporter, ) //Just removes one attempt. Now we have 3 tests to execute with all_success mode reporter.removeTest(test, 1) @@ -601,7 +631,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1)) @@ -622,7 +653,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -643,7 +675,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), failFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -664,7 +697,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), allSuccessConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -685,7 +719,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test, test, test)), failFastConfig, - track + track, + NoopProgressReporter, ) reporter.testStarted(device, test) reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1)) @@ -712,7 +747,8 @@ class PoolProgressAccumulatorTest { poolId, TestShard(listOf(test1, test2, test3)), anySuccessConfig, - track + track, + NoopProgressReporter, ) reporter.progress().shouldBeEqualTo(.0f) diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt index 90dbe75b9..7f4557e55 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt @@ -13,6 +13,7 @@ import com.malinskiy.marathon.execution.TestBatchResults import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestShard import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.report.NoopProgressReporter import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator import org.mockito.kotlin.KArgumentCaptor import org.mockito.kotlin.any @@ -264,7 +265,7 @@ private fun createQueueActor( devicePoolId, mock(), null, - PoolProgressAccumulator(devicePoolId, testShard, configuration, track), + PoolProgressAccumulator(devicePoolId, testShard, configuration, track, NoopProgressReporter), job, Dispatchers.Unconfined ) diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt index e662d1d00..49d7ba950 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt @@ -11,6 +11,7 @@ import com.malinskiy.marathon.device.DevicePoolId import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestShard import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.report.NoopProgressReporter import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator import com.malinskiy.marathon.generateTest import org.mockito.kotlin.inOrder @@ -46,14 +47,16 @@ class TestResultReporterAllSuccessTest { poolId, TestShard(listOf(test, test, test)), defaultConfig, - track + track, + NoopProgressReporter, ) private fun filterStrict() = PoolProgressAccumulator( poolId, TestShard(listOf(test, test, test)), strictConfig, - track + track, + NoopProgressReporter, ) private val deviceInfo = createDeviceInfo() diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/NoRetryStrategyTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/NoRetryStrategyTest.kt index 13279ea63..30a55a2c2 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/NoRetryStrategyTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/NoRetryStrategyTest.kt @@ -7,6 +7,7 @@ import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration import com.malinskiy.marathon.config.vendor.VendorConfiguration import com.malinskiy.marathon.device.DevicePoolId import com.malinskiy.marathon.execution.TestShard +import com.malinskiy.marathon.report.NoopProgressReporter import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator import com.malinskiy.marathon.generateTestResults import com.malinskiy.marathon.generateTests @@ -39,7 +40,8 @@ class NoRetryStrategyTest { devicePoolId, TestShard(tests), anySuccessConfig, - track + track, + NoopProgressReporter, ) val result = strategy.process(devicePoolId, testResults, testShard, accumulator) diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/fixedquota/FixedQuotaRetryStrategyTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/fixedquota/FixedQuotaRetryStrategyTest.kt index 74413281c..bae4a9573 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/fixedquota/FixedQuotaRetryStrategyTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/fixedquota/FixedQuotaRetryStrategyTest.kt @@ -7,8 +7,8 @@ import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration import com.malinskiy.marathon.config.vendor.VendorConfiguration import com.malinskiy.marathon.device.DevicePoolId -import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestShard +import com.malinskiy.marathon.report.NoopProgressReporter import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator import com.malinskiy.marathon.extension.toRetryStrategy import com.malinskiy.marathon.generateTestResults @@ -42,7 +42,8 @@ class FixedQuotaRetryStrategyTest { poolId, TestShard(tests), anySuccessConfig, - track + track, + NoopProgressReporter, ) strategy.process(poolId, testResults, TestShard(tests), accumulator).size shouldBe 1 @@ -58,7 +59,8 @@ class FixedQuotaRetryStrategyTest { poolId, TestShard(tests), anySuccessConfig, - track + track, + NoopProgressReporter, ) strategy.process(poolId, testResults, TestShard(tests), accumulator).size shouldBe 10 } @@ -73,7 +75,8 @@ class FixedQuotaRetryStrategyTest { poolId, TestShard(tests), anySuccessConfig, - track + track, + NoopProgressReporter, ) strategy.process(poolId, testResults, TestShard(tests), accumulator).size shouldBe 50 @@ -89,7 +92,8 @@ class FixedQuotaRetryStrategyTest { poolId, TestShard(tests), anySuccessConfig, - track + track, + NoopProgressReporter, ) strategy.process( @@ -110,7 +114,8 @@ class FixedQuotaRetryStrategyTest { poolId, TestShard(tests), anySuccessConfig, - track + track, + NoopProgressReporter, ) val deviceInfo = getDevice() diff --git a/core/src/test/kotlin/com/malinskiy/marathon/report/ExecutionReportTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/report/ExecutionReportTest.kt index 6b00105a9..52af2d771 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/report/ExecutionReportTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/report/ExecutionReportTest.kt @@ -50,8 +50,9 @@ class ExecutionReportTest { testEvents = listOf( createTestEvent(device, "test1", TestStatus.INCOMPLETE), createTestEvent(device, "test2", TestStatus.PASSED), - createTestEvent(device, "test3", TestStatus.FAILURE) - ) + createTestEvent(device, "test3", TestStatus.FAILURE), + ), + defaultStart = Instant.now(), ) } @@ -79,7 +80,8 @@ class ExecutionReportTest { createTestEvent(device, "test3", TestStatus.FAILURE, false), createTestEvent(device, "test3", TestStatus.FAILURE, false), createTestEvent(device, "test3", TestStatus.FAILURE, true) - ) + ), + defaultStart = Instant.now(), ) } diff --git a/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt index 503955eaf..ff57e2663 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt @@ -42,11 +42,12 @@ class JUnitReporterTest { createTestEvent(device, "test1", TestStatus.PASSED), createTestEvent(device, "test2", TestStatus.PASSED), createTestEvent(device, "test3", TestStatus.PASSED) - ) + ), + defaultStart = Instant.now(), ) val configuration = getConfiguration() val junitReport = JUnitReporter(configuration.outputDir) - junitReport.generate(report) + junitReport.end(report) File(configuration.outputDir.absolutePath + "/tests/myPool/marathon_junit_report.xml") .shouldBeEqualToAsXML(File(javaClass.getResource("/output/tests/myPool/xxyyzz/marathon_junit_report_passed_tests.xml").file)) @@ -69,12 +70,13 @@ class JUnitReporterTest { createTestEvent(device, "test2", TestStatus.INCOMPLETE), createTestEvent(device, "test2", TestStatus.FAILURE), createTestEvent(device, "test3", TestStatus.FAILURE) - ) + ), + defaultStart = Instant.now(), ) val configuration = getConfiguration() println(configuration.outputDir) val junitReport = JUnitReporter(configuration.outputDir) - junitReport.generate(report) + junitReport.end(report) File(configuration.outputDir.absolutePath + "/tests/myPool/marathon_junit_report.xml") .shouldBeEqualToAsXML(File(javaClass.getResource("/output/tests/myPool/xxyyzz/marathon_junit_report_failed_tests.xml").file)) @@ -101,12 +103,13 @@ class JUnitReporterTest { createTestEvent(device, "test2", TestStatus.IGNORED), createTestEvent(device, "test2", TestStatus.INCOMPLETE), createTestEvent(device, "test3", TestStatus.ASSUMPTION_FAILURE, stackTrace) - ) + ), + defaultStart = Instant.now(), ) val configuration = getConfiguration() println(configuration.outputDir) val junitReport = JUnitReporter(configuration.outputDir) - junitReport.generate(report) + junitReport.end(report) File(configuration.outputDir.absolutePath + "/tests/myPool/marathon_junit_report.xml") .shouldBeEqualToAsXML(File(javaClass.getResource("/output/tests/myPool/xxyyzz/marathon_junit_report_failed_tests_with_stacktrace.xml").file)) @@ -132,12 +135,13 @@ class JUnitReporterTest { testEvents = listOf( createTestEvent(device, "test1", TestStatus.FAILURE, stackTrace, false), createTestEvent(device, "test1", TestStatus.PASSED, final = true) - ) + ), + defaultStart = Instant.now(), ) val configuration = getConfiguration() println(configuration.outputDir) val junitReport = JUnitReporter(configuration.outputDir) - junitReport.generate(report) + junitReport.end(report) File(configuration.outputDir.absolutePath + "/tests/myPool/marathon_junit_report.xml") .shouldBeEqualToAsXML(File(javaClass.getResource("/output/tests/myPool/xxyyzz/marathon_junit_report_failed_to_passed_test.xml").file)) @@ -163,12 +167,13 @@ class JUnitReporterTest { testEvents = listOf( createTestEvent(device, "test1", TestStatus.PASSED, final = false), createTestEvent(device, "test1", TestStatus.FAILURE, stackTrace) - ) + ), + defaultStart = Instant.now(), ) val configuration = getConfiguration() println(configuration.outputDir) val junitReport = JUnitReporter(configuration.outputDir) - junitReport.generate(report) + junitReport.end(report) File(configuration.outputDir.absolutePath + "/tests/myPool/marathon_junit_report.xml") .shouldBeEqualToAsXML(File(javaClass.getResource("/output/tests/myPool/xxyyzz/marathon_junit_report_passed_to_failed_test.xml").file)) diff --git a/core/src/test/kotlin/com/malinskiy/marathon/scenario/DeviceFilteringScenarioTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/scenario/DeviceFilteringScenarioTest.kt index e23a14b7f..50ba28857 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/scenario/DeviceFilteringScenarioTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/scenario/DeviceFilteringScenarioTest.kt @@ -2,6 +2,8 @@ package com.malinskiy.marathon.scenario +import com.malinskiy.marathon.config.ProgressConfiguration +import com.malinskiy.marathon.config.ProgressReporterConfiguration import com.malinskiy.marathon.device.DeviceProvider import com.malinskiy.marathon.execution.TestStatus import com.malinskiy.marathon.test.StubDevice @@ -45,6 +47,7 @@ class DeviceFilteringScenarioTest { excludeSerialRegexes = listOf("""emulator-5002""".toRegex()) includeSerialRegexes = emptyList() + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext @@ -92,6 +95,7 @@ class DeviceFilteringScenarioTest { excludeSerialRegexes = emptyList() includeSerialRegexes = listOf("""emulator-5002""".toRegex()) + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext @@ -139,6 +143,7 @@ class DeviceFilteringScenarioTest { excludeSerialRegexes = listOf("""emulator-5002""".toRegex()) includeSerialRegexes = listOf("""emulator-500[2,4]""".toRegex()) + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext diff --git a/core/src/test/kotlin/com/malinskiy/marathon/scenario/DisconnectingScenariosTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/scenario/DisconnectingScenariosTest.kt index 0fdb236f9..de8318887 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/scenario/DisconnectingScenariosTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/scenario/DisconnectingScenariosTest.kt @@ -2,6 +2,8 @@ package com.malinskiy.marathon.scenario +import com.malinskiy.marathon.config.ProgressConfiguration +import com.malinskiy.marathon.config.ProgressReporterConfiguration import com.malinskiy.marathon.device.DeviceProvider import com.malinskiy.marathon.execution.TestStatus import com.malinskiy.marathon.test.StubDevice @@ -49,6 +51,7 @@ class DisconnectingScenariosTest { listOf(test1, test2) } + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext devices { diff --git a/core/src/test/kotlin/com/malinskiy/marathon/scenario/SuccessScenariosTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/scenario/SuccessScenariosTest.kt index a8aea6018..9ec39d3ea 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/scenario/SuccessScenariosTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/scenario/SuccessScenariosTest.kt @@ -2,6 +2,8 @@ package com.malinskiy.marathon.scenario +import com.malinskiy.marathon.config.ProgressConfiguration +import com.malinskiy.marathon.config.ProgressReporterConfiguration import com.malinskiy.marathon.device.DeviceProvider import com.malinskiy.marathon.execution.TestStatus import com.malinskiy.marathon.test.MetaProperty @@ -52,6 +54,7 @@ class SuccessScenariosTest { listOf(test) } + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext devices { diff --git a/core/src/test/kotlin/com/malinskiy/marathon/scenario/UncompletedScenariosTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/scenario/UncompletedScenariosTest.kt index d987586ab..525458ebe 100644 --- a/core/src/test/kotlin/com/malinskiy/marathon/scenario/UncompletedScenariosTest.kt +++ b/core/src/test/kotlin/com/malinskiy/marathon/scenario/UncompletedScenariosTest.kt @@ -2,6 +2,8 @@ package com.malinskiy.marathon.scenario +import com.malinskiy.marathon.config.ProgressConfiguration +import com.malinskiy.marathon.config.ProgressReporterConfiguration import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration import com.malinskiy.marathon.device.DeviceProvider import com.malinskiy.marathon.execution.TestStatus @@ -48,6 +50,7 @@ class UncompletedScenariosTest { uncompletedTestRetryQuota = 100 + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext devices { @@ -90,6 +93,7 @@ class UncompletedScenariosTest { uncompletedTestRetryQuota = 100 + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext devices { @@ -133,6 +137,7 @@ class UncompletedScenariosTest { uncompletedTestRetryQuota = 3 + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext devices { @@ -174,6 +179,7 @@ class UncompletedScenariosTest { uncompletedTestRetryQuota = 3 retryStrategy = RetryStrategyConfiguration.FixedQuotaRetryStrategyConfiguration(10, 3) + progressConfiguration = ProgressConfiguration.Custom(listOf(ProgressReporterConfiguration.Raw)) deviceProvider.context = coroutineContext devices { diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt index 5b0aa6072..281e3c06b 100644 --- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt +++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt @@ -6,7 +6,6 @@ import com.malinskiy.marathon.device.toDeviceInfo import com.malinskiy.marathon.execution.TestBatchResults import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestStatus -import com.malinskiy.marathon.execution.progress.ProgressReporter import com.malinskiy.marathon.execution.result.TemporalTestResult import com.malinskiy.marathon.report.attachment.AttachmentCollector import com.malinskiy.marathon.log.MarathonLogging diff --git a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/ConfigurationFactory.kt b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/ConfigurationFactory.kt index 1aa210b51..a5e39c351 100644 --- a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/ConfigurationFactory.kt +++ b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/ConfigurationFactory.kt @@ -3,6 +3,7 @@ package com.malinskiy.marathon.test.factory import com.malinskiy.marathon.config.AnalyticsConfiguration import com.malinskiy.marathon.config.Configuration import com.malinskiy.marathon.config.FilteringConfiguration +import com.malinskiy.marathon.config.ProgressConfiguration import com.malinskiy.marathon.config.ScreenRecordingPolicy import com.malinskiy.marathon.config.strategy.BatchingStrategyConfiguration import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration @@ -43,6 +44,7 @@ class ConfigurationFactory(val testParser: LocalTestParser, val deviceProvider: var analyticsTracking = false var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_ANY var deviceInitializationTimeoutMillis = 60_000L + var progressConfiguration = ProgressConfiguration.Custom(emptyList()) suspend fun tests(block: () -> List) { whenever(testParser.extract()).thenReturn(block.invoke()) @@ -76,5 +78,6 @@ class ConfigurationFactory(val testParser: LocalTestParser, val deviceProvider: screenRecordingPolicy = this@ConfigurationFactory.screenRecordingPolicy analyticsTracking = this@ConfigurationFactory.analyticsTracking deviceInitializationTimeoutMillis = this@ConfigurationFactory.deviceInitializationTimeoutMillis + progressConfiguration = this@ConfigurationFactory.progressConfiguration }.build() } diff --git a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt index cd39b7689..0c1524c81 100644 --- a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt +++ b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt @@ -11,6 +11,9 @@ import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand import com.malinskiy.marathon.io.FileManager import com.malinskiy.marathon.json.FileSerializer import com.malinskiy.marathon.log.MarathonLogConfigurator +import com.malinskiy.marathon.report.NoopProgressReporter +import com.malinskiy.marathon.report.ProgressReporter +import com.malinskiy.marathon.report.ProgressReporterFactory import com.malinskiy.marathon.test.Mocks import com.malinskiy.marathon.test.StubDeviceProvider import com.malinskiy.marathon.time.SystemTimer @@ -54,7 +57,11 @@ class MarathonFactory { val configuration = get() MarathonTestParseCommand(configuration.outputDir) } - single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) } + single { + val factory = ProgressReporterFactory(get(), get(), get(), get(), get()) + factory.create() + } + single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) } } val configurationModule = module {