fix(android): implement test run failing to detect misconfiguration o…

…f am instrument listener argument
MarathonLabs · Jan 22, 2024 · c3c0976 · c3c0976
1 parent 68d5981
commit c3c0976
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 10 deletions.
diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt
@@ -6,7 +6,7 @@ object Versions {
     val coroutinesTest = coroutines
 
     val androidCommon = "31.1.3"
-    val adam = "0.5.2"
+    val adam = "0.5.3"
     val dexTestParser = "2.3.4"
     val kotlinLogging = "3.0.5"
     val logbackClassic = "1.4.11"

diff --git a/...dor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AmInstrumentTestParser.kt b/...dor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AmInstrumentTestParser.kt
@@ -8,6 +8,7 @@ import com.malinskiy.adam.request.testrunner.TestFailed
 import com.malinskiy.adam.request.testrunner.TestIgnored
 import com.malinskiy.adam.request.testrunner.TestRunEnded
 import com.malinskiy.adam.request.testrunner.TestRunFailed
+import com.malinskiy.adam.request.testrunner.TestRunFailing
 import com.malinskiy.adam.request.testrunner.TestRunStartedEvent
 import com.malinskiy.adam.request.testrunner.TestRunStopped
 import com.malinskiy.adam.request.testrunner.TestRunnerRequest
@@ -52,10 +53,11 @@ class AmInstrumentTestParser(
                 return@withRetry parseTests(device, configuration, vendorConfiguration, testBundles, blockListenerArgumentOverride)
             } catch (e: CancellationException) {
                 throw e
-            } catch (e: PossibleListenerIssueException) {
-                logger.warn { "The previous parse operation failed. The most possible reason is " +
-                    "a developer missed this step https://docs.marathonlabs.io/android/configure#test-parser. " +
-                    "The next attempt will be done without overridden testrun listener." }
+            } catch (e: TestAnnotationProducerNotFoundException) {
+                logger.warn {
+                    "Previous parsing attempt failed due to test parser misconfiguration: test annotation producer was not found. See https://docs.marathonlabs.io/runner/android/configure#test-parser. " +
+                        "Next parsing attempt will remove overridden test run listener."
+                }
                 blockListenerArgumentOverride = true
                 throw e
             } catch (throwable: Throwable) {
@@ -84,6 +86,7 @@ class AmInstrumentTestParser(
                         .filterNot { it.key == LISTENER_ARGUMENT && it.value == TEST_ANNOTATION_PRODUCER }
                     else testParserConfiguration.instrumentationArgs
                 }
+
                 else -> emptyMap()
             }
 
@@ -127,12 +130,17 @@ class AmInstrumentTestParser(
                                 testBundleIdentifier.put(test, androidTestBundle)
                             }
 
-                            is TestRunFailed -> {
-                                if (overrides.containsKey(LISTENER_ARGUMENT) && overrides[LISTENER_ARGUMENT] == TEST_ANNOTATION_PRODUCER)
-                                    throw PossibleListenerIssueException()
+                            is TestRunFailing -> {
+                                // Error message is stable, see https://github.com/android/android-test/blame/1ae53b93e02cc363311f6564a35edeea1b075103/runner/android_junit_runner/java/androidx/test/internal/runner/RunnerArgs.java#L624
+                                if (event.error.contains("Could not find extra class $TEST_ANNOTATION_PRODUCER")) {
+                                    throw TestAnnotationProducerNotFoundException()
+                                }
                             }
+
+                            is TestRunFailed -> Unit
                             is TestRunStopped -> Unit
                             is TestRunEnded -> Unit
+
                         }
                     }
                 }
@@ -141,7 +149,7 @@ class AmInstrumentTestParser(
             if (!observedAnnotations) {
                 logger.warn {
                     "Bundle ${bundle.id} did not report any test annotations. If you need test annotations retrieval, remote test parser requires additional setup " +
-                        "see https://docs.marathonlabs.io/android/configure#test-parser"
+                        "see https://docs.marathonlabs.io/runner/android/configure#test-parser"
                 }
             }
 
@@ -150,4 +158,4 @@ class AmInstrumentTestParser(
     }
 }
 
-private class PossibleListenerIssueException : RuntimeException()
+private class TestAnnotationProducerNotFoundException : RuntimeException()
diff --git a/...or-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AndroidDeviceTestRunner.kt b/...or-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AndroidDeviceTestRunner.kt
@@ -8,6 +8,7 @@ import com.malinskiy.adam.request.testrunner.TestFailed
 import com.malinskiy.adam.request.testrunner.TestIgnored
 import com.malinskiy.adam.request.testrunner.TestRunEnded
 import com.malinskiy.adam.request.testrunner.TestRunFailed
+import com.malinskiy.adam.request.testrunner.TestRunFailing
 import com.malinskiy.adam.request.testrunner.TestRunStartedEvent
 import com.malinskiy.adam.request.testrunner.TestRunStopped
 import com.malinskiy.adam.request.testrunner.TestRunnerRequest
@@ -116,6 +117,7 @@ class AndroidDeviceTestRunner(private val device: AdamAndroidDevice, private val
                 is TestRunFailed -> listener.testRunFailed(event.error)
                 is TestRunStopped -> listener.testRunStopped(event.elapsedTimeMillis)
                 is TestRunEnded -> listener.testRunEnded(event.elapsedTimeMillis, event.metrics)
+                is TestRunFailing -> listener.testRunFailing(event.error, event.stackTrace)
             }
         }
     }

diff --git a/...c/main/kotlin/com/malinskiy/marathon/android/executor/listeners/AndroidTestRunListener.kt b/...c/main/kotlin/com/malinskiy/marathon/android/executor/listeners/AndroidTestRunListener.kt
@@ -18,6 +18,8 @@ interface AndroidTestRunListener {
 
     suspend fun testEnded(test: TestIdentifier, testMetrics: Map<String, String>) {}
 
+    suspend fun testRunFailing(error: String, stackTrace: String) {}
+
     suspend fun testRunFailed(errorMessage: String) {}
 
     suspend fun testRunStopped(elapsedTime: Long) {}

diff --git a/...src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/DebugTestRunListener.kt b/...src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/DebugTestRunListener.kt
@@ -36,6 +36,10 @@ class DebugTestRunListener(private val device: AndroidDevice) : AndroidTestRunLi
         logger.info { "testIgnored ${device.serialNumber} test = $test" }
     }
 
+    override suspend fun testRunFailing(error: String, stackTrace: String) {
+        logger.info { "testRunFailing ${device.serialNumber} errorMessage = $error trace = $stackTrace" }
+    }
+
     override suspend fun testRunFailed(errorMessage: String) {
         logger.info { "testRunFailed ${device.serialNumber} errorMessage = $errorMessage" }
     }

diff --git a/.../src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/NoOpTestRunListener.kt b/.../src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/NoOpTestRunListener.kt
@@ -15,6 +15,8 @@ open class NoOpTestRunListener : AndroidTestRunListener {
 
     override suspend fun testEnded(test: TestIdentifier, testMetrics: Map<String, String>) {}
 
+    override suspend fun testRunFailing(error: String, stackTrace: String) {}
+
     override suspend fun testRunFailed(errorMessage: String) {}
 
     override suspend fun testRunStopped(elapsedTime: Long) {}