feat: Adds Insights Notification

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+## 1.39.10
+
+- Update install script to tidy up old installation binaries
+
+## 1.39.9
+
+- Revert: Error in showing keyboard during input and erase commands on iOS
+- Fix: applesimutils affecting granting location permission
+- Fix: Setting host and port from the optional arguments
+- Feature: New `maestro login` command for logging in Robin.
+- Feature: Improved `maestro record` video to scroll and follow the currently executing commands
+- Fix: Enable running Maestro on Windows without WSL
+- Feature: Add console.log messages directly to the maestro log file.
+
 ## 1.39.8
 
 - Fix: Debug message not showing up when we execute commands on maestro cli anymore

README.md

@@ -2,7 +2,7 @@
 
 # Maestro
 
-Maestro is the easiest way to automate UI testing for your mobile app.
+Maestro is the easiest way to UI testing for your mobile or web app.
 
 <img src="https://user-images.githubusercontent.com/847683/187275009-ddbdf963-ce1d-4e07-ac08-b10f145e8894.gif" />
 

gradle.properties

@@ -2,7 +2,7 @@ android.useAndroidX=true
 android.enableJetifier=true
 kotlin.code.style=official
 GROUP=dev.mobile
-VERSION_NAME=1.39.8
+VERSION_NAME=1.39.10
 POM_DESCRIPTION=Maestro is a server-driven platform-agnostic library that allows to drive tests for both iOS and Android using the same implementation through an intuitive API.
 POM_URL=https://github.com/mobile-dev-inc/maestro
 POM_SCM_URL=https://github.com/mobile-dev-inc/maestro

gradle/libs.versions.toml

@@ -17,7 +17,7 @@ axml = "2.1.2"
 commons-codec = "1.17.0"
 commons-lang3 = "3.13.0" # 3.14.0 causes weird crashes during dexing
 commons-io = "2.16.1"
-dadb = "1.2.7"
+dadb = "1.2.9"
 detekt = "1.19.0"
 googleFindbugs = "3.0.2"
 googleGson = "2.11.0"

maestro-ai/src/main/java/maestro/ai/Prediction.kt

@@ -270,4 +270,4 @@ object Prediction {
         return response.text ?: ""
     }
 
-}
+}

maestro-cli/build.gradle.kts

@@ -31,6 +31,10 @@ tasks.named<JavaExec>("run") {
     workingDir = rootDir
 }
 
+tasks.named<CreateStartScripts>("startScripts") {
+    classpath = files("$buildDir/libs/*")
+}
+
 dependencies {
     implementation(project(path = ":maestro-utils"))
     annotationProcessor(libs.picocli.codegen)

maestro-cli/gradle.properties

@@ -1 +1 @@
-CLI_VERSION=1.39.8
+CLI_VERSION=1.39.10

maestro-cli/src/main/java/maestro/cli/App.kt

@@ -33,6 +33,7 @@ import maestro.cli.command.StartDeviceCommand
 import maestro.cli.command.StudioCommand
 import maestro.cli.command.TestCommand
 import maestro.cli.command.UploadCommand
+import maestro.cli.insights.TestAnalysisManager
 import maestro.cli.update.Updates
 import maestro.cli.util.ChangeLogUtils
 import maestro.cli.util.ErrorReporter
@@ -143,6 +144,7 @@ fun main(args: Array<String>) {
         .execute(*args)
 
     DebugLogStore.finalizeRun()
+    TestAnalysisManager.maybeNotify()
 
     val newVersion = Updates.checkForUpdates()
     if (newVersion != null) {

maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt

@@ -1,6 +1,8 @@
 package maestro.cli.api
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.github.michaelbull.result.Err
@@ -9,6 +11,7 @@ import com.github.michaelbull.result.Result
 import maestro.cli.CliError
 import maestro.cli.analytics.Analytics
 import maestro.cli.analytics.AnalyticsReport
+import maestro.cli.insights.FlowFiles
 import maestro.cli.model.FlowStatus
 import maestro.cli.runner.resultview.AnsiResultView
 import maestro.cli.util.CiUtils
@@ -34,7 +37,6 @@ import java.nio.file.Path
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.exists
 import kotlin.time.Duration.Companion.minutes
-import okhttp3.MediaType
 
 class ApiClient(
     private val baseUrl: String,
@@ -185,7 +187,11 @@ class ApiClient(
         val baseUrl = "https://maestro-record.ngrok.io"
         val body = MultipartBody.Builder()
             .setType(MultipartBody.FORM)
-            .addFormDataPart("screenRecording", screenRecording.name, screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener))
+            .addFormDataPart(
+                "screenRecording",
+                screenRecording.name,
+                screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener)
+            )
             .addFormDataPart("frames", JSON.writeValueAsString(frames))
             .build()
         val request = Request.Builder()
@@ -267,15 +273,27 @@ class ApiClient(
 
         val bodyBuilder = MultipartBody.Builder()
             .setType(MultipartBody.FORM)
-            .addFormDataPart("workspace", "workspace.zip", workspaceZip.toFile().asRequestBody("application/zip".toMediaType()))
+            .addFormDataPart(
+                "workspace",
+                "workspace.zip",
+                workspaceZip.toFile().asRequestBody("application/zip".toMediaType())
+            )
             .addFormDataPart("request", JSON.writeValueAsString(requestPart))
 
         if (appFile != null) {
-            bodyBuilder.addFormDataPart("app_binary", "app.zip", appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener))
+            bodyBuilder.addFormDataPart(
+                "app_binary",
+                "app.zip",
+                appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener)
+            )
         }
 
         if (mappingFile != null) {
-            bodyBuilder.addFormDataPart("mapping", "mapping.txt", mappingFile.toFile().asRequestBody("text/plain".toMediaType()))
+            bodyBuilder.addFormDataPart(
+                "mapping",
+                "mapping.txt",
+                mappingFile.toFile().asRequestBody("text/plain".toMediaType())
+            )
         }
 
         val body = bodyBuilder.build()
@@ -286,7 +304,7 @@ class ApiClient(
                 throw CliError(message)
             }
 
-            PrintUtils.message("$message, retrying (${completedRetries+1}/$maxRetryCount)...")
+            PrintUtils.message("$message, retrying (${completedRetries + 1}/$maxRetryCount)...")
             Thread.sleep(BASE_RETRY_DELAY_MS + (2000 * completedRetries))
 
             return upload(
@@ -440,6 +458,58 @@ class ApiClient(
         }
     }
 
+    fun analyze(
+        authToken: String,
+        flowFiles: List<FlowFiles>,
+    ): AnalyzeResponse {
+        if (flowFiles.isEmpty()) throw CliError("Missing flow files to analyze")
+
+        val screenshots = mutableListOf<Pair<String, ByteArray>>()
+        val logs = mutableListOf<Pair<String, ByteArray>>()
+
+        flowFiles.forEach { flowFile ->
+            flowFile.imageFiles.forEach { (imageData, path) ->
+                val imageName = path.fileName.toString()
+                screenshots.add(Pair(imageName, imageData))
+            }
+
+            flowFile.textFiles.forEach { (textData, path) ->
+                val textName = path.fileName.toString()
+                logs.add(Pair(textName, textData))
+            }
+        }
+
+        val requestBody = mapOf(
+            "screenshots" to screenshots,
+            "logs" to logs
+        )
+
+        val mediaType = "application/json; charset=utf-8".toMediaType()
+        val body = JSON.writeValueAsString(requestBody).toRequestBody(mediaType)
+
+        val url = "$baseUrl/v2/analyze"
+
+        val request = Request.Builder()
+            .header("Authorization", "Bearer $authToken")
+            .url(url)
+            .post(body)
+            .build()
+
+        val response = client.newCall(request).execute()
+
+        response.use {
+            if (!response.isSuccessful) {
+                val errorMessage = response.body?.string().takeIf { it?.isNotEmpty() == true } ?: "Unknown"
+                throw CliError("Analyze request failed (${response.code}): $errorMessage")
+            }
+
+            val parsed = JSON.readValue(response.body?.bytes(), AnalyzeResponse::class.java)
+
+            return parsed;
+        }
+    }
+
+
     data class ApiException(
         val statusCode: Int?,
     ) : Exception("Request failed. Status code: $statusCode")
@@ -459,7 +529,7 @@ data class RobinUploadResponse(
     val appId: String,
     val deviceConfiguration: DeviceConfiguration?,
     val appBinaryId: String?,
-): UploadResponse()
+) : UploadResponse()
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 data class MaestroCloudUploadResponse(
@@ -468,7 +538,7 @@ data class MaestroCloudUploadResponse(
     val uploadId: String,
     val appBinaryId: String?,
     val deviceInfo: DeviceInfo?
-): UploadResponse()
+) : UploadResponse()
 
 data class DeviceConfiguration(
     val platform: String,
@@ -514,7 +584,6 @@ data class UploadStatus(
         STOPPED
     }
 
-
     // These values must match backend monorepo models
     // in package models.benchmark.BenchmarkCancellationReason
     enum class CancellationReason {
@@ -580,3 +649,46 @@ class SystemInformationInterceptor : Interceptor {
         return chain.proceed(newRequest)
     }
 }
+
+data class Insight(
+    val category: String,
+    val reasoning: String,
+)
+
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.PROPERTY,
+    property = "category"
+)
+@JsonSubTypes(
+    JsonSubTypes.Type(value = AnalyzeResponse.HtmlOutput::class, name = "HTML_OUTPUT"),
+    JsonSubTypes.Type(value = AnalyzeResponse.CliOutput::class, name = "CLI_OUTPUT")
+)
+sealed class AnalyzeResponse(
+    open val status: Status,
+    open val output: String,
+    open val insights: List<Insight>
+) {
+
+    data class HtmlOutput(
+        override val status: Status,
+        override val output: String,
+        override val insights: List<Insight>,
+        val category: Category = Category.HTML_OUTPUT
+    ) : AnalyzeResponse(status, output, insights)
+
+    data class CliOutput(
+        override val status: Status,
+        override val output: String,
+        override val insights: List<Insight>,
+        val category: Category = Category.CLI_OUTPUT
+    ) : AnalyzeResponse(status, output, insights)
+
+    enum class Status {
+        SUCCESS, ERROR
+    }
+
+    enum class Category {
+        HTML_OUTPUT, CLI_OUTPUT
+    }
+}

maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt

@@ -1,6 +1,7 @@
 package maestro.cli.cloud
 
 import maestro.cli.CliError
+import maestro.cli.api.AnalyzeResponse
 import maestro.cli.api.ApiClient
 import maestro.cli.api.DeviceConfiguration
 import maestro.cli.api.DeviceInfo
@@ -9,10 +10,12 @@ import maestro.cli.api.RobinUploadResponse
 import maestro.cli.api.UploadStatus
 import maestro.cli.auth.Auth
 import maestro.cli.device.Platform
+import maestro.cli.insights.FlowFiles
 import maestro.cli.model.FlowStatus
 import maestro.cli.model.RunningFlow
 import maestro.cli.model.RunningFlows
 import maestro.cli.model.TestExecutionSummary
+import maestro.cli.report.HtmlInsightsAnalysisReporter
 import maestro.cli.report.ReportFormat
 import maestro.cli.report.ReporterFactory
 import maestro.cli.util.EnvUtils
@@ -35,6 +38,8 @@ import okio.sink
 import org.rauschig.jarchivelib.ArchiveFormat
 import org.rauschig.jarchivelib.ArchiverFactory
 import java.io.File
+import java.nio.file.Path
+import java.util.*
 import java.util.concurrent.TimeUnit
 import kotlin.io.path.absolute
 
@@ -78,10 +83,7 @@ class CloudInteractor(
         if (mapping?.exists() == false) throw CliError("File does not exist: ${mapping.absolutePath}")
         if (async && reportFormat != ReportFormat.NOOP) throw CliError("Cannot use --format with --async")
 
-        val authToken = apiKey              // Check for API key
-            ?: auth.getCachedAuthToken()    // Otherwise, if the user has already logged in, use the cached auth token
-            ?: EnvUtils.maestroCloudApiKey()        // Resolve API key from shell if set
-            ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow
+        val authToken = getAuthToken(apiKey)
 
         PrintUtils.message("Uploading Flow(s)...")
 
@@ -487,4 +489,55 @@ class CloudInteractor(
             duration = runningFlows.duration
         )
     }
+
+    private fun getAuthToken(apiKey: String?): String {
+        return apiKey // Check for API key
+            ?: auth.getCachedAuthToken() // Otherwise, if the user has already logged in, use the cached auth token
+            ?: EnvUtils.maestroCloudApiKey() // Resolve API key from shell if set
+            ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow
+    }
+
+    fun analyze(
+        apiKey: String?,
+        flowFiles: List<FlowFiles>,
+        debugOutputPath: Path,
+    ): Int {
+        val authToken = getAuthToken(apiKey)
+
+        PrintUtils.info("\n\uD83D\uDD0E Analyzing Flow(s)...")
+
+        try {
+            val response = client.analyze(authToken, flowFiles)
+            if (response.status == AnalyzeResponse.Status.ERROR) {
+                PrintUtils.err("Unexpected error while analyzing Flow(s): ${response.output}")
+                return 1
+            }
+
+            when (response) {
+                is AnalyzeResponse.HtmlOutput -> {
+                    val outputFilePath = HtmlInsightsAnalysisReporter().report(response.output, debugOutputPath)
+                    val os = System.getProperty("os.name").lowercase(Locale.getDefault())
+
+                    PrintUtils.success(
+                        listOf(
+                            "To view the report, open the following link in your browser:",
+                            "file:${if (os.contains("win")) "///" else "//"}${outputFilePath}\n",
+                            "Analyze support is in Beta. We would appreciate your feedback!"
+                        ).joinToString("\n")
+                    )
+
+                    return 0;
+                }
+
+                is AnalyzeResponse.CliOutput -> {
+                    PrintUtils.message(response.output)
+                    return 0
+                }
+            }
+        } catch (error: CliError) {
+            PrintUtils.err("Unexpected error while analyzing Flow(s): ${error.message}")
+            return 1
+        }
+    }
+
 }