feat(analyze): Adds Analyze option to test command

maestro-ai/src/main/java/maestro/ai/Prediction.kt

@@ -270,4 +270,4 @@ object Prediction {
         return response.text ?: ""
     }
 
-}
+}

maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt

@@ -1,6 +1,8 @@
 package maestro.cli.api
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.github.michaelbull.result.Err
@@ -13,6 +15,7 @@ import maestro.cli.model.FlowStatus
 import maestro.cli.runner.resultview.AnsiResultView
 import maestro.cli.util.CiUtils
 import maestro.cli.util.EnvUtils
+import maestro.cli.util.FlowFiles
 import maestro.cli.util.PrintUtils
 import maestro.utils.HttpClient
 import okhttp3.Interceptor
@@ -34,7 +37,6 @@ import java.nio.file.Path
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.exists
 import kotlin.time.Duration.Companion.minutes
-import okhttp3.MediaType
 
 class ApiClient(
     private val baseUrl: String,
@@ -185,7 +187,11 @@ class ApiClient(
         val baseUrl = "https://maestro-record.ngrok.io"
         val body = MultipartBody.Builder()
             .setType(MultipartBody.FORM)
-            .addFormDataPart("screenRecording", screenRecording.name, screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener))
+            .addFormDataPart(
+                "screenRecording",
+                screenRecording.name,
+                screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener)
+            )
             .addFormDataPart("frames", JSON.writeValueAsString(frames))
             .build()
         val request = Request.Builder()
@@ -267,15 +273,27 @@ class ApiClient(
 
         val bodyBuilder = MultipartBody.Builder()
             .setType(MultipartBody.FORM)
-            .addFormDataPart("workspace", "workspace.zip", workspaceZip.toFile().asRequestBody("application/zip".toMediaType()))
+            .addFormDataPart(
+                "workspace",
+                "workspace.zip",
+                workspaceZip.toFile().asRequestBody("application/zip".toMediaType())
+            )
             .addFormDataPart("request", JSON.writeValueAsString(requestPart))
 
         if (appFile != null) {
-            bodyBuilder.addFormDataPart("app_binary", "app.zip", appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener))
+            bodyBuilder.addFormDataPart(
+                "app_binary",
+                "app.zip",
+                appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener)
+            )
         }
 
         if (mappingFile != null) {
-            bodyBuilder.addFormDataPart("mapping", "mapping.txt", mappingFile.toFile().asRequestBody("text/plain".toMediaType()))
+            bodyBuilder.addFormDataPart(
+                "mapping",
+                "mapping.txt",
+                mappingFile.toFile().asRequestBody("text/plain".toMediaType())
+            )
         }
 
         val body = bodyBuilder.build()
@@ -286,7 +304,7 @@ class ApiClient(
                 throw CliError(message)
             }
 
-            PrintUtils.message("$message, retrying (${completedRetries+1}/$maxRetryCount)...")
+            PrintUtils.message("$message, retrying (${completedRetries + 1}/$maxRetryCount)...")
             Thread.sleep(BASE_RETRY_DELAY_MS + (2000 * completedRetries))
 
             return upload(
@@ -440,6 +458,58 @@ class ApiClient(
         }
     }
 
+    fun analyze(
+        authToken: String,
+        flowFiles: List<FlowFiles>,
+    ): AnalyzeResponse {
+        if (flowFiles.isEmpty()) throw CliError("Missing flow files to analyze")
+
+        val screenshots = mutableListOf<Pair<String, ByteArray>>()
+        val logs = mutableListOf<Pair<String, ByteArray>>()
+
+        flowFiles.forEach { flowFile ->
+            flowFile.imageFiles.forEach { (imageData, path) ->
+                val imageName = path.fileName.toString()
+                screenshots.add(Pair(imageName, imageData))
+            }
+
+            flowFile.textFiles.forEach { (textData, path) ->
+                val textName = path.fileName.toString()
+                logs.add(Pair(textName, textData))
+            }
+        }
+
+        val requestBody = mapOf(
+            "screenshots" to screenshots,
+            "logs" to logs
+        )
+
+        val mediaType = "application/json; charset=utf-8".toMediaType()
+        val body = JSON.writeValueAsString(requestBody).toRequestBody(mediaType)
+
+        val url = "$baseUrl/v2/analyze"
+
+        val request = Request.Builder()
+            .header("Authorization", "Bearer $authToken")
+            .url(url)
+            .post(body)
+            .build()
+
+        val response = client.newCall(request).execute()
+
+        response.use {
+            if (!response.isSuccessful) {
+                val errorMessage = response.body?.string().takeIf { it?.isNotEmpty() == true } ?: "Unknown"
+                throw CliError("Analyze request failed (${response.code}): $errorMessage")
+            }
+
+            val parsed = JSON.readValue(response.body?.bytes(), AnalyzeResponse::class.java)
+
+            return parsed;
+        }
+    }
+
+
     data class ApiException(
         val statusCode: Int?,
     ) : Exception("Request failed. Status code: $statusCode")
@@ -459,7 +529,7 @@ data class RobinUploadResponse(
     val appId: String,
     val deviceConfiguration: DeviceConfiguration?,
     val appBinaryId: String?,
-): UploadResponse()
+) : UploadResponse()
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 data class MaestroCloudUploadResponse(
@@ -468,7 +538,7 @@ data class MaestroCloudUploadResponse(
     val uploadId: String,
     val appBinaryId: String?,
     val deviceInfo: DeviceInfo?
-): UploadResponse()
+) : UploadResponse()
 
 data class DeviceConfiguration(
     val platform: String,
@@ -514,7 +584,6 @@ data class UploadStatus(
         STOPPED
     }
 
-
     // These values must match backend monorepo models
     // in package models.benchmark.BenchmarkCancellationReason
     enum class CancellationReason {
@@ -580,3 +649,46 @@ class SystemInformationInterceptor : Interceptor {
         return chain.proceed(newRequest)
     }
 }
+
+data class Insight(
+    val category: String,
+    val reasoning: String,
+)
+
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.PROPERTY,
+    property = "category"
+)
+@JsonSubTypes(
+    JsonSubTypes.Type(value = AnalyzeResponse.HtmlOutput::class, name = "HTML_OUTPUT"),
+    JsonSubTypes.Type(value = AnalyzeResponse.CliOutput::class, name = "CLI_OUTPUT")
+)
+sealed class AnalyzeResponse(
+    open val status: Status,
+    open val output: String,
+    open val insights: List<Insight>
+) {
+
+    data class HtmlOutput(
+        override val status: Status,
+        override val output: String,
+        override val insights: List<Insight>,
+        val category: Category = Category.HTML_OUTPUT
+    ) : AnalyzeResponse(status, output, insights)
+
+    data class CliOutput(
+        override val status: Status,
+        override val output: String,
+        override val insights: List<Insight>,
+        val category: Category = Category.CLI_OUTPUT
+    ) : AnalyzeResponse(status, output, insights)
+
+    enum class Status {
+        SUCCESS, ERROR
+    }
+
+    enum class Category {
+        HTML_OUTPUT, CLI_OUTPUT
+    }
+}

maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt

@@ -1,6 +1,7 @@
 package maestro.cli.cloud
 
 import maestro.cli.CliError
+import maestro.cli.api.AnalyzeResponse
 import maestro.cli.api.ApiClient
 import maestro.cli.api.DeviceConfiguration
 import maestro.cli.api.DeviceInfo
@@ -13,11 +14,13 @@ import maestro.cli.model.FlowStatus
 import maestro.cli.model.RunningFlow
 import maestro.cli.model.RunningFlows
 import maestro.cli.model.TestExecutionSummary
+import maestro.cli.report.HtmlInsightsAnalysisReporter
 import maestro.cli.report.ReportFormat
 import maestro.cli.report.ReporterFactory
 import maestro.cli.util.EnvUtils
 import maestro.cli.util.FileUtils.isWebFlow
 import maestro.cli.util.FileUtils.isZip
+import maestro.cli.util.FlowFiles
 import maestro.cli.util.PrintUtils
 import maestro.cli.util.TimeUtils
 import maestro.cli.util.WorkspaceUtils
@@ -35,6 +38,8 @@ import okio.sink
 import org.rauschig.jarchivelib.ArchiveFormat
 import org.rauschig.jarchivelib.ArchiverFactory
 import java.io.File
+import java.nio.file.Path
+import java.util.*
 import java.util.concurrent.TimeUnit
 import kotlin.io.path.absolute
 
@@ -78,10 +83,7 @@ class CloudInteractor(
         if (mapping?.exists() == false) throw CliError("File does not exist: ${mapping.absolutePath}")
         if (async && reportFormat != ReportFormat.NOOP) throw CliError("Cannot use --format with --async")
 
-        val authToken = apiKey              // Check for API key
-            ?: auth.getCachedAuthToken()    // Otherwise, if the user has already logged in, use the cached auth token
-            ?: EnvUtils.maestroCloudApiKey()        // Resolve API key from shell if set
-            ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow
+        val authToken = getAuthToken(apiKey)
 
         PrintUtils.message("Uploading Flow(s)...")
 
@@ -487,4 +489,55 @@ class CloudInteractor(
             duration = runningFlows.duration
         )
     }
+
+    private fun getAuthToken(apiKey: String?): String {
+        return apiKey // Check for API key
+            ?: auth.getCachedAuthToken() // Otherwise, if the user has already logged in, use the cached auth token
+            ?: EnvUtils.maestroCloudApiKey() // Resolve API key from shell if set
+            ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow
+    }
+
+    fun analyze(
+        apiKey: String?,
+        flowFiles: List<FlowFiles>,
+        debugOutputPath: Path,
+    ): Int {
+        val authToken = getAuthToken(apiKey)
+
+        PrintUtils.info("\uD83D\uDD0E Analyzing Flow(s)...\n")
+
+        try {
+            val response = client.analyze(authToken, flowFiles)
+            if (response.status == AnalyzeResponse.Status.ERROR) {
+                PrintUtils.err("Unexpected error while analyzing Flow(s): ${response.output}")
+                return 1
+            }
+
+            when (response) {
+                is AnalyzeResponse.HtmlOutput -> {
+                    val outputFilePath = HtmlInsightsAnalysisReporter().report(response.output, debugOutputPath)
+                    val os = System.getProperty("os.name").lowercase(Locale.getDefault())
+
+                    PrintUtils.message(
+                        listOf(
+                            "To view the report, open the following link in your browser:",
+                            "file:${if (os.contains("win")) "///" else "//"}${outputFilePath}\n",
+                            "Analyze support is in Beta. We would appreciate your feedback!"
+                        ).joinToString("\n")
+                    )
+
+                    return 0;
+                }
+
+                is AnalyzeResponse.CliOutput -> {
+                    PrintUtils.message(response.output)
+                    return 0
+                }
+            }
+        } catch (error: CliError) {
+            PrintUtils.err("Unexpected error while analyzing Flow(s): ${error.message}")
+            return 1
+        }
+    }
+
 }

maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt

@@ -10,6 +10,7 @@ import java.util.concurrent.Callable
 import kotlin.io.path.absolutePathString
 import maestro.cli.report.TestDebugReporter
 import maestro.debuglog.LogConfig
+import picocli.CommandLine.Option
 
 @CommandLine.Command(
     name = "login",
@@ -25,8 +26,11 @@ class LoginCommand : Callable<Int> {
     @CommandLine.Mixin
     var showHelpMixin: ShowHelpMixin? = null
 
+    @Option(names = ["--apiUrl"], description = ["API base URL"])
+    private var apiUrl: String = "https://api.copilot.mobile.dev"
+
     private val auth by lazy {
-        Auth(ApiClient("https://api.copilot.mobile.dev/v2"))
+        Auth(ApiClient("$apiUrl/v2"))
     }
 
     override fun call(): Int {