diff --git a/maestro-ai/src/main/java/maestro/ai/Prediction.kt b/maestro-ai/src/main/java/maestro/ai/Prediction.kt index 332c6bb79d..95d41da4ca 100644 --- a/maestro-ai/src/main/java/maestro/ai/Prediction.kt +++ b/maestro-ai/src/main/java/maestro/ai/Prediction.kt @@ -270,4 +270,4 @@ object Prediction { return response.text ?: "" } -} +} \ No newline at end of file diff --git a/maestro-cli/src/main/java/maestro/cli/App.kt b/maestro-cli/src/main/java/maestro/cli/App.kt index 915eaf57ac..757c66ff06 100644 --- a/maestro-cli/src/main/java/maestro/cli/App.kt +++ b/maestro-cli/src/main/java/maestro/cli/App.kt @@ -33,6 +33,7 @@ import maestro.cli.command.StartDeviceCommand import maestro.cli.command.StudioCommand import maestro.cli.command.TestCommand import maestro.cli.command.UploadCommand +import maestro.cli.insights.TestAnalysisManager import maestro.cli.update.Updates import maestro.cli.util.ChangeLogUtils import maestro.cli.util.ErrorReporter @@ -143,6 +144,7 @@ fun main(args: Array) { .execute(*args) DebugLogStore.finalizeRun() + TestAnalysisManager.maybeNotify() val newVersion = Updates.checkForUpdates() if (newVersion != null) { diff --git a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt index 18f1b3d79a..ee4e79ec82 100644 --- a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt +++ b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt @@ -9,6 +9,7 @@ import com.github.michaelbull.result.Result import maestro.cli.CliError import maestro.cli.analytics.Analytics import maestro.cli.analytics.AnalyticsReport +import maestro.cli.insights.AnalysisDebugFiles import maestro.cli.model.FlowStatus import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.util.CiUtils @@ -34,7 +35,6 @@ import java.nio.file.Path import kotlin.io.path.absolutePathString import kotlin.io.path.exists import kotlin.time.Duration.Companion.minutes -import okhttp3.MediaType class ApiClient( private val baseUrl: String, @@ -185,7 +185,11 @@ class ApiClient( val baseUrl = "https://maestro-record.ngrok.io" val body = MultipartBody.Builder() .setType(MultipartBody.FORM) - .addFormDataPart("screenRecording", screenRecording.name, screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener)) + .addFormDataPart( + "screenRecording", + screenRecording.name, + screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener) + ) .addFormDataPart("frames", JSON.writeValueAsString(frames)) .build() val request = Request.Builder() @@ -267,15 +271,27 @@ class ApiClient( val bodyBuilder = MultipartBody.Builder() .setType(MultipartBody.FORM) - .addFormDataPart("workspace", "workspace.zip", workspaceZip.toFile().asRequestBody("application/zip".toMediaType())) + .addFormDataPart( + "workspace", + "workspace.zip", + workspaceZip.toFile().asRequestBody("application/zip".toMediaType()) + ) .addFormDataPart("request", JSON.writeValueAsString(requestPart)) if (appFile != null) { - bodyBuilder.addFormDataPart("app_binary", "app.zip", appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener)) + bodyBuilder.addFormDataPart( + "app_binary", + "app.zip", + appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener) + ) } if (mappingFile != null) { - bodyBuilder.addFormDataPart("mapping", "mapping.txt", mappingFile.toFile().asRequestBody("text/plain".toMediaType())) + bodyBuilder.addFormDataPart( + "mapping", + "mapping.txt", + mappingFile.toFile().asRequestBody("text/plain".toMediaType()) + ) } val body = bodyBuilder.build() @@ -286,7 +302,7 @@ class ApiClient( throw CliError(message) } - PrintUtils.message("$message, retrying (${completedRetries+1}/$maxRetryCount)...") + PrintUtils.message("$message, retrying (${completedRetries + 1}/$maxRetryCount)...") Thread.sleep(BASE_RETRY_DELAY_MS + (2000 * completedRetries)) return upload( @@ -440,6 +456,36 @@ class ApiClient( } } + fun analyze( + authToken: String, + debugFiles: AnalysisDebugFiles, + ): AnalyzeResponse { + val mediaType = "application/json; charset=utf-8".toMediaType() + val body = JSON.writeValueAsString(debugFiles).toRequestBody(mediaType) + + val url = "$baseUrl/v2/analyze" + + val request = Request.Builder() + .header("Authorization", "Bearer $authToken") + .url(url) + .post(body) + .build() + + val response = client.newCall(request).execute() + + response.use { + if (!response.isSuccessful) { + val errorMessage = response.body?.string().takeIf { it?.isNotEmpty() == true } ?: "Unknown" + throw CliError("Analyze request failed (${response.code}): $errorMessage") + } + + val parsed = JSON.readValue(response.body?.bytes(), AnalyzeResponse::class.java) + + return parsed; + } + } + + data class ApiException( val statusCode: Int?, ) : Exception("Request failed. Status code: $statusCode") @@ -459,7 +505,7 @@ data class RobinUploadResponse( val appId: String, val deviceConfiguration: DeviceConfiguration?, val appBinaryId: String?, -): UploadResponse() +) : UploadResponse() @JsonIgnoreProperties(ignoreUnknown = true) data class MaestroCloudUploadResponse( @@ -468,7 +514,7 @@ data class MaestroCloudUploadResponse( val uploadId: String, val appBinaryId: String?, val deviceInfo: DeviceInfo? -): UploadResponse() +) : UploadResponse() data class DeviceConfiguration( val platform: String, @@ -514,7 +560,6 @@ data class UploadStatus( STOPPED } - // These values must match backend monorepo models // in package models.benchmark.BenchmarkCancellationReason enum class CancellationReason { @@ -580,3 +625,14 @@ class SystemInformationInterceptor : Interceptor { return chain.proceed(newRequest) } } + +data class Insight( + val category: String, + val reasoning: String, +) + +class AnalyzeResponse( + val htmlReport: String?, + val output: String, + val insights: List +) diff --git a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt index 48f5a49c1c..5bd16019f3 100644 --- a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt @@ -9,10 +9,12 @@ import maestro.cli.api.RobinUploadResponse import maestro.cli.api.UploadStatus import maestro.cli.auth.Auth import maestro.cli.device.Platform +import maestro.cli.insights.AnalysisDebugFiles import maestro.cli.model.FlowStatus import maestro.cli.model.RunningFlow import maestro.cli.model.RunningFlows import maestro.cli.model.TestExecutionSummary +import maestro.cli.report.HtmlInsightsAnalysisReporter import maestro.cli.report.ReportFormat import maestro.cli.report.ReporterFactory import maestro.cli.util.EnvUtils @@ -35,6 +37,8 @@ import okio.sink import org.rauschig.jarchivelib.ArchiveFormat import org.rauschig.jarchivelib.ArchiverFactory import java.io.File +import java.nio.file.Path +import java.util.* import java.util.concurrent.TimeUnit import kotlin.io.path.absolute @@ -78,10 +82,7 @@ class CloudInteractor( if (mapping?.exists() == false) throw CliError("File does not exist: ${mapping.absolutePath}") if (async && reportFormat != ReportFormat.NOOP) throw CliError("Cannot use --format with --async") - val authToken = apiKey // Check for API key - ?: auth.getCachedAuthToken() // Otherwise, if the user has already logged in, use the cached auth token - ?: EnvUtils.maestroCloudApiKey() // Resolve API key from shell if set - ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow + val authToken = getAuthToken(apiKey) PrintUtils.message("Uploading Flow(s)...") @@ -487,4 +488,44 @@ class CloudInteractor( duration = runningFlows.duration ) } + + private fun getAuthToken(apiKey: String?): String { + return apiKey // Check for API key + ?: auth.getCachedAuthToken() // Otherwise, if the user has already logged in, use the cached auth token + ?: EnvUtils.maestroCloudApiKey() // Resolve API key from shell if set + ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow + } + + fun analyze( + apiKey: String?, + debugFiles: AnalysisDebugFiles, + debugOutputPath: Path, + ): Int { + val authToken = getAuthToken(apiKey) + + PrintUtils.info("\n\uD83D\uDD0E Analyzing Flow(s)...") + + try { + val response = client.analyze(authToken, debugFiles) + + if (response.htmlReport.isNullOrEmpty()) { + PrintUtils.info(response.output) + return 0 + } + + val outputFilePath = HtmlInsightsAnalysisReporter().report(response.htmlReport, debugOutputPath) + val os = System.getProperty("os.name").lowercase(Locale.getDefault()) + + val formattedOutput = response.output.replace( + "{{outputFilePath}}", + "file:${if (os.contains("win")) "///" else "//"}${outputFilePath}\n" + ) + + PrintUtils.info(formattedOutput); + return 0; + } catch (error: CliError) { + PrintUtils.err("Unexpected error while analyzing Flow(s): ${error.message}") + return 1 + } + } } diff --git a/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt index 5338beaa44..a6a2520510 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt @@ -10,6 +10,7 @@ import java.util.concurrent.Callable import kotlin.io.path.absolutePathString import maestro.cli.report.TestDebugReporter import maestro.debuglog.LogConfig +import picocli.CommandLine.Option @CommandLine.Command( name = "login", @@ -25,8 +26,11 @@ class LoginCommand : Callable { @CommandLine.Mixin var showHelpMixin: ShowHelpMixin? = null + @Option(names = ["--api-url", "--apiUrl"], description = ["API base URL"]) + private var apiUrl: String = "https://api.copilot.mobile.dev" + private val auth by lazy { - Auth(ApiClient("https://api.copilot.mobile.dev/v2")) + Auth(ApiClient("$apiUrl/v2")) } override fun call(): Int { diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index 58dfcb23ea..585b4c3916 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -43,6 +43,7 @@ import maestro.cli.session.MaestroSessionManager import maestro.cli.util.EnvUtils import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.PrintUtils +import maestro.cli.insights.TestAnalysisManager import maestro.cli.view.box import maestro.orchestra.error.ValidationError import maestro.orchestra.util.Env.withDefaultEnvVars @@ -51,7 +52,6 @@ import maestro.orchestra.workspace.WorkspaceExecutionPlanner import maestro.orchestra.workspace.WorkspaceExecutionPlanner.ExecutionPlan import maestro.utils.isSingleFile import okio.sink -import org.jetbrains.skiko.hostId import org.slf4j.LoggerFactory import picocli.CommandLine import picocli.CommandLine.Option @@ -158,6 +158,19 @@ class TestCommand : Callable { ) private var headless: Boolean = false + @Option( + names = ["--analyze"], + description = ["[Beta] Enhance the test output analysis with AI Insights"], + ) + private var analyze: Boolean = false + + @Option(names = ["--api-url"], description = ["[Beta] API base URL"]) + private var apiUrl: String = "https://api.copilot.mobile.dev" + + + @Option(names = ["--api-key"], description = ["[Beta] API key"]) + private var apiKey: String? = null + @CommandLine.Spec lateinit var commandSpec: CommandLine.Model.CommandSpec @@ -172,6 +185,7 @@ class TestCommand : Callable { return false } + override fun call(): Int { TestDebugReporter.install( debugOutputPathAsString = debugOutput, @@ -290,6 +304,7 @@ class TestCommand : Callable { suites.mergeSummaries()?.saveReport() if (effectiveShards > 1) printShardsMessage(passed, total, suites) + if (analyze) TestAnalysisManager(apiUrl = apiUrl, apiKey = apiKey).runAnalysis(debugOutputPath) if (passed == total) 0 else 1 } @@ -333,7 +348,7 @@ class TestCommand : Callable { if (!flattenDebugOutput) { TestDebugReporter.deleteOldFiles() } - TestRunner.runContinuous(maestro, device, flowFile, env) + TestRunner.runContinuous(maestro, device, flowFile, env, analyze) } else { runSingleFlow(maestro, device, flowFile, debugOutputPath) } @@ -371,6 +386,7 @@ class TestCommand : Callable { env = env, resultView = resultView, debugOutputPath = debugOutputPath, + analyze = analyze ) if (resultSingle == 1) { diff --git a/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt b/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt new file mode 100644 index 0000000000..4828adbdd9 --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt @@ -0,0 +1,179 @@ +package maestro.cli.insights + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.github.romankh3.image.comparison.ImageComparisonUtil +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors +import maestro.cli.api.ApiClient +import maestro.cli.cloud.CloudInteractor +import maestro.cli.util.EnvUtils +import maestro.cli.util.PrintUtils +import maestro.cli.view.box +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.readText +import kotlin.io.path.writeText + +data class AnalysisScreenshot ( + val data: ByteArray, + val path: Path, +) + +data class AnalysisLog ( + val data: ByteArray, + val path: Path, +) + +data class AnalysisDebugFiles( + val screenshots: List, + val logs: List, + val commands: List, +) + +class TestAnalysisManager(private val apiUrl: String, private val apiKey: String?) { + private val apiClient by lazy { + ApiClient(apiUrl) + } + + fun runAnalysis(debugOutputPath: Path): Int { + val debugFiles = processDebugFiles(debugOutputPath) + if (debugFiles == null) { + PrintUtils.warn("No screenshots or debug artifacts found for analysis.") + return 0; + } + + return CloudInteractor(apiClient).analyze( + apiKey = apiKey, + debugFiles = debugFiles, + debugOutputPath = debugOutputPath + ) + } + + private fun processDebugFiles(outputPath: Path): AnalysisDebugFiles? { + val files = Files.walk(outputPath) + .filter(Files::isRegularFile) + .collect(Collectors.toList()) + + if (files.isEmpty()) { + return null + } + + return getDebugFiles(files) + } + + private fun getDebugFiles(files: List): AnalysisDebugFiles { + val logs = mutableListOf() + val commands = mutableListOf() + val screenshots = mutableListOf() + + files.forEach { path -> + val data = Files.readAllBytes(path) + val fileName = path.fileName.toString().lowercase() + + when { + fileName.endsWith(".png") || fileName.endsWith(".jpg") || fileName.endsWith(".jpeg") -> { + screenshots.add(AnalysisScreenshot(data = data, path = path)) + } + + fileName.startsWith("commands") -> { + commands.add(AnalysisLog(data = data, path = path)) + } + + fileName == "maestro.log" -> { + logs.add(AnalysisLog(data = data, path = path)) + } + } + } + + val filteredScreenshots = filterSimilarScreenshots(screenshots) + + return AnalysisDebugFiles( + logs = logs, + commands = commands, + screenshots = filteredScreenshots, + ) + } + + private val screenshotsDifferenceThreshold = 5.0 + + private fun filterSimilarScreenshots( + screenshots: List + ): List { + val uniqueScreenshots = mutableListOf() + + for (screenshot in screenshots) { + val isSimilar = uniqueScreenshots.any { existingScreenshot -> + val diffPercent = ImageComparisonUtil.getDifferencePercent( + ImageComparisonUtil.readImageFromResources(existingScreenshot.path.toString()), + ImageComparisonUtil.readImageFromResources(screenshot.path.toString()) + ) + diffPercent <= screenshotsDifferenceThreshold + } + + if (!isSimilar) { + uniqueScreenshots.add(screenshot) + } + } + + return uniqueScreenshots + } + + /** + * The Notification system for Test Analysis. + * - Uses configuration from $XDG_CONFIG_HOME/maestro/analyze-notification.json. + */ + companion object AnalysisNotification { + private const val MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED = "MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED" + private val disabled: Boolean + get() = System.getenv(MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED) == "true" + + private val notificationStatePath: Path = EnvUtils.xdgStateHome().resolve("analyze-notification.json") + + private val JSON = jacksonObjectMapper().apply { + registerModule(JavaTimeModule()) + enable(SerializationFeature.INDENT_OUTPUT) + } + + private val shouldNotNotify: Boolean + get() = disabled || notificationStatePath.exists() && notificationState.acknowledged + + private val notificationState: AnalysisNotificationState + get() = JSON.readValue(notificationStatePath.readText()) + + fun maybeNotify() { + if (shouldNotNotify) return + + println( + listOf( + "Try out our new Analyze with Ai feature.\n", + "See what's new:", + "> https://maestro.mobile.dev/cli/test-suites-and-reports#analyze", + "Analyze command:", + "$ maestro test flow-file.yaml --analyze | bash\n", + "To disable this notification, set $MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED environment variable to \"true\" before running Maestro." + ).joinToString("\n").box() + ) + ack(); + } + + private fun ack() { + val state = AnalysisNotificationState( + acknowledged = true + ) + + val stateJson = JSON.writeValueAsString(state) + notificationStatePath.parent.createDirectories() + notificationStatePath.writeText(stateJson + "\n") + } + } +} + +@JsonIgnoreProperties(ignoreUnknown = true) +data class AnalysisNotificationState( + val acknowledged: Boolean = false +) diff --git a/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt new file mode 100644 index 0000000000..6d55cd0d8c --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt @@ -0,0 +1,23 @@ +package maestro.cli.report + +import java.nio.file.Files +import java.nio.file.Path + +class HtmlInsightsAnalysisReporter { + + fun report( + html: String, + outputDestination: Path + ): Path { + if (!Files.isDirectory(outputDestination)) { + throw IllegalArgumentException("Output destination must be a directory") + } + + val fileName = "insights-report.html" + val filePath = outputDestination.resolve(fileName) + + Files.write(filePath, html.toByteArray()) + + return filePath + } +} diff --git a/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt index 11ceef4d9d..313dac892b 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt @@ -56,6 +56,7 @@ object MaestroCommandRunner { commands: List, debugOutput: FlowDebugOutput, aiOutput: FlowAIOutput, + analyze: Boolean = false ): Boolean { val config = YamlCommandReader.getConfig(commands) val onFlowComplete = config?.onFlowComplete @@ -89,6 +90,9 @@ object MaestroCommandRunner { } refreshUi() + if (analyze) { + ScreenshotUtils.takeDebugScreenshotByCommand(maestro, debugOutput, CommandStatus.PENDING) + } val orchestra = Orchestra( maestro = maestro, @@ -106,6 +110,10 @@ object MaestroCommandRunner { onCommandComplete = { _, command -> logger.info("${command.description()} COMPLETED") commandStatuses[command] = CommandStatus.COMPLETED + if (analyze) { + ScreenshotUtils.takeDebugScreenshotByCommand(maestro, debugOutput, CommandStatus.COMPLETED) + } + debugOutput.commands[command]?.apply { status = CommandStatus.COMPLETED calculateDuration() diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt index 3bfee3e381..1730ebf5c8 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt @@ -43,7 +43,8 @@ object TestRunner { flowFile: File, env: Map, resultView: ResultView, - debugOutputPath: Path + debugOutputPath: Path, + analyze: Boolean = false ): Int { val debugOutput = FlowDebugOutput() var aiOutput = FlowAIOutput( @@ -68,6 +69,7 @@ object TestRunner { commands = commands, debugOutput = debugOutput, aiOutput = aiOutput, + analyze = analyze, ) } @@ -94,6 +96,7 @@ object TestRunner { device: Device?, flowFile: File, env: Map, + analyze: Boolean = false, ): Nothing { val resultView = AnsiResultView("> Press [ENTER] to restart the Flow\n\n", useEmojis = !EnvUtils.isWindows()) @@ -133,6 +136,7 @@ object TestRunner { flowName = "TODO", flowFile = flowFile, ), + analyze = analyze ) }.get() } diff --git a/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt b/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt index c3a6e1a2f6..140e71d2b2 100644 --- a/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt +++ b/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt @@ -35,6 +35,25 @@ object ScreenshotUtils { return result.getOrNull() } + fun takeDebugScreenshotByCommand(maestro: Maestro, debugOutput: FlowDebugOutput, status: CommandStatus): File? { + val result = kotlin.runCatching { + val out = File + .createTempFile("screenshot-${status}-${System.currentTimeMillis()}", ".png") + .also { it.deleteOnExit() } // save to another dir before exiting + maestro.takeScreenshot(out.sink(), false) + debugOutput.screenshots.add( + FlowDebugOutput.Screenshot( + screenshot = out, + timestamp = System.currentTimeMillis(), + status = status + ) + ) + out + } + + return result.getOrNull() + } + fun writeAIscreenshot(buffer: Buffer): File { val out = File .createTempFile("ai-screenshot-${System.currentTimeMillis()}", ".png")