diff --git a/maestro-cli/src/main/java/maestro/cli/App.kt b/maestro-cli/src/main/java/maestro/cli/App.kt index 915eaf57ac..757c66ff06 100644 --- a/maestro-cli/src/main/java/maestro/cli/App.kt +++ b/maestro-cli/src/main/java/maestro/cli/App.kt @@ -33,6 +33,7 @@ import maestro.cli.command.StartDeviceCommand import maestro.cli.command.StudioCommand import maestro.cli.command.TestCommand import maestro.cli.command.UploadCommand +import maestro.cli.insights.TestAnalysisManager import maestro.cli.update.Updates import maestro.cli.util.ChangeLogUtils import maestro.cli.util.ErrorReporter @@ -143,6 +144,7 @@ fun main(args: Array) { .execute(*args) DebugLogStore.finalizeRun() + TestAnalysisManager.maybeNotify() val newVersion = Updates.checkForUpdates() if (newVersion != null) { diff --git a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt index 067c35b8be..e49086ec4b 100644 --- a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt +++ b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt @@ -11,6 +11,7 @@ import com.github.michaelbull.result.Result import maestro.cli.CliError import maestro.cli.analytics.Analytics import maestro.cli.analytics.AnalyticsReport +import maestro.cli.insights.FlowFiles import maestro.cli.model.FlowStatus import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.util.CiUtils diff --git a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt index ad9a792bcb..04d16b616e 100644 --- a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt @@ -10,6 +10,7 @@ import maestro.cli.api.RobinUploadResponse import maestro.cli.api.UploadStatus import maestro.cli.auth.Auth import maestro.cli.device.Platform +import maestro.cli.insights.FlowFiles import maestro.cli.model.FlowStatus import maestro.cli.model.RunningFlow import maestro.cli.model.RunningFlows diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index ce44143870..585b4c3916 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -43,7 +43,7 @@ import maestro.cli.session.MaestroSessionManager import maestro.cli.util.EnvUtils import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.PrintUtils -import maestro.cli.util.TestAnalysisReporter +import maestro.cli.insights.TestAnalysisManager import maestro.cli.view.box import maestro.orchestra.error.ValidationError import maestro.orchestra.util.Env.withDefaultEnvVars @@ -304,7 +304,7 @@ class TestCommand : Callable { suites.mergeSummaries()?.saveReport() if (effectiveShards > 1) printShardsMessage(passed, total, suites) - if (analyze) TestAnalysisReporter(apiUrl = apiUrl, apiKey = apiKey).runAnalysis(debugOutputPath) + if (analyze) TestAnalysisManager(apiUrl = apiUrl, apiKey = apiKey).runAnalysis(debugOutputPath) if (passed == total) 0 else 1 } diff --git a/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt b/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt new file mode 100644 index 0000000000..2bd1d05f37 --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt @@ -0,0 +1,142 @@ +package maestro.cli.insights + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors +import maestro.cli.api.ApiClient +import maestro.cli.cloud.CloudInteractor +import maestro.cli.util.EnvUtils +import maestro.cli.util.PrintUtils +import maestro.cli.view.box +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.readText +import kotlin.io.path.writeText + +data class FlowFiles( + val imageFiles: List>, + val textFiles: List> +) + +class TestAnalysisManager(private val apiUrl: String, private val apiKey: String?) { + private val apiclient by lazy { + ApiClient(apiUrl) + } + + fun runAnalysis(debugOutputPath: Path): Int { + val flowFiles = processFilesByFlowName(debugOutputPath) + if (flowFiles.isEmpty()) { + PrintUtils.warn("No screenshots or debug artifacts found for analysis.") + return 0; + } + + return CloudInteractor(apiclient).analyze( + apiKey = apiKey, + flowFiles = flowFiles, + debugOutputPath = debugOutputPath + ) + } + + private fun processFilesByFlowName(outputPath: Path): List { + val files = Files.walk(outputPath) + .filter(Files::isRegularFile) + .collect(Collectors.toList()) + + return if (files.isNotEmpty()) { + val (imageFiles, textFiles) = getDebugFiles(files) + listOf( + FlowFiles( + imageFiles = imageFiles, + textFiles = textFiles + ) + ) + } else { + emptyList() + } + } + + private fun getDebugFiles(files: List): Pair>, List>> { + val imageFiles = mutableListOf>() + val textFiles = mutableListOf>() + + files.forEach { filePath -> + val content = Files.readAllBytes(filePath) + val fileName = filePath.fileName.toString().lowercase() + + when { + fileName.endsWith(".png") || fileName.endsWith(".jpg") || fileName.endsWith(".jpeg") -> { + imageFiles.add(content to filePath) + } + + fileName.startsWith("commands") -> { + textFiles.add(content to filePath) + } + + fileName == "maestro.log" -> { + textFiles.add(content to filePath) + } + } + } + + return Pair(imageFiles, textFiles) + } + + /** + * The analytics system for Test Analysis. + * - Uses configuration from $XDG_CONFIG_HOME/maestro/analyze-analytics.json. + */ + companion object Analytics { + private const val DISABLE_INSIGHTS_ENV_VAR = "MAESTRO_CLI_INSIGHTS_NOTIFICATION_DISABLED" + private val disabled: Boolean + get() = System.getenv(DISABLE_INSIGHTS_ENV_VAR) == "true" + + private val analyticsStatePath: Path = EnvUtils.xdgStateHome().resolve("analyze-analytics.json") + + private val JSON = jacksonObjectMapper().apply { + registerModule(JavaTimeModule()) + enable(SerializationFeature.INDENT_OUTPUT) + } + + private val shouldNotNotify: Boolean + get() = disabled || analyticsStatePath.exists() && analyticsState.acknowledged + + private val analyticsState: AnalyticsState + get() = JSON.readValue(analyticsStatePath.readText()) + + fun maybeNotify() { + if (shouldNotNotify) return + + println( + listOf( + "Tryout our new Analyze with Ai feature.\n", + "See what's new:", + "> https://maestro.mobile.dev/cli/test-suites-and-reports#analyze", + "Analyze command:", + "$ maestro test flow-file.yaml --analyze | bash\n", + "To disable this notification, set $DISABLE_INSIGHTS_ENV_VAR environment variable to \"true\" before running Maestro." + ).joinToString("\n").box() + ) + ack(); + } + + private fun ack() { + val state = AnalyticsState( + acknowledged = true + ) + + val stateJson = JSON.writeValueAsString(state) + analyticsStatePath.parent.createDirectories() + analyticsStatePath.writeText(stateJson + "\n") + } + } +} + +@JsonIgnoreProperties(ignoreUnknown = true) +data class AnalyticsState( + val acknowledged: Boolean = false +)