work on saving files

mobile-dev-inc · Aug 13, 2024 · 644c94c · 644c94c
1 parent 6ba3d0a
commit 644c94c
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 50 deletions.
diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt
@@ -256,6 +256,8 @@ class TestCommand : Callable<Int> {
                         val device = session.device
 
                         if (flowFile.isDirectory || format != ReportFormat.NOOP) {
+                            // Run multiple flows
+
                             if (continuous) {
                                 throw CommandLine.ParameterException(
                                     commandSpec.commandLine(),
@@ -279,6 +281,8 @@ class TestCommand : Callable<Int> {
                             }
                             Triple(suiteResult.passedCount, suiteResult.totalTests, suiteResult)
                         } else {
+                            // Run a single flow
+
                             if (continuous) {
                                 if (!flattenDebugOutput) {
                                     TestDebugReporter.deleteOldFiles()

diff --git a/maestro-cli/src/main/java/maestro/cli/report/TestDebugReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/TestDebugReporter.kt
@@ -1,6 +1,7 @@
 package maestro.cli.report
 
 import com.fasterxml.jackson.annotation.JsonInclude
+import com.fasterxml.jackson.annotation.JsonProperty
 import com.fasterxml.jackson.databind.JsonMappingException
 import com.fasterxml.jackson.databind.ObjectMapper
 import maestro.Driver
@@ -13,6 +14,7 @@ import maestro.cli.util.IOSEnvUtils
 import maestro.debuglog.DebugLogStore
 import maestro.debuglog.LogConfig
 import maestro.orchestra.MaestroCommand
+import maestro.orchestra.ai.Defect
 import org.slf4j.LoggerFactory
 import java.io.File
 import java.nio.file.Files
@@ -29,6 +31,7 @@ import java.util.Properties
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.exists
 
+// TODO(bartekpacia): Rename to TestOutputReporter, because it's not only for "debug" stuff
 object TestDebugReporter {
 
     private val logger = LoggerFactory.getLogger(TestDebugReporter::class.java)
@@ -45,11 +48,12 @@ object TestDebugReporter {
         mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY)
     }
 
-    fun saveFlow(flowName: String, data: FlowDebugMetadata, path: Path) {
+    fun saveFlow(flowName: String, debugOutput: FlowDebugOutput, aiOutput: AIOutput, path: Path) {
+        println("TestDebugReporter.saveFlow: saving flow metadata to $path")
 
         // commands
         try {
-            val commandMetadata = data.commands
+            val commandMetadata = debugOutput.commands
             if (commandMetadata.isNotEmpty()) {
                 val commandsFilename = "commands-(${flowName.replace("/", "_")}).json"
                 val file = File(path.absolutePathString(), commandsFilename)
@@ -62,7 +66,7 @@ object TestDebugReporter {
         }
 
         // screenshots
-        data.screenshots.forEach {
+        debugOutput.screenshots.forEach {
             val status = when (it.status) {
                 CommandStatus.COMPLETED -> "✅"
                 CommandStatus.FAILED -> "❌"
@@ -80,31 +84,25 @@ object TestDebugReporter {
             val currentTime = Instant.now()
             val daysLimit = currentTime.minus(Duration.of(days, ChronoUnit.DAYS))
 
-            Files.walk(getDebugOutputPath())
+            Files
+                .walk(getDebugOutputPath())
                 .filter {
                     val fileTime = Files.getAttribute(it, "basic:lastModifiedTime") as FileTime
                     val isOlderThanLimit = fileTime.toInstant().isBefore(daysLimit)
                     Files.isDirectory(it) && isOlderThanLimit
                 }
                 .sorted(Comparator.reverseOrder())
-                .forEach {
-                    Files.walk(it)
+                .forEach { dir ->
+                    Files.walk(dir)
                         .sorted(Comparator.reverseOrder())
-                        .forEach { Files.delete(it) }
+                        .forEach { file -> Files.delete(file) }
                 }
         } catch (e: Exception) {
             logger.warn("Failed to delete older files", e)
         }
     }
 
     private fun logSystemInfo() {
-        val appVersion = runCatching {
-            val props = Driver::class.java.classLoader.getResourceAsStream("version.properties").use {
-                Properties().apply { load(it) }
-            }
-            props["version"].toString()
-        }
-
         val logger = LoggerFactory.getLogger("MAESTRO")
         logger.info("---- System Info ----")
         logger.info("Maestro Version: ${EnvUtils.CLI_VERSION ?: "Undefined"}")
@@ -131,9 +129,9 @@ object TestDebugReporter {
     fun getDebugOutputPath(): Path {
         if (debugOutputPath != null) return debugOutputPath as Path
 
-        val debugRootPath = if(debugOutputPathAsString != null) debugOutputPathAsString!! else System.getProperty("user.home")        
+        val debugRootPath = if(debugOutputPathAsString != null) debugOutputPathAsString!! else System.getProperty("user.home")
         val debugOutput = if(flattenDebugOutput) Paths.get(debugRootPath) else buildDefaultDebugOutputPath(debugRootPath)
-        
+
         if (!debugOutput.exists()) {
             Files.createDirectories(debugOutput)
         }
@@ -150,8 +148,7 @@ object TestDebugReporter {
 }
 
 private data class CommandDebugWrapper(
-    val command: MaestroCommand,
-    val metadata: CommandDebugMetadata
+    val command: MaestroCommand, val metadata: CommandDebugMetadata
 )
 
 data class CommandDebugMetadata(
@@ -168,14 +165,23 @@ data class CommandDebugMetadata(
     }
 }
 
-data class ScreenshotDebugMetadata(
-    val screenshot: File,
-    val timestamp: Long,
-    val status: CommandStatus
+data class FlowDebugOutput(
+    val commands: IdentityHashMap<MaestroCommand, CommandDebugMetadata> = IdentityHashMap<MaestroCommand, CommandDebugMetadata>(),
+    val screenshots: MutableList<Screenshot> = mutableListOf(),
+    var exception: MaestroException? = null,
+) {
+    data class Screenshot(
+        val screenshot: File, val timestamp: Long, val status: CommandStatus
+    )
+}
+
+data class FlowAIOutput(
+    @JsonProperty("flow_name") var flowName: String,
+    @JsonProperty("flow_file_path") var flowFilePath: String,
+    val outputs: MutableList<AIOutput> = mutableListOf(),
 )
 
-data class FlowDebugMetadata(
-    val commands: IdentityHashMap<MaestroCommand, CommandDebugMetadata> = IdentityHashMap<MaestroCommand, CommandDebugMetadata>(),
-    val screenshots: MutableList<ScreenshotDebugMetadata> = mutableListOf(),
-    var exception: MaestroException? = null
+data class AIOutput(
+    @JsonProperty("screenshot_path") val screenshotPath: File,
+    val defect: List<Defect>,
 )
diff --git a/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt
@@ -23,8 +23,7 @@ import maestro.Maestro
 import maestro.MaestroException
 import maestro.cli.device.Device
 import maestro.cli.report.CommandDebugMetadata
-import maestro.cli.report.FlowDebugMetadata
-import maestro.cli.report.ScreenshotDebugMetadata
+import maestro.cli.report.FlowDebugOutput
 import maestro.cli.runner.resultview.ResultView
 import maestro.cli.runner.resultview.UiState
 import maestro.orchestra.ApplyConfigurationCommand
@@ -47,7 +46,7 @@ object MaestroCommandRunner {
         device: Device?,
         view: ResultView,
         commands: List<MaestroCommand>,
-        debug: FlowDebugMetadata
+        debug: FlowDebugOutput
     ): Result {
         val config = YamlCommandReader.getConfig(commands)
         val initFlow = config?.initFlow
@@ -74,7 +73,7 @@ object MaestroCommandRunner {
                     .also { it.deleteOnExit() } // save to another dir before exiting
                 maestro.takeScreenshot(out, false)
                 debugScreenshots.add(
-                    ScreenshotDebugMetadata(
+                    FlowDebugOutput.Screenshot(
                         screenshot = out,
                         timestamp = System.currentTimeMillis(),
                         status = status

diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt
@@ -8,29 +8,23 @@ import com.github.michaelbull.result.getOr
 import com.github.michaelbull.result.onFailure
 import maestro.Maestro
 import maestro.cli.device.Device
-import maestro.cli.report.FlowDebugMetadata
+import maestro.cli.report.FlowAIOutput
+import maestro.cli.report.FlowDebugOutput
 import maestro.cli.report.TestDebugReporter
 import maestro.cli.runner.resultview.AnsiResultView
 import maestro.cli.runner.resultview.ResultView
 import maestro.cli.runner.resultview.UiState
 import maestro.cli.util.PrintUtils
 import maestro.cli.view.ErrorViewUtils
-import maestro.debuglog.DebugLogStore
-import maestro.debuglog.LogConfig
 import maestro.orchestra.MaestroCommand
 import maestro.orchestra.MaestroInitFlow
 import maestro.orchestra.OrchestraAppState
 import maestro.orchestra.util.Env.withEnv
 import maestro.orchestra.yaml.YamlCommandReader
 import org.slf4j.LoggerFactory
 import java.io.File
-import java.nio.file.Files
 import java.nio.file.Path
-import java.nio.file.Paths
-import java.time.LocalDateTime
-import java.time.format.DateTimeFormatter
 import kotlin.concurrent.thread
-import kotlin.io.path.absolutePathString
 
 object TestRunner {
 
@@ -44,9 +38,8 @@ object TestRunner {
         resultView: ResultView,
         debugOutputPath: Path
     ): Int {
-
-        // debug
-        val debug = FlowDebugMetadata()
+        val debugOutput = FlowDebugOutput()
+        val aiOutput = FlowAIOutput()
 
         val result = runCatching(resultView, maestro) {
             val commands = YamlCommandReader.readCommands(flowFile.toPath())
@@ -56,12 +49,19 @@ object TestRunner {
                 device,
                 resultView,
                 commands,
-                debug
+                debugOutput,
+                aiOutput,
             )
         }
 
-        TestDebugReporter.saveFlow(flowFile.name, debug, debugOutputPath)
-        if (debug.exception != null) PrintUtils.err("${debug.exception?.message}")
+        TestDebugReporter.saveFlow(
+            flowName = flowFile.name,
+            debugOutput = debugOutput,
+            aiOutput = aiOutput,
+            path = debugOutputPath,
+        )
+
+        if (debugOutput.exception != null) PrintUtils.err("${debugOutput.exception?.message}")
 
         return if (result.get()?.flowSuccess == true) 0 else 1
     }
@@ -112,7 +112,7 @@ object TestRunner {
                                 device,
                                 resultView,
                                 commands,
-                                FlowDebugMetadata()
+                                FlowDebugOutput()
                             )
                         }.get()
                     }

diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt
@@ -17,14 +17,17 @@ import maestro.orchestra.util.Env.withEnv
 import maestro.orchestra.workspace.WorkspaceExecutionPlanner
 import maestro.orchestra.yaml.YamlCommandReader
 import okio.Sink
-import okio.sink
 import org.slf4j.LoggerFactory
 import java.io.File
 import java.nio.file.Path
-import kotlin.math.roundToLong
 import kotlin.system.measureTimeMillis
 import kotlin.time.Duration.Companion.seconds
 
+/**
+ * Similar to [TestRunner], but:
+ *  * can run many flows at once
+ *  * does not support continuous mode
+ */
 class TestSuiteInteractor(
     private val maestro: Maestro,
     private val device: Device? = null,
@@ -129,7 +132,7 @@ class TestSuiteInteractor(
         var errorMessage: String? = null
 
         // debug
-        val debug = FlowDebugMetadata()
+        val debug = FlowDebugOutput()
         val debugCommands = debug.commands
         val debugScreenshots = debug.screenshots
 
@@ -221,7 +224,7 @@ class TestSuiteInteractor(
         }
         val flowDuration = TimeUtils.durationInSeconds(flowTimeMillis)
 
-        TestDebugReporter.saveFlow(flowName, debug, debugOutputPath)
+        TestDebugReporter.saveFlow(flowName = flowName, persistentOutput = debug, path = debugOutputPath)
 
         TestSuiteStatusView.showFlowCompletion(
             TestSuiteViewModel.FlowResult(

diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt b/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt
@@ -241,7 +241,7 @@ class Orchestra(
     }
 
     private fun initAI(): AI? {
-        val apikey = System.getenv("OPENAI_TOKEN_COPILOT")
+        val apikey = System.getenv(AI.AI_KEY_ENV_VAR)
         return if (apikey != null) OpenAI(apiKey = apikey) else null
     }
 

diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/ai/AI.kt b/maestro-orchestra/src/main/java/maestro/orchestra/ai/AI.kt
@@ -26,6 +26,8 @@ abstract class AI : Closeable {
     ): CompletionData
 
     companion object {
+        const val AI_KEY_ENV_VAR = "MAESTRO_CLI_AI_KEY"
+
         // We use JSON mode/Structured Outputs to define the schema of the response we expect from the LLM.
         // * OpenAI: https://platform.openai.com/docs/guides/structured-outputs
         // * Gemini: https://ai.google.dev/gemini-api/docs/json-mode