From da8e844ec244dd8e792a5603294ad9b552fe5d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Takahashi?= Date: Mon, 13 Jan 2025 11:20:26 +0000 Subject: [PATCH 01/16] feat: Adds Insights Notification Allows users to disable insights notification based on the new Environment Variable --- maestro-cli/src/main/java/maestro/cli/App.kt | 3 ++ .../java/maestro/cli/insights/Insights.kt | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 maestro-cli/src/main/java/maestro/cli/insights/Insights.kt diff --git a/maestro-cli/src/main/java/maestro/cli/App.kt b/maestro-cli/src/main/java/maestro/cli/App.kt index 915eaf57ac..056b5d974a 100644 --- a/maestro-cli/src/main/java/maestro/cli/App.kt +++ b/maestro-cli/src/main/java/maestro/cli/App.kt @@ -33,6 +33,7 @@ import maestro.cli.command.StartDeviceCommand import maestro.cli.command.StudioCommand import maestro.cli.command.TestCommand import maestro.cli.command.UploadCommand +import maestro.cli.insights.Insights import maestro.cli.update.Updates import maestro.cli.util.ChangeLogUtils import maestro.cli.util.ErrorReporter @@ -144,6 +145,8 @@ fun main(args: Array) { DebugLogStore.finalizeRun() + Insights.maybeNotifyInsights() + val newVersion = Updates.checkForUpdates() if (newVersion != null) { Updates.fetchChangelogAsync() diff --git a/maestro-cli/src/main/java/maestro/cli/insights/Insights.kt b/maestro-cli/src/main/java/maestro/cli/insights/Insights.kt new file mode 100644 index 0000000000..cd72bc1483 --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/insights/Insights.kt @@ -0,0 +1,31 @@ +package maestro.cli.insights + +import maestro.cli.view.box + +/** + * The Insights helper for Maestro CLI. + * - Uses MAESTRO_CLI_INSIGHTS_NOTIFICATION_DISABLED env var to disable insights notifications + */ +object Insights { + + private const val DISABLE_INSIGHTS_ENV_VAR = "MAESTRO_CLI_INSIGHTS_NOTIFICATION_DISABLED" + private val disabled: Boolean + get() = System.getenv(DISABLE_INSIGHTS_ENV_VAR) == "true" + + fun maybeNotifyInsights() { + if (disabled) return + + println() + println( + listOf( + "Tryout our new Analyze with Ai feature.\n", + "See what's new:", + // TODO: Add final link to analyze with Ai Docs + "https://github.com/mobile-dev-inc/maestro/blob/main/CHANGELOG.md#blaaa", + "Analyze command:", + "maestro analyze android-flow.yaml | bash\n", + "To disable this notification, set $DISABLE_INSIGHTS_ENV_VAR environment variable to \"true\" before running Maestro." + ).joinToString("\n").box() + ) + } +} From 9f07924afd76f4031f67e231bbc60e7ca4b700e0 Mon Sep 17 00:00:00 2001 From: Leland Takamine Date: Tue, 14 Jan 2025 06:08:58 -0800 Subject: [PATCH 02/16] Scroll local video rendering to follow currently executing command (#2232) --- .../maestro/cli/graphics/SkiaFrameRenderer.kt | 38 +++++------ .../maestro/cli/graphics/SkiaTextClipper.kt | 64 +++++++++++++++++++ 2 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 maestro-cli/src/main/java/maestro/cli/graphics/SkiaTextClipper.kt diff --git a/maestro-cli/src/main/java/maestro/cli/graphics/SkiaFrameRenderer.kt b/maestro-cli/src/main/java/maestro/cli/graphics/SkiaFrameRenderer.kt index 2a150aa50a..b358a3a5e0 100644 --- a/maestro-cli/src/main/java/maestro/cli/graphics/SkiaFrameRenderer.kt +++ b/maestro-cli/src/main/java/maestro/cli/graphics/SkiaFrameRenderer.kt @@ -3,14 +3,9 @@ package maestro.cli.graphics import org.jetbrains.skia.Canvas import org.jetbrains.skia.Color import org.jetbrains.skia.Font -import org.jetbrains.skia.FontMgr import org.jetbrains.skia.Paint import org.jetbrains.skia.Rect import org.jetbrains.skia.Surface -import org.jetbrains.skia.paragraph.FontCollection -import org.jetbrains.skia.paragraph.ParagraphBuilder -import org.jetbrains.skia.paragraph.ParagraphStyle -import org.jetbrains.skia.paragraph.TextStyle import org.jetbrains.skiko.toImage import java.awt.image.BufferedImage import javax.imageio.ImageIO @@ -42,13 +37,10 @@ class SkiaFrameRenderer : FrameRenderer { private val footerText = "maestro.mobile.dev" private val terminalBgColor = Color.makeARGB(220, 0, 0, 0) - private val terminalTextStyle = TextStyle().apply { - fontFamilies = SkiaFonts.MONOSPACE_FONT_FAMILIES.toTypedArray() - fontSize = 24f - color = Color.WHITE - } private val terminalContentPadding = 40f + private val textClipper = SkiaTextClipper() + override fun render( outputWidthPx: Int, outputHeightPx: Int, @@ -154,14 +146,24 @@ class SkiaFrameRenderer : FrameRenderer { val contentRect = Rect.makeLTRB(terminalRect.left, headerRect.bottom, terminalRect.right, footerRect.top) canvas.drawRect(contentRect, Paint().apply { color = terminalBgColor }) - val paddedContentRect = contentRect.inflate(-terminalContentPadding) + val paddedContentRect = Rect.makeLTRB( + l = contentRect.left + terminalContentPadding, + t = contentRect.top + terminalContentPadding, + r = contentRect.right - terminalContentPadding, + b = contentRect.bottom - terminalContentPadding / 4f, + ) + + val focusedLineIndex = getFocusedLineIndex(string) + val focusedLinePadding = 5 + textClipper.renderClippedText(canvas, paddedContentRect, string, focusedLineIndex + focusedLinePadding) + } - val fontCollection = FontCollection().setDefaultFontManager(FontMgr.default) - val p = ParagraphBuilder(ParagraphStyle(), fontCollection) - .pushStyle(terminalTextStyle) - .addText(string) - .build() - p.layout(paddedContentRect.width) - p.paint(canvas, paddedContentRect.left, paddedContentRect.top) + private fun getFocusedLineIndex(text: String): Int { + val lines = text.lines() + val indexOfFirstPendingLine = lines.indexOfFirst { it.contains("\uD83D\uDD32") } + if (indexOfFirstPendingLine != -1) return indexOfFirstPendingLine + val indexOfLastCheck = lines.indexOfLast { it.contains("✅") } + if (indexOfLastCheck != -1) return indexOfLastCheck + return 0 } } diff --git a/maestro-cli/src/main/java/maestro/cli/graphics/SkiaTextClipper.kt b/maestro-cli/src/main/java/maestro/cli/graphics/SkiaTextClipper.kt new file mode 100644 index 0000000000..e23795a9b3 --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/graphics/SkiaTextClipper.kt @@ -0,0 +1,64 @@ +package maestro.cli.graphics + +import org.jetbrains.skia.Canvas +import org.jetbrains.skia.Color +import org.jetbrains.skia.FontMgr +import org.jetbrains.skia.Rect +import org.jetbrains.skia.paragraph.FontCollection +import org.jetbrains.skia.paragraph.Paragraph +import org.jetbrains.skia.paragraph.ParagraphBuilder +import org.jetbrains.skia.paragraph.ParagraphStyle +import org.jetbrains.skia.paragraph.RectHeightMode +import org.jetbrains.skia.paragraph.RectWidthMode +import org.jetbrains.skia.paragraph.TextStyle +import kotlin.math.min + +class SkiaTextClipper { + + private val terminalTextStyle = TextStyle().apply { + fontFamilies = SkiaFonts.MONOSPACE_FONT_FAMILIES.toTypedArray() + fontSize = 24f + color = Color.WHITE + } + + fun renderClippedText(canvas: Canvas, rect: Rect, text: String, focusedLine: Int) { + val p = createParagraph(text, rect.width) + val focusedLineRange = getRangeForLine(text, focusedLine) + val focusedLineBottom = p.getRectsForRange( + start = focusedLineRange.first, + end = focusedLineRange.second, + rectHeightMode = RectHeightMode.MAX, + rectWidthMode = RectWidthMode.MAX + ).maxOf { it.rect.bottom } + val offsetY = min(0f, rect.height - focusedLineBottom) + canvas.save() + canvas.clipRect(rect) + p.paint(canvas, rect.left, rect.top + offsetY) + canvas.restore() + } + + private fun getRangeForLine(text: String, lineIndex: Int): Pair { + var start = 0 + var end = 0 + var currentLine = 0 + while (currentLine <= lineIndex) { + start = end + end = text.indexOf('\n', start + 1) + if (end == -1) { + end = text.length + break + } + currentLine++ + } + return Pair(start, end) + } + + private fun createParagraph(text: String, width: Float): Paragraph { + val fontCollection = FontCollection().setDefaultFontManager(FontMgr.default) + return ParagraphBuilder(ParagraphStyle(), fontCollection) + .pushStyle(terminalTextStyle) + .addText(text) + .build() + .apply { layout(width) } + } +} From 5deb64a278007628f9e8da29a333b4f2f13625d6 Mon Sep 17 00:00:00 2001 From: Leland Takamine Date: Wed, 15 Jan 2025 15:13:42 -0800 Subject: [PATCH 03/16] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0281bf4159..165d41ad54 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Maestro -Maestro is the easiest way to automate UI testing for your mobile app. +Maestro is the easiest way to UI testing for your mobile or web app. From 97e83a5a29102c5496c46b36bf6bfa76f1166849 Mon Sep 17 00:00:00 2001 From: Leland Takamine Date: Thu, 16 Jan 2025 07:56:43 -0800 Subject: [PATCH 04/16] Web fixes (#2250) * Select web when platform is specified * Do not log errors on device screen endpoint since this floods the terminal with unnecessary noise when killing studio * Add url input in studio for web * Handle openLink without scheme * Remove comment * Revert unnecessary change * Remove unused import --- .../cli/session/MaestroSessionManager.kt | 2 +- .../main/java/maestro/drivers/WebDriver.kt | 9 ++- .../main/java/maestro/studio/DeviceService.kt | 7 ++- .../src/main/java/maestro/studio/Models.kt | 3 + .../BrowserActionBar.tsx | 58 +++++++++++++++++++ .../interact/InteractPageLayout.tsx | 21 ++++++- maestro-studio/web/src/helpers/models.ts | 2 + .../storybook/BrowserActionBar.stories.tsx | 41 +++++++++++++ 8 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx create mode 100644 maestro-studio/web/src/storybook/BrowserActionBar.stories.tsx diff --git a/maestro-cli/src/main/java/maestro/cli/session/MaestroSessionManager.kt b/maestro-cli/src/main/java/maestro/cli/session/MaestroSessionManager.kt index 0a68cf3948..932305fa1f 100644 --- a/maestro-cli/src/main/java/maestro/cli/session/MaestroSessionManager.kt +++ b/maestro-cli/src/main/java/maestro/cli/session/MaestroSessionManager.kt @@ -112,7 +112,7 @@ object MaestroSessionManager { deviceId: String?, platform: Platform? = null, ): SelectedDevice { - if (deviceId == "chromium") { + if (deviceId == "chromium" || platform == Platform.WEB) { return SelectedDevice( platform = Platform.WEB ) diff --git a/maestro-client/src/main/java/maestro/drivers/WebDriver.kt b/maestro-client/src/main/java/maestro/drivers/WebDriver.kt index cecc2be782..c5f37d77bd 100644 --- a/maestro-client/src/main/java/maestro/drivers/WebDriver.kt +++ b/maestro-client/src/main/java/maestro/drivers/WebDriver.kt @@ -221,8 +221,11 @@ class WebDriver( return TreeNode(attributes = attributes, children = children.map { parse(it) }) } - - return parse(contentDesc as Map) + val root = parse(contentDesc as Map) + seleniumDriver?.currentUrl?.let { url -> + root.attributes["url"] = url + } + return root } private fun detectWindowChange() { @@ -367,7 +370,7 @@ class WebDriver( override fun openLink(link: String, appId: String?, autoVerify: Boolean, browser: Boolean) { val driver = ensureOpen() - driver.get(link) + driver.get(if (link.startsWith("http")) link else "https://$link") } override fun hideKeyboard() { diff --git a/maestro-studio/server/src/main/java/maestro/studio/DeviceService.kt b/maestro-studio/server/src/main/java/maestro/studio/DeviceService.kt index 298958d9e4..00686ee1c5 100644 --- a/maestro-studio/server/src/main/java/maestro/studio/DeviceService.kt +++ b/maestro-studio/server/src/main/java/maestro/studio/DeviceService.kt @@ -83,9 +83,9 @@ object DeviceService { val deviceScreen = getDeviceScreen(maestro) writeStringUtf8("data: $deviceScreen\n\n") flush() - } catch (e: Exception) { + } catch (_: Exception) { // Ignoring the exception to prevent SSE stream from dying - e.printStackTrace() + // Don't log since this floods the terminal after killing studio } } } @@ -187,8 +187,9 @@ object DeviceService { val deviceWidth = deviceInfo.widthGrid val deviceHeight = deviceInfo.heightGrid + val url = tree.attributes["url"] val elements = treeToElements(tree) - val deviceScreen = DeviceScreen("/screenshot/${screenshotFile.name}", deviceWidth, deviceHeight, elements) + val deviceScreen = DeviceScreen(deviceInfo.platform, "/screenshot/${screenshotFile.name}", deviceWidth, deviceHeight, elements, url) return jacksonObjectMapper() .setSerializationInclusion(JsonInclude.Include.NON_NULL) .writeValueAsString(deviceScreen) diff --git a/maestro-studio/server/src/main/java/maestro/studio/Models.kt b/maestro-studio/server/src/main/java/maestro/studio/Models.kt index 9f744c62ff..f2d61f6e83 100644 --- a/maestro-studio/server/src/main/java/maestro/studio/Models.kt +++ b/maestro-studio/server/src/main/java/maestro/studio/Models.kt @@ -1,13 +1,16 @@ package maestro.studio import com.fasterxml.jackson.annotation.JsonProperty +import maestro.Platform import java.util.UUID data class DeviceScreen( + val platform: Platform, val screenshot: String, val width: Int, val height: Int, val elements: List, + val url: String?, ) data class UIElementBounds( diff --git a/maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx b/maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx new file mode 100644 index 0000000000..753af2cab8 --- /dev/null +++ b/maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx @@ -0,0 +1,58 @@ +import React, {useLayoutEffect} from "react"; +import {twMerge} from "tailwind-merge"; +import {Input} from "../design-system/input"; + +const GlobeIcon = ({ className }: { className?: string }) => ( + + + +); + +const BrowserActionBar = ({currentUrl, onUrlUpdated, isLoading}: { + currentUrl?: string, + onUrlUpdated: (url: string) => void, + isLoading: boolean +}) => { + const [isEditing, setIsEditing] = React.useState(false) + const [editedUrl, setEditedUrl] = React.useState(currentUrl) + useLayoutEffect(() => { + if (!isEditing && !isLoading) { + setEditedUrl(currentUrl) + } + }, [isLoading, isEditing, currentUrl]); + return ( +
+
+ +
+ setEditedUrl(e.target.value)} + onFocus={() => setIsEditing(true)} + onBlur={() => setIsEditing(false)} + onKeyDown={(e) => { + if (e.key === 'Enter' && isEditing && editedUrl) { + onUrlUpdated(editedUrl); + e.currentTarget.blur(); + setIsEditing(false); + } + }} + /> +
+ ) +} + +export default BrowserActionBar \ No newline at end of file diff --git a/maestro-studio/web/src/components/interact/InteractPageLayout.tsx b/maestro-studio/web/src/components/interact/InteractPageLayout.tsx index 76f5d37601..d26e203a6f 100644 --- a/maestro-studio/web/src/components/interact/InteractPageLayout.tsx +++ b/maestro-studio/web/src/components/interact/InteractPageLayout.tsx @@ -1,4 +1,4 @@ -import { useState } from "react"; +import React, { useState } from "react"; import clsx from "clsx"; import InteractableDevice from "../device-and-device-elements/InteractableDevice"; @@ -12,6 +12,7 @@ import { useDeviceContext } from "../../context/DeviceContext"; import { Spinner } from "../design-system/spinner"; import { useRepl } from '../../context/ReplContext'; import { DeviceScreen } from "../../helpers/models"; +import BrowserActionBar from "../device-and-device-elements/BrowserActionBar"; const InteractPageLayout = () => { const { @@ -24,6 +25,7 @@ const InteractPageLayout = () => { const { runCommandYaml } = useRepl(); const [showElementsPanel, setShowElementsPanel] = useState(false); + const [isUrlLoading, setIsUrlLoading] = useState(false); const onEdit = (example: CommandExample) => { if (example.status === "unavailable") return; @@ -44,6 +46,16 @@ const InteractPageLayout = () => { await runCommandYaml(example.content); }; + const onUrlUpdated = (url: string) => { + setIsUrlLoading(true); + runCommandYaml(`openLink: ${url}`).finally(() => { + // Wait some time to update the url from the device screen + setTimeout(() => { + setIsUrlLoading(false); + }, 1000); + }); + } + if (isLoading) return (
@@ -76,6 +88,13 @@ const InteractPageLayout = () => { Search Elements with Text or Id )} + {deviceScreen?.platform === 'WEB' && ( + + )} diff --git a/maestro-studio/web/src/helpers/models.ts b/maestro-studio/web/src/helpers/models.ts index 7b84169560..3e6d2b36ec 100644 --- a/maestro-studio/web/src/helpers/models.ts +++ b/maestro-studio/web/src/helpers/models.ts @@ -26,10 +26,12 @@ export type UIElement = { }; export type DeviceScreen = { + platform: string; screenshot: string; width: number; height: number; elements: UIElement[]; + url?: string; }; export type ReplCommandStatus = diff --git a/maestro-studio/web/src/storybook/BrowserActionBar.stories.tsx b/maestro-studio/web/src/storybook/BrowserActionBar.stories.tsx new file mode 100644 index 0000000000..d1441b9d2f --- /dev/null +++ b/maestro-studio/web/src/storybook/BrowserActionBar.stories.tsx @@ -0,0 +1,41 @@ +import BrowserActionBar from "../components/device-and-device-elements/BrowserActionBar"; +import {useEffect, useState} from "react"; + +export default { + title: "BrowserActionBar", +}; + +export const Main = () => { + const [currentUrl, setCurrentUrl] = useState("https://google.com"); + const [isLoading, setIsLoading] = useState(false); + useEffect(() => { + const interval = setInterval(() => { + setCurrentUrl(old => { + // if no scheme, add https + if (!old.startsWith("http")) { + old = "https://" + old; + } + const url = new URL(old); + url.searchParams.set("ts", Date.now().toString()); + return url.toString(); + }); + }, 1000); + return () => clearInterval(interval); + }, []); + const onUrlUpdated = (url: string) => { + setIsLoading(true); + setTimeout(() => { + setCurrentUrl(url); + setIsLoading(false); + }, 1000); + } + return ( +
+ +
+ ); +}; From 84acfe48f7345a9243e6f749f9fa551927f7b96d Mon Sep 17 00:00:00 2001 From: Leland Takamine Date: Thu, 16 Jan 2025 11:58:47 -0800 Subject: [PATCH 05/16] Enable running Maestro on Windows without WSL (#2248) * Fix Windows script error by making classpath shorting using globs * Update dadb * Do not use emojis on Windows * Fix Studio npm builds on Windows * Make OS comparison case-insensitive * Use smiley face instead of check * Use + instead of special char * Remove debug condition --------- Co-authored-by: Dan Caseley --- gradle/libs.versions.toml | 2 +- maestro-cli/build.gradle.kts | 4 +++ .../java/maestro/cli/command/TestCommand.kt | 8 +++-- .../java/maestro/cli/runner/TestRunner.kt | 3 +- .../cli/runner/resultview/AnsiResultView.kt | 34 +++++++++++++------ .../main/java/maestro/cli/util/EnvUtils.kt | 4 +++ maestro-studio/web/build.gradle | 12 +++++-- 7 files changed, 50 insertions(+), 17 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5952122c04..d8ae45667d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,7 @@ axml = "2.1.2" commons-codec = "1.17.0" commons-lang3 = "3.13.0" # 3.14.0 causes weird crashes during dexing commons-io = "2.16.1" -dadb = "1.2.7" +dadb = "1.2.9" detekt = "1.19.0" googleFindbugs = "3.0.2" googleGson = "2.11.0" diff --git a/maestro-cli/build.gradle.kts b/maestro-cli/build.gradle.kts index a0e27a186a..476892d7a0 100644 --- a/maestro-cli/build.gradle.kts +++ b/maestro-cli/build.gradle.kts @@ -31,6 +31,10 @@ tasks.named("run") { workingDir = rootDir } +tasks.named("startScripts") { + classpath = files("$buildDir/libs/*") +} + dependencies { implementation(project(path = ":maestro-utils")) annotationProcessor(libs.picocli.codegen) diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index 717732de84..58dfcb23ea 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -40,6 +40,7 @@ import maestro.cli.runner.TestSuiteInteractor import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.runner.resultview.PlainTextResultView import maestro.cli.session.MaestroSessionManager +import maestro.cli.util.EnvUtils import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.PrintUtils import maestro.cli.view.box @@ -353,8 +354,11 @@ class TestCommand : Callable { debugOutputPath: Path, ): Triple { val resultView = - if (DisableAnsiMixin.ansiEnabled) AnsiResultView() - else PlainTextResultView() + if (DisableAnsiMixin.ansiEnabled) { + AnsiResultView(useEmojis = !EnvUtils.isWindows()) + } else { + PlainTextResultView() + } env = env .withInjectedShellEnvVars() diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt index 4fd0c6aa21..3bfee3e381 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt @@ -14,6 +14,7 @@ import maestro.cli.report.TestDebugReporter import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.runner.resultview.ResultView import maestro.cli.runner.resultview.UiState +import maestro.cli.util.EnvUtils import maestro.cli.util.PrintUtils import maestro.cli.view.ErrorViewUtils import maestro.orchestra.MaestroCommand @@ -94,7 +95,7 @@ object TestRunner { flowFile: File, env: Map, ): Nothing { - val resultView = AnsiResultView("> Press [ENTER] to restart the Flow\n\n") + val resultView = AnsiResultView("> Press [ENTER] to restart the Flow\n\n", useEmojis = !EnvUtils.isWindows()) val fileWatcher = FileWatcher() diff --git a/maestro-cli/src/main/java/maestro/cli/runner/resultview/AnsiResultView.kt b/maestro-cli/src/main/java/maestro/cli/runner/resultview/AnsiResultView.kt index a4adfc1bf0..5c0e3d165f 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/resultview/AnsiResultView.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/resultview/AnsiResultView.kt @@ -37,6 +37,7 @@ import org.fusesource.jansi.Ansi class AnsiResultView( private val prompt: String? = null, private val printCommandLogs: Boolean = true, + private val useEmojis: Boolean = true, ) : ResultView { private val startTimestamp = System.currentTimeMillis() @@ -248,18 +249,29 @@ class AnsiResultView( return Frame(System.currentTimeMillis() - startTimestamp, content) } - data class Frame(val timestamp: Long, val content: String) -} - -internal fun status(status: CommandStatus): String { - return when (status) { - CommandStatus.COMPLETED -> "✅ " - CommandStatus.FAILED -> "❌ " - CommandStatus.RUNNING -> "⏳ " - CommandStatus.PENDING -> "\uD83D\uDD32 " // 🔲 - CommandStatus.WARNED -> "⚠️ " - CommandStatus.SKIPPED -> "⚪️ " + private fun status(status: CommandStatus): String { + if (useEmojis) { + return when (status) { + CommandStatus.COMPLETED -> "✅ " + CommandStatus.FAILED -> "❌ " + CommandStatus.RUNNING -> "⏳ " + CommandStatus.PENDING -> "\uD83D\uDD32 " // 🔲 + CommandStatus.WARNED -> "⚠️ " + CommandStatus.SKIPPED -> "⚪️ " + } + } else { + return when (status) { + CommandStatus.COMPLETED -> "+ " + CommandStatus.FAILED -> "X " + CommandStatus.RUNNING -> "> " + CommandStatus.PENDING -> " " + CommandStatus.WARNED -> "! " + CommandStatus.SKIPPED -> "- " + } + } } + + data class Frame(val timestamp: Long, val content: String) } // Helper launcher to play around with presentation diff --git a/maestro-cli/src/main/java/maestro/cli/util/EnvUtils.kt b/maestro-cli/src/main/java/maestro/cli/util/EnvUtils.kt index d305942ce1..cd5883aa5e 100644 --- a/maestro-cli/src/main/java/maestro/cli/util/EnvUtils.kt +++ b/maestro-cli/src/main/java/maestro/cli/util/EnvUtils.kt @@ -75,6 +75,10 @@ object EnvUtils { return false } + fun isWindows(): Boolean { + return OS_NAME.lowercase().startsWith("windows") + } + /** * Returns major version of Java, e.g. 8, 11, 17, 21. */ diff --git a/maestro-studio/web/build.gradle b/maestro-studio/web/build.gradle index 64004225a3..a8fbdee673 100644 --- a/maestro-studio/web/build.gradle +++ b/maestro-studio/web/build.gradle @@ -1,7 +1,11 @@ tasks.register("deps", Exec.class) { inputs.file(layout.projectDirectory.file("package.json")) outputs.dir(layout.projectDirectory.dir("node_modules")) - commandLine("npm", "install") + if (System.getProperty('os.name').toLowerCase(Locale.ROOT).contains('windows')) { + commandLine 'npm.cmd', 'install' + } else { + commandLine 'npm', 'install' + } } tasks.register("build", Exec.class) { @@ -11,5 +15,9 @@ tasks.register("build", Exec.class) { inputs.files(inputFiles) outputs.dir(layout.projectDirectory.dir("build")) dependsOn(tasks.deps) - commandLine("npm", "run", "build") + if (System.getProperty('os.name').toLowerCase(Locale.ROOT).contains('windows')) { + commandLine 'npm.cmd', 'run', 'build' + } else { + commandLine("npm", "run", "build") + } } From 899f83df10ef72c3039b44c0de43a32d5641b41c Mon Sep 17 00:00:00 2001 From: Dan Caseley Date: Fri, 17 Jan 2025 07:09:20 +0000 Subject: [PATCH 06/16] Add console.log messages directly to the maestro log (#2249) --- maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt b/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt index b2aba45244..6dbad5a052 100644 --- a/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt +++ b/maestro-orchestra/src/main/java/maestro/orchestra/Orchestra.kt @@ -172,6 +172,7 @@ class Orchestra( command, metadata.copy(logMessages = metadata.logMessages + msg) ) + logger.info("JsConsole: $msg") } val evaluatedCommand = command.evaluateScripts(jsEngine) From 18962b07e4038c74a5d3c51225fd4800fe6c32ac Mon Sep 17 00:00:00 2001 From: Dmitry Zaytsev Date: Sat, 18 Jan 2025 13:40:04 +0100 Subject: [PATCH 07/16] Fix: dark mode for browser action bar (#2255) --- .../components/device-and-device-elements/BrowserActionBar.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx b/maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx index 753af2cab8..43b3434707 100644 --- a/maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx +++ b/maestro-studio/web/src/components/device-and-device-elements/BrowserActionBar.tsx @@ -34,7 +34,7 @@ const BrowserActionBar = ({currentUrl, onUrlUpdated, isLoading}: {
Date: Mon, 20 Jan 2025 12:13:21 +0530 Subject: [PATCH 08/16] Prepare release 1.39.9 (#2245) --- CHANGELOG.md | 10 ++++++++++ maestro-cli/gradle.properties | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9852947401..0fcdc5b8eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +## 1.39.9 + +- Revert: Error in showing keyboard during input and erase commands on iOS +- Fix: applesimutils affecting granting location permission +- Fix: Setting host and port from the optional arguments +- Feature: New `maestro login` command for logging in Robin. +- Feature: Improved `maestro record` video to scroll and follow the currently executing commands +- Fix: Enable running Maestro on Windows without WSL +- Feature: Add console.log messages directly to the maestro log file. + ## 1.39.8 - Fix: Debug message not showing up when we execute commands on maestro cli anymore diff --git a/maestro-cli/gradle.properties b/maestro-cli/gradle.properties index 1f1d4ca3ed..521707981f 100644 --- a/maestro-cli/gradle.properties +++ b/maestro-cli/gradle.properties @@ -1 +1 @@ -CLI_VERSION=1.39.8 \ No newline at end of file +CLI_VERSION=1.39.9 \ No newline at end of file From d82a19675de8db3ccaf89503900feb822a0b9c9c Mon Sep 17 00:00:00 2001 From: Dan Caseley Date: Mon, 20 Jan 2025 19:52:04 +0000 Subject: [PATCH 09/16] Improved install script (#2263) * Remove unused variable in install script * Remove previous files when installing maestro-cli --- scripts/install.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index a7f21069df..f8e652a7e7 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -62,7 +62,6 @@ export MAESTRO_DIR # Local variables maestro_tmp_folder="${MAESTRO_DIR}/tmp" maestro_bash_profile="${HOME}/.bash_profile" -maestro_profile="${HOME}/.profile" maestro_bashrc="${HOME}/.bashrc" maestro_zshrc="${ZDOTDIR:-${HOME}}/.zshrc" @@ -114,6 +113,15 @@ if [[ "$cygwin" == 'true' ]]; then fi unzip -qo "$maestro_zip_file" -d "$maestro_tmp_folder" +# Empty destinations +echo "* Remove previous installation (if any)" +if [[ -d "$MAESTRO_DIR/lib" ]]; then + rm -rf "${MAESTRO_DIR:?}/lib" +fi +if [[ -d "$MAESTRO_DIR/bin" ]]; then + rm -rf "${MAESTRO_DIR:?}/bin" +fi + # Copy in place echo "* Copying archive contents..." cp -rf "${maestro_tmp_folder}"/maestro/* "$MAESTRO_DIR" From 25fc003019357f756d1f95b94ecc746cacf23c4b Mon Sep 17 00:00:00 2001 From: Dan Caseley Date: Mon, 20 Jan 2025 20:24:35 +0000 Subject: [PATCH 10/16] Prepare for release 1.39.10 (#2264) --- CHANGELOG.md | 4 ++++ gradle.properties | 2 +- maestro-cli/gradle.properties | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fcdc5b8eb..fe3ea46c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.39.10 + +- Update install script to tidy up old installation binaries + ## 1.39.9 - Revert: Error in showing keyboard during input and erase commands on iOS diff --git a/gradle.properties b/gradle.properties index 26070bcfaf..bedd3a45b6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,7 +2,7 @@ android.useAndroidX=true android.enableJetifier=true kotlin.code.style=official GROUP=dev.mobile -VERSION_NAME=1.39.8 +VERSION_NAME=1.39.10 POM_DESCRIPTION=Maestro is a server-driven platform-agnostic library that allows to drive tests for both iOS and Android using the same implementation through an intuitive API. POM_URL=https://github.com/mobile-dev-inc/maestro POM_SCM_URL=https://github.com/mobile-dev-inc/maestro diff --git a/maestro-cli/gradle.properties b/maestro-cli/gradle.properties index 521707981f..d4f1ffcd5f 100644 --- a/maestro-cli/gradle.properties +++ b/maestro-cli/gradle.properties @@ -1 +1 @@ -CLI_VERSION=1.39.9 \ No newline at end of file +CLI_VERSION=1.39.10 \ No newline at end of file From adb27689f37dc5b3ef21a2a6fcc489d4f72d3300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Takahashi?= Date: Sun, 19 Jan 2025 17:21:15 +0000 Subject: [PATCH 11/16] feat(analyze): Adds initial Analyze option to test command locally --- .../src/main/java/maestro/ai/Prediction.kt | 125 +++++++++++- .../main/resources/askForInsights_schema.json | 32 +++ .../java/maestro/cli/command/TestCommand.kt | 31 ++- .../report/HtmlInsightsAnalysisReporter.kt | 184 ++++++++++++++++++ .../cli/runner/MaestroCommandRunner.kt | 8 + .../java/maestro/cli/runner/TestRunner.kt | 6 +- .../java/maestro/cli/util/ScreenshotUtils.kt | 19 ++ .../maestro/cli/util/TestAnalysisReporter.kt | 97 +++++++++ 8 files changed, 499 insertions(+), 3 deletions(-) create mode 100644 maestro-ai/src/main/resources/askForInsights_schema.json create mode 100644 maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt create mode 100644 maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt diff --git a/maestro-ai/src/main/java/maestro/ai/Prediction.kt b/maestro-ai/src/main/java/maestro/ai/Prediction.kt index 332c6bb79d..11385160c2 100644 --- a/maestro-ai/src/main/java/maestro/ai/Prediction.kt +++ b/maestro-ai/src/main/java/maestro/ai/Prediction.kt @@ -4,6 +4,13 @@ import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import kotlinx.serialization.json.jsonObject import maestro.ai.openai.OpenAI +import java.nio.file.Path + +data class FlowFiles( + val jsonFiles: List>, + val imageFiles: List>, + val textFiles: List> +) @Serializable data class Defect( @@ -21,12 +28,27 @@ private data class ExtractTextResponse( val text: String? ) +@Serializable +data class Insight( + val category: String, + val reasoning: String, +) + +@Serializable +private data class AskForInsightsResponse( + val insights: List, +) + object Prediction { private val askForDefectsSchema by lazy { readSchema("askForDefects") } + private val askForInsightsSchema by lazy { + readSchema("askForInsights") + } + private val extractTextSchema by lazy { readSchema("extractText") } @@ -51,6 +73,12 @@ object Prediction { "layout" to "Some UI elements are overlapping or are cropped", ) + private val insightsCategories = listOf( + "visual" to "Insights related to UI elements that are overlapping or cropped", + "text" to " insights on Grammar and spelling, like suggestions and optimizations on the text on the page, or Inconsistent use of language, for example, mixed English and Portuguese", + "maestro" to "Insights on the maestro testing tool usage, best practices, tips on debugging, optimizing the workspace, or how to make the best usage of maestro commands, APIs, or extra features like maestro studio, or Robin cloud", + ) + private val allDefectCategories = defectCategories + listOf("assertion" to "The assertion is not true") suspend fun findDefects( @@ -144,6 +172,101 @@ object Prediction { return defects.defects } + suspend fun generateInsights( + aiClient: AI, + flowFiles: List, + printRawResponse: Boolean = false, + ): List { + val prompt = buildString { + appendLine( + """ + You are a QA engineer performing quality assurance for a mobile application. + Identify any defects in the provided screenshots and optionally + + You are using Maestro for e2e mobile testing, understand the tool API and best practices on how to use it based on its Docs + You are given screenshots of the application and the JSON and text files artifacts from the debug artifacts of maestro e2e testing tool. + + Given the following maestro flows + """.trimIndent() + ) + + flowFiles.forEach { + appendLine( + """ + You are going to transcribe the screenshots and analyze every file below: + ${if (it.jsonFiles.isNotEmpty()) "Based on this JSON files: ${it.jsonFiles.joinToString("\n",transform = { (content) -> String(content) })}\n" else ""} + ${if (it.textFiles.isNotEmpty()) "Based on this files: ${it.textFiles.joinToString("\n",transform = { (content) -> String(content) })}\n" else ""} + """.trimIndent() + ) + } + + append( + """ + | + |RULES: + | + |Your task is to generate Insights following the RULES: + |* You must explain understand each context based on the provided data analyzsing each flow. + |* + |* All Insights you find must belong to one of the following categories: + |${insightsCategories.joinToString(separator = "\n") { " * ${it.first}: ${it.second}" }} + |* If you see Insights, your response MUST only include defect name and detailed reasoning for each defect. + |* Provide response as a list of JSON objects, each representing : + |* Do not repeat the context text into the insights, make it useful for the QA developer reading the insights. + |* Do not generate duplicated or similar insights just changing the category. + |* Do not generate spam insights that are too obvious based on the screenshot. + |* Do not raise false positives. Some example responses that have a high chance of being a false positive: + | * button is partially cropped at the bottom + | * button is not aligned horizontally/vertically within its container + | * element not found because it does not exist on the current screen + | * ensure that the app is in the correct state before looking for the text + """.trimMargin("|") + ) + + append( + """ + | + |* You must provide result as a valid JSON object, matching this structure: + | + | { + | "insights": [ + | { + | "category": "", + | "reasoning": "" + | }, + | { + | "category": "", + | "reasoning": "" + | } + | ] + | } + | + |DO NOT output any other information in the JSON object. + """.trimMargin("|") + ) + } + + val aiResponse = aiClient.chatCompletion( + prompt, + model = aiClient.defaultModel, + maxTokens = 4096, + identifier = "find-defects", + imageDetail = "high", + images = flowFiles.flatMap { it.imageFiles.map { (content) -> content } }, + jsonSchema = if (aiClient is OpenAI) json.parseToJsonElement(askForInsightsSchema).jsonObject else null, + ) + + if (printRawResponse) { + println("--- RAW RESPONSE START ---") + println(aiResponse.response) + println("--- RAW RESPONSE END ---") + } + + val insights = json.decodeFromString(aiResponse.response) + + return insights.insights; + } + suspend fun performAssertion( aiClient: AI, screen: ByteArray, @@ -270,4 +393,4 @@ object Prediction { return response.text ?: "" } -} +} \ No newline at end of file diff --git a/maestro-ai/src/main/resources/askForInsights_schema.json b/maestro-ai/src/main/resources/askForInsights_schema.json new file mode 100644 index 0000000000..7e4082c264 --- /dev/null +++ b/maestro-ai/src/main/resources/askForInsights_schema.json @@ -0,0 +1,32 @@ +{ + "name": "askForInsights", + "description": "Returns a list of possible insights found in the mobile app's UI", + "strict": true, + "schema": { + "type": "object", + "required": ["insights"], + "additionalProperties": false, + "properties": { + "insights": { + "type": "array", + "items": { + "type": "object", + "required": ["category", "reasoning"], + "additionalProperties": false, + "properties": { + "category": { + "type": "string", + "enum": [ + "layout", + "localization" + ] + }, + "reasoning": { + "type": "string" + } + } + } + } + } + } +} diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index 58dfcb23ea..2a1aa80311 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -29,6 +29,8 @@ import maestro.cli.App import maestro.cli.CliError import maestro.cli.DisableAnsiMixin import maestro.cli.ShowHelpMixin +import maestro.cli.api.ApiClient +import maestro.cli.auth.Auth import maestro.cli.device.Device import maestro.cli.device.DeviceService import maestro.cli.model.TestExecutionSummary @@ -43,6 +45,7 @@ import maestro.cli.session.MaestroSessionManager import maestro.cli.util.EnvUtils import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.PrintUtils +import maestro.cli.util.TestAnalysisReporter import maestro.cli.view.box import maestro.orchestra.error.ValidationError import maestro.orchestra.util.Env.withDefaultEnvVars @@ -56,6 +59,7 @@ import org.slf4j.LoggerFactory import picocli.CommandLine import picocli.CommandLine.Option import java.io.File +import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.Callable import java.util.concurrent.ConcurrentHashMap @@ -158,11 +162,20 @@ class TestCommand : Callable { ) private var headless: Boolean = false + @Option( + names = ["--analyze"], + description = ["[Beta] Enhance the test output analysis with AI Insights"], + ) + private var analyze: Boolean = false + @CommandLine.Spec lateinit var commandSpec: CommandLine.Model.CommandSpec private val usedPorts = ConcurrentHashMap() private val logger = LoggerFactory.getLogger(TestCommand::class.java) + private val auth by lazy { + Auth(ApiClient("https://api.copilot.mobile.dev/v2")) + } private fun isWebFlow(): Boolean { if (flowFiles.isSingleFile) { @@ -172,6 +185,7 @@ class TestCommand : Callable { return false } + override fun call(): Int { TestDebugReporter.install( debugOutputPathAsString = debugOutput, @@ -193,6 +207,19 @@ class TestCommand : Callable { throw CliError("The config file ${configFile?.absolutePath} does not exist.") } + // TODO: Integrate with `maestro login` + // if (analyze) { + // if (auth.getCachedAuthToken() == null) { + // throw CliError(listOf( + // "❌ Login Required\n", + // "You need to sign in before using the --analyze option.", + // "Please run:", + // "`maestro login`\n", + // "After signing in, try running your command again." + // ).joinToString("\n").box()) + // } + // } + val executionPlan = try { WorkspaceExecutionPlanner.plan( input = flowFiles.map { it.toPath().toAbsolutePath() }.toSet(), @@ -290,6 +317,7 @@ class TestCommand : Callable { suites.mergeSummaries()?.saveReport() if (effectiveShards > 1) printShardsMessage(passed, total, suites) + if (analyze) TestAnalysisReporter().runAnalysis(debugOutputPath) if (passed == total) 0 else 1 } @@ -333,7 +361,7 @@ class TestCommand : Callable { if (!flattenDebugOutput) { TestDebugReporter.deleteOldFiles() } - TestRunner.runContinuous(maestro, device, flowFile, env) + TestRunner.runContinuous(maestro, device, flowFile, env, analyze) } else { runSingleFlow(maestro, device, flowFile, debugOutputPath) } @@ -371,6 +399,7 @@ class TestCommand : Callable { env = env, resultView = resultView, debugOutputPath = debugOutputPath, + analyze = analyze ) if (resultSingle == 1) { diff --git a/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt new file mode 100644 index 0000000000..53ff260ca5 --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt @@ -0,0 +1,184 @@ +package maestro.cli.report + +import kotlinx.html.a +import kotlinx.html.body +import kotlinx.html.div +import kotlinx.html.h1 +import kotlinx.html.h2 +import kotlinx.html.h3 +import kotlinx.html.h4 +import kotlinx.html.head +import kotlinx.html.html +import kotlinx.html.img +import kotlinx.html.lang +import kotlinx.html.li +import kotlinx.html.meta +import kotlinx.html.p +import kotlinx.html.pre +import kotlinx.html.script +import kotlinx.html.span +import kotlinx.html.stream.appendHTML +import kotlinx.html.style +import kotlinx.html.svg +import kotlinx.html.title +import kotlinx.html.ul +import kotlinx.html.unsafe +import maestro.ai.FlowFiles +import maestro.ai.Insight +import java.nio.file.Files +import java.nio.file.Path +import java.util.* + +class HtmlInsightsAnalysisReporter { + + fun report( + flowFiles: List, + insights: List, + outputDestination: Path + ): Path { + if (!Files.isDirectory(outputDestination)) { + throw IllegalArgumentException("Output destination must be a directory") + } + + val htmlContent = buildInsightHtmlReport(flowFiles, insights) + val fileName = "Test_Report.html" + val filePath = outputDestination.resolve(fileName) + + Files.write(filePath, htmlContent.toByteArray()) + + return filePath + } + + private fun buildInsightHtmlReport(flowFiles: List, insights: List): String { + return buildString { + appendLine("") + appendHTML().html { + lang = "en" + head { + meta { charset = "UTF-8" } + meta { name = "viewport"; content = "width=device-width, initial-scale=1.0" } + title { +"Maestro Insights Report" } + script { src = "https://cdn.tailwindcss.com" } + style { + unsafe { + +""" + @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap'); + body { + font-family: 'Inter', sans-serif; + } + .image-carousel { + display: flex; + overflow-x: auto; + scroll-snap-type: x mandatory; + gap: 1rem; + padding: 1rem 0; + } + .image-carousel::-webkit-scrollbar { + display: none; + } + .carousel-item { + flex: 0 0 auto; + scroll-snap-align: start; + } + img { + max-height: 450px; + width: auto; + border-radius: 8px; + } + .file-link { + color: #2563eb; + text-decoration: underline; + margin-top: 0.5rem; + display: inline-block; + } + """ + } + } + } + body(classes = "bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-100") { + div(classes = "container mx-auto px-4 py-8") { + h1(classes = "text-4xl font-bold mb-8") { +"Maestro Insight Report" } + h2(classes = "text-2xl font-semibold mb-4") { +"Insights" } + div(classes = "grid grid-cols-1 md:grid-cols-2 gap-6") { + insights.groupBy { it.category }.forEach { (category, categoryInsights) -> + val emoji = when (category) { + "visual" -> "🎨" + "text" -> "📝" + "maestro" -> "🛠️" + else -> "🔍" + } + div(classes = "bg-white dark:bg-gray-800 rounded-lg shadow-lg overflow-hidden") { + div(classes = "p-6 h-full flex flex-col") { + div(classes = "flex items-center mb-4") { + span(classes = "text-4xl mr-3") { +emoji } + h3(classes = "text-2xl font-semibold") { +category.capitalize(Locale.ROOT) } + } + ul(classes = "space-y-4 flex-grow") { + categoryInsights.forEach { insight -> + li(classes = "flex items-start") { + svg(classes = "w-6 h-6 text-green-500 mr-2 flex-shrink-0") { + unsafe { + +""" + + """ + } + } + p(classes = "text-base") { +insight.reasoning } + } + } + } + } + } + } + } + flowFiles.forEach { fileGroup -> + h2(classes = "text-2xl font-semibold my-8") { +"Debug Artifacts" } + div(classes = "image-carousel") { + fileGroup.imageFiles.forEachIndexed { index, imageFile -> + div(classes = "carousel-item") { + img { + alt = "Image ${index + 1}" + src = imageFile.second.toString() + } + a(classes = "file-link", href = imageFile.second.toString(), target = "_blank") { +"Open Image ${index + 1}" } + } + } + } + div(classes = "grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6 my-4") { + fileGroup.jsonFiles.forEachIndexed { index, jsonFile -> + div(classes = "bg-white dark:bg-gray-800 rounded-lg shadow-lg overflow-hidden") { + div(classes = "p-4") { + h4(classes = "font-semibold text-lg mb-2") { +"JSON File ${index + 1}" } + pre(classes = "text-sm bg-gray-100 dark:bg-gray-700 p-2 rounded overflow-hidden") { + style = "max-height: 100px" + +String(jsonFile.first) + } + a(classes = "file-link", href = jsonFile.second.toString(), target = "_blank") { +"Open JSON File ${index + 1}" } + } + } + } + fileGroup.textFiles.forEachIndexed { index, textFile -> + div(classes = "bg-white dark:bg-gray-800 rounded-lg shadow-lg overflow-hidden") { + div(classes = "p-4") { + h4(classes = "font-semibold text-lg mb-2") { +"Text File ${index + 1}" } + pre(classes = "text-sm bg-gray-100 dark:bg-gray-700 p-2 rounded overflow-hidden") { + style = "max-height: 100px" + +String(textFile.first) + } + a(classes = "file-link", href = textFile.second.toString(), target = "_blank") { +"Open Text File ${index + 1}" } + } + } + } + } + } + } + } + } + } + } +} diff --git a/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt index 11ceef4d9d..313dac892b 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt @@ -56,6 +56,7 @@ object MaestroCommandRunner { commands: List, debugOutput: FlowDebugOutput, aiOutput: FlowAIOutput, + analyze: Boolean = false ): Boolean { val config = YamlCommandReader.getConfig(commands) val onFlowComplete = config?.onFlowComplete @@ -89,6 +90,9 @@ object MaestroCommandRunner { } refreshUi() + if (analyze) { + ScreenshotUtils.takeDebugScreenshotByCommand(maestro, debugOutput, CommandStatus.PENDING) + } val orchestra = Orchestra( maestro = maestro, @@ -106,6 +110,10 @@ object MaestroCommandRunner { onCommandComplete = { _, command -> logger.info("${command.description()} COMPLETED") commandStatuses[command] = CommandStatus.COMPLETED + if (analyze) { + ScreenshotUtils.takeDebugScreenshotByCommand(maestro, debugOutput, CommandStatus.COMPLETED) + } + debugOutput.commands[command]?.apply { status = CommandStatus.COMPLETED calculateDuration() diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt index 3bfee3e381..1730ebf5c8 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt @@ -43,7 +43,8 @@ object TestRunner { flowFile: File, env: Map, resultView: ResultView, - debugOutputPath: Path + debugOutputPath: Path, + analyze: Boolean = false ): Int { val debugOutput = FlowDebugOutput() var aiOutput = FlowAIOutput( @@ -68,6 +69,7 @@ object TestRunner { commands = commands, debugOutput = debugOutput, aiOutput = aiOutput, + analyze = analyze, ) } @@ -94,6 +96,7 @@ object TestRunner { device: Device?, flowFile: File, env: Map, + analyze: Boolean = false, ): Nothing { val resultView = AnsiResultView("> Press [ENTER] to restart the Flow\n\n", useEmojis = !EnvUtils.isWindows()) @@ -133,6 +136,7 @@ object TestRunner { flowName = "TODO", flowFile = flowFile, ), + analyze = analyze ) }.get() } diff --git a/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt b/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt index c3a6e1a2f6..140e71d2b2 100644 --- a/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt +++ b/maestro-cli/src/main/java/maestro/cli/util/ScreenshotUtils.kt @@ -35,6 +35,25 @@ object ScreenshotUtils { return result.getOrNull() } + fun takeDebugScreenshotByCommand(maestro: Maestro, debugOutput: FlowDebugOutput, status: CommandStatus): File? { + val result = kotlin.runCatching { + val out = File + .createTempFile("screenshot-${status}-${System.currentTimeMillis()}", ".png") + .also { it.deleteOnExit() } // save to another dir before exiting + maestro.takeScreenshot(out.sink(), false) + debugOutput.screenshots.add( + FlowDebugOutput.Screenshot( + screenshot = out, + timestamp = System.currentTimeMillis(), + status = status + ) + ) + out + } + + return result.getOrNull() + } + fun writeAIscreenshot(buffer: Buffer): File { val out = File .createTempFile("ai-screenshot-${System.currentTimeMillis()}", ".png") diff --git a/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt b/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt new file mode 100644 index 0000000000..2769161d4a --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt @@ -0,0 +1,97 @@ +package maestro.cli.util + +import maestro.ai.AI + +import maestro.ai.Prediction +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors +import kotlinx.coroutines.runBlocking +import maestro.ai.FlowFiles +import maestro.ai.Insight +import maestro.ai.openai.OpenAI +import maestro.cli.report.HtmlInsightsAnalysisReporter +import java.util.Locale + +class TestAnalysisReporter { + + private fun initAI(): AI { + val apiKey = "API-KEY" + val modelName = "gpt-4o" + + return OpenAI(apiKey = apiKey, defaultModel = modelName) + } + + fun runAnalysis(debugOutputPath: Path) { + val filesByFlow = processFilesByFlowName(debugOutputPath) + if (filesByFlow.isEmpty()) { + PrintUtils.warn("No files found for analysis.") + return + } + + PrintUtils.info("\nAnalysing and generating insights...\n") + + val insights = generateInsights(filesByFlow) + val outputFilePath = HtmlInsightsAnalysisReporter().report(filesByFlow, insights, debugOutputPath) + val os = System.getProperty("os.name").lowercase(Locale.getDefault()) + + PrintUtils.message(listOf( + "To view the report, open the following link in your browser:", + "file:${if (os.contains("win")) "///" else "//"}${outputFilePath}\n", + "Analyze support is in Beta. We would appreciate your feedback!" + ).joinToString("\n")) + } + + private fun generateInsights(flowFiles: List): List = runBlocking { + val aiClient = initAI() + + Prediction.generateInsights( + aiClient = aiClient, + flowFiles = flowFiles, + ) + } + + private fun processFilesByFlowName(outputPath: Path): List { + val files = Files.walk(outputPath) + .filter(Files::isRegularFile) + .collect(Collectors.toList()) + + return if (files.isNotEmpty()) { + val (imageFiles, jsonFiles, textFiles) = getFilesByType(files) + listOf( + FlowFiles( + jsonFiles = jsonFiles, + imageFiles = imageFiles, + textFiles = textFiles + ) + ) + } else { + emptyList() + } + } + + private fun getFilesByType(files: List): Triple>, List>, List>> { + val imageFiles = mutableListOf>() + val jsonFiles = mutableListOf>() + val textFiles = mutableListOf>() + + files.forEach { filePath -> + val content = Files.readAllBytes(filePath) + val fileName = filePath.fileName.toString() + + when { + fileName.endsWith(".png", true) || fileName.endsWith(".jpg", true) || fileName.endsWith(".jpeg", true) -> { + imageFiles.add(content to filePath) + } + fileName.endsWith(".json", true) -> { + jsonFiles.add(content to filePath) + } + else -> { + textFiles.add(content to filePath) + } + } + } + + return Triple(imageFiles, jsonFiles, textFiles) + } +} From dda9ade679705ef3328e9c6bb0e0bb16e5c16855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Takahashi?= Date: Wed, 22 Jan 2025 12:32:23 +0000 Subject: [PATCH 12/16] refactor(analyze): Test analyze report and generation to use server side output --- .../src/main/java/maestro/ai/Prediction.kt | 123 ------------- .../main/resources/askForInsights_schema.json | 32 ---- .../main/java/maestro/cli/api/ApiClient.kt | 130 +++++++++++++- .../java/maestro/cli/cloud/CloudInteractor.kt | 61 ++++++- .../java/maestro/cli/command/LoginCommand.kt | 6 +- .../java/maestro/cli/command/TestCommand.kt | 29 +-- .../report/HtmlInsightsAnalysisReporter.kt | 167 +----------------- .../maestro/cli/util/TestAnalysisReporter.kt | 75 +++----- 8 files changed, 221 insertions(+), 402 deletions(-) delete mode 100644 maestro-ai/src/main/resources/askForInsights_schema.json diff --git a/maestro-ai/src/main/java/maestro/ai/Prediction.kt b/maestro-ai/src/main/java/maestro/ai/Prediction.kt index 11385160c2..95d41da4ca 100644 --- a/maestro-ai/src/main/java/maestro/ai/Prediction.kt +++ b/maestro-ai/src/main/java/maestro/ai/Prediction.kt @@ -4,13 +4,6 @@ import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import kotlinx.serialization.json.jsonObject import maestro.ai.openai.OpenAI -import java.nio.file.Path - -data class FlowFiles( - val jsonFiles: List>, - val imageFiles: List>, - val textFiles: List> -) @Serializable data class Defect( @@ -28,27 +21,12 @@ private data class ExtractTextResponse( val text: String? ) -@Serializable -data class Insight( - val category: String, - val reasoning: String, -) - -@Serializable -private data class AskForInsightsResponse( - val insights: List, -) - object Prediction { private val askForDefectsSchema by lazy { readSchema("askForDefects") } - private val askForInsightsSchema by lazy { - readSchema("askForInsights") - } - private val extractTextSchema by lazy { readSchema("extractText") } @@ -73,12 +51,6 @@ object Prediction { "layout" to "Some UI elements are overlapping or are cropped", ) - private val insightsCategories = listOf( - "visual" to "Insights related to UI elements that are overlapping or cropped", - "text" to " insights on Grammar and spelling, like suggestions and optimizations on the text on the page, or Inconsistent use of language, for example, mixed English and Portuguese", - "maestro" to "Insights on the maestro testing tool usage, best practices, tips on debugging, optimizing the workspace, or how to make the best usage of maestro commands, APIs, or extra features like maestro studio, or Robin cloud", - ) - private val allDefectCategories = defectCategories + listOf("assertion" to "The assertion is not true") suspend fun findDefects( @@ -172,101 +144,6 @@ object Prediction { return defects.defects } - suspend fun generateInsights( - aiClient: AI, - flowFiles: List, - printRawResponse: Boolean = false, - ): List { - val prompt = buildString { - appendLine( - """ - You are a QA engineer performing quality assurance for a mobile application. - Identify any defects in the provided screenshots and optionally - - You are using Maestro for e2e mobile testing, understand the tool API and best practices on how to use it based on its Docs - You are given screenshots of the application and the JSON and text files artifacts from the debug artifacts of maestro e2e testing tool. - - Given the following maestro flows - """.trimIndent() - ) - - flowFiles.forEach { - appendLine( - """ - You are going to transcribe the screenshots and analyze every file below: - ${if (it.jsonFiles.isNotEmpty()) "Based on this JSON files: ${it.jsonFiles.joinToString("\n",transform = { (content) -> String(content) })}\n" else ""} - ${if (it.textFiles.isNotEmpty()) "Based on this files: ${it.textFiles.joinToString("\n",transform = { (content) -> String(content) })}\n" else ""} - """.trimIndent() - ) - } - - append( - """ - | - |RULES: - | - |Your task is to generate Insights following the RULES: - |* You must explain understand each context based on the provided data analyzsing each flow. - |* - |* All Insights you find must belong to one of the following categories: - |${insightsCategories.joinToString(separator = "\n") { " * ${it.first}: ${it.second}" }} - |* If you see Insights, your response MUST only include defect name and detailed reasoning for each defect. - |* Provide response as a list of JSON objects, each representing : - |* Do not repeat the context text into the insights, make it useful for the QA developer reading the insights. - |* Do not generate duplicated or similar insights just changing the category. - |* Do not generate spam insights that are too obvious based on the screenshot. - |* Do not raise false positives. Some example responses that have a high chance of being a false positive: - | * button is partially cropped at the bottom - | * button is not aligned horizontally/vertically within its container - | * element not found because it does not exist on the current screen - | * ensure that the app is in the correct state before looking for the text - """.trimMargin("|") - ) - - append( - """ - | - |* You must provide result as a valid JSON object, matching this structure: - | - | { - | "insights": [ - | { - | "category": "", - | "reasoning": "" - | }, - | { - | "category": "", - | "reasoning": "" - | } - | ] - | } - | - |DO NOT output any other information in the JSON object. - """.trimMargin("|") - ) - } - - val aiResponse = aiClient.chatCompletion( - prompt, - model = aiClient.defaultModel, - maxTokens = 4096, - identifier = "find-defects", - imageDetail = "high", - images = flowFiles.flatMap { it.imageFiles.map { (content) -> content } }, - jsonSchema = if (aiClient is OpenAI) json.parseToJsonElement(askForInsightsSchema).jsonObject else null, - ) - - if (printRawResponse) { - println("--- RAW RESPONSE START ---") - println(aiResponse.response) - println("--- RAW RESPONSE END ---") - } - - val insights = json.decodeFromString(aiResponse.response) - - return insights.insights; - } - suspend fun performAssertion( aiClient: AI, screen: ByteArray, diff --git a/maestro-ai/src/main/resources/askForInsights_schema.json b/maestro-ai/src/main/resources/askForInsights_schema.json deleted file mode 100644 index 7e4082c264..0000000000 --- a/maestro-ai/src/main/resources/askForInsights_schema.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "askForInsights", - "description": "Returns a list of possible insights found in the mobile app's UI", - "strict": true, - "schema": { - "type": "object", - "required": ["insights"], - "additionalProperties": false, - "properties": { - "insights": { - "type": "array", - "items": { - "type": "object", - "required": ["category", "reasoning"], - "additionalProperties": false, - "properties": { - "category": { - "type": "string", - "enum": [ - "layout", - "localization" - ] - }, - "reasoning": { - "type": "string" - } - } - } - } - } - } -} diff --git a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt index 18f1b3d79a..067c35b8be 100644 --- a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt +++ b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt @@ -1,6 +1,8 @@ package maestro.cli.api import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.github.michaelbull.result.Err @@ -13,6 +15,7 @@ import maestro.cli.model.FlowStatus import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.util.CiUtils import maestro.cli.util.EnvUtils +import maestro.cli.util.FlowFiles import maestro.cli.util.PrintUtils import maestro.utils.HttpClient import okhttp3.Interceptor @@ -34,7 +37,6 @@ import java.nio.file.Path import kotlin.io.path.absolutePathString import kotlin.io.path.exists import kotlin.time.Duration.Companion.minutes -import okhttp3.MediaType class ApiClient( private val baseUrl: String, @@ -185,7 +187,11 @@ class ApiClient( val baseUrl = "https://maestro-record.ngrok.io" val body = MultipartBody.Builder() .setType(MultipartBody.FORM) - .addFormDataPart("screenRecording", screenRecording.name, screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener)) + .addFormDataPart( + "screenRecording", + screenRecording.name, + screenRecording.asRequestBody("application/mp4".toMediaType()).observable(progressListener) + ) .addFormDataPart("frames", JSON.writeValueAsString(frames)) .build() val request = Request.Builder() @@ -267,15 +273,27 @@ class ApiClient( val bodyBuilder = MultipartBody.Builder() .setType(MultipartBody.FORM) - .addFormDataPart("workspace", "workspace.zip", workspaceZip.toFile().asRequestBody("application/zip".toMediaType())) + .addFormDataPart( + "workspace", + "workspace.zip", + workspaceZip.toFile().asRequestBody("application/zip".toMediaType()) + ) .addFormDataPart("request", JSON.writeValueAsString(requestPart)) if (appFile != null) { - bodyBuilder.addFormDataPart("app_binary", "app.zip", appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener)) + bodyBuilder.addFormDataPart( + "app_binary", + "app.zip", + appFile.toFile().asRequestBody("application/zip".toMediaType()).observable(progressListener) + ) } if (mappingFile != null) { - bodyBuilder.addFormDataPart("mapping", "mapping.txt", mappingFile.toFile().asRequestBody("text/plain".toMediaType())) + bodyBuilder.addFormDataPart( + "mapping", + "mapping.txt", + mappingFile.toFile().asRequestBody("text/plain".toMediaType()) + ) } val body = bodyBuilder.build() @@ -286,7 +304,7 @@ class ApiClient( throw CliError(message) } - PrintUtils.message("$message, retrying (${completedRetries+1}/$maxRetryCount)...") + PrintUtils.message("$message, retrying (${completedRetries + 1}/$maxRetryCount)...") Thread.sleep(BASE_RETRY_DELAY_MS + (2000 * completedRetries)) return upload( @@ -440,6 +458,58 @@ class ApiClient( } } + fun analyze( + authToken: String, + flowFiles: List, + ): AnalyzeResponse { + if (flowFiles.isEmpty()) throw CliError("Missing flow files to analyze") + + val screenshots = mutableListOf>() + val logs = mutableListOf>() + + flowFiles.forEach { flowFile -> + flowFile.imageFiles.forEach { (imageData, path) -> + val imageName = path.fileName.toString() + screenshots.add(Pair(imageName, imageData)) + } + + flowFile.textFiles.forEach { (textData, path) -> + val textName = path.fileName.toString() + logs.add(Pair(textName, textData)) + } + } + + val requestBody = mapOf( + "screenshots" to screenshots, + "logs" to logs + ) + + val mediaType = "application/json; charset=utf-8".toMediaType() + val body = JSON.writeValueAsString(requestBody).toRequestBody(mediaType) + + val url = "$baseUrl/v2/analyze" + + val request = Request.Builder() + .header("Authorization", "Bearer $authToken") + .url(url) + .post(body) + .build() + + val response = client.newCall(request).execute() + + response.use { + if (!response.isSuccessful) { + val errorMessage = response.body?.string().takeIf { it?.isNotEmpty() == true } ?: "Unknown" + throw CliError("Analyze request failed (${response.code}): $errorMessage") + } + + val parsed = JSON.readValue(response.body?.bytes(), AnalyzeResponse::class.java) + + return parsed; + } + } + + data class ApiException( val statusCode: Int?, ) : Exception("Request failed. Status code: $statusCode") @@ -459,7 +529,7 @@ data class RobinUploadResponse( val appId: String, val deviceConfiguration: DeviceConfiguration?, val appBinaryId: String?, -): UploadResponse() +) : UploadResponse() @JsonIgnoreProperties(ignoreUnknown = true) data class MaestroCloudUploadResponse( @@ -468,7 +538,7 @@ data class MaestroCloudUploadResponse( val uploadId: String, val appBinaryId: String?, val deviceInfo: DeviceInfo? -): UploadResponse() +) : UploadResponse() data class DeviceConfiguration( val platform: String, @@ -514,7 +584,6 @@ data class UploadStatus( STOPPED } - // These values must match backend monorepo models // in package models.benchmark.BenchmarkCancellationReason enum class CancellationReason { @@ -580,3 +649,46 @@ class SystemInformationInterceptor : Interceptor { return chain.proceed(newRequest) } } + +data class Insight( + val category: String, + val reasoning: String, +) + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "category" +) +@JsonSubTypes( + JsonSubTypes.Type(value = AnalyzeResponse.HtmlOutput::class, name = "HTML_OUTPUT"), + JsonSubTypes.Type(value = AnalyzeResponse.CliOutput::class, name = "CLI_OUTPUT") +) +sealed class AnalyzeResponse( + open val status: Status, + open val output: String, + open val insights: List +) { + + data class HtmlOutput( + override val status: Status, + override val output: String, + override val insights: List, + val category: Category = Category.HTML_OUTPUT + ) : AnalyzeResponse(status, output, insights) + + data class CliOutput( + override val status: Status, + override val output: String, + override val insights: List, + val category: Category = Category.CLI_OUTPUT + ) : AnalyzeResponse(status, output, insights) + + enum class Status { + SUCCESS, ERROR + } + + enum class Category { + HTML_OUTPUT, CLI_OUTPUT + } +} \ No newline at end of file diff --git a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt index 48f5a49c1c..6c5596228b 100644 --- a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt @@ -1,6 +1,7 @@ package maestro.cli.cloud import maestro.cli.CliError +import maestro.cli.api.AnalyzeResponse import maestro.cli.api.ApiClient import maestro.cli.api.DeviceConfiguration import maestro.cli.api.DeviceInfo @@ -13,11 +14,13 @@ import maestro.cli.model.FlowStatus import maestro.cli.model.RunningFlow import maestro.cli.model.RunningFlows import maestro.cli.model.TestExecutionSummary +import maestro.cli.report.HtmlInsightsAnalysisReporter import maestro.cli.report.ReportFormat import maestro.cli.report.ReporterFactory import maestro.cli.util.EnvUtils import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.FileUtils.isZip +import maestro.cli.util.FlowFiles import maestro.cli.util.PrintUtils import maestro.cli.util.TimeUtils import maestro.cli.util.WorkspaceUtils @@ -35,6 +38,8 @@ import okio.sink import org.rauschig.jarchivelib.ArchiveFormat import org.rauschig.jarchivelib.ArchiverFactory import java.io.File +import java.nio.file.Path +import java.util.* import java.util.concurrent.TimeUnit import kotlin.io.path.absolute @@ -78,10 +83,7 @@ class CloudInteractor( if (mapping?.exists() == false) throw CliError("File does not exist: ${mapping.absolutePath}") if (async && reportFormat != ReportFormat.NOOP) throw CliError("Cannot use --format with --async") - val authToken = apiKey // Check for API key - ?: auth.getCachedAuthToken() // Otherwise, if the user has already logged in, use the cached auth token - ?: EnvUtils.maestroCloudApiKey() // Resolve API key from shell if set - ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow + val authToken = getAuthToken(apiKey) PrintUtils.message("Uploading Flow(s)...") @@ -487,4 +489,55 @@ class CloudInteractor( duration = runningFlows.duration ) } + + private fun getAuthToken(apiKey: String?): String { + return apiKey // Check for API key + ?: auth.getCachedAuthToken() // Otherwise, if the user has already logged in, use the cached auth token + ?: EnvUtils.maestroCloudApiKey() // Resolve API key from shell if set + ?: auth.triggerSignInFlow() // Otherwise, trigger the sign-in flow + } + + fun analyze( + apiKey: String?, + flowFiles: List, + debugOutputPath: Path, + ): Int { + val authToken = getAuthToken(apiKey) + + PrintUtils.info("\uD83D\uDD0E Analyzing Flow(s)...\n") + + try { + val response = client.analyze(authToken, flowFiles) + if (response.status == AnalyzeResponse.Status.ERROR) { + PrintUtils.err("Unexpected error while analyzing Flow(s): ${response.output}") + return 1 + } + + when (response) { + is AnalyzeResponse.HtmlOutput -> { + val outputFilePath = HtmlInsightsAnalysisReporter().report(response.output, debugOutputPath) + val os = System.getProperty("os.name").lowercase(Locale.getDefault()) + + PrintUtils.message( + listOf( + "To view the report, open the following link in your browser:", + "file:${if (os.contains("win")) "///" else "//"}${outputFilePath}\n", + "Analyze support is in Beta. We would appreciate your feedback!" + ).joinToString("\n") + ) + + return 0; + } + + is AnalyzeResponse.CliOutput -> { + PrintUtils.message(response.output) + return 0 + } + } + } catch (error: CliError) { + PrintUtils.err("Unexpected error while analyzing Flow(s): ${error.message}") + return 1 + } + } + } diff --git a/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt index 5338beaa44..405a2e1295 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt @@ -10,6 +10,7 @@ import java.util.concurrent.Callable import kotlin.io.path.absolutePathString import maestro.cli.report.TestDebugReporter import maestro.debuglog.LogConfig +import picocli.CommandLine.Option @CommandLine.Command( name = "login", @@ -25,8 +26,11 @@ class LoginCommand : Callable { @CommandLine.Mixin var showHelpMixin: ShowHelpMixin? = null + @Option(names = ["--apiUrl"], description = ["API base URL"]) + private var apiUrl: String = "https://api.copilot.mobile.dev" + private val auth by lazy { - Auth(ApiClient("https://api.copilot.mobile.dev/v2")) + Auth(ApiClient("$apiUrl/v2")) } override fun call(): Int { diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index 2a1aa80311..b8580ceed1 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -29,8 +29,6 @@ import maestro.cli.App import maestro.cli.CliError import maestro.cli.DisableAnsiMixin import maestro.cli.ShowHelpMixin -import maestro.cli.api.ApiClient -import maestro.cli.auth.Auth import maestro.cli.device.Device import maestro.cli.device.DeviceService import maestro.cli.model.TestExecutionSummary @@ -54,12 +52,10 @@ import maestro.orchestra.workspace.WorkspaceExecutionPlanner import maestro.orchestra.workspace.WorkspaceExecutionPlanner.ExecutionPlan import maestro.utils.isSingleFile import okio.sink -import org.jetbrains.skiko.hostId import org.slf4j.LoggerFactory import picocli.CommandLine import picocli.CommandLine.Option import java.io.File -import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.Callable import java.util.concurrent.ConcurrentHashMap @@ -168,14 +164,18 @@ class TestCommand : Callable { ) private var analyze: Boolean = false + @Option(names = ["--apiUrl"], description = ["[Beta] API base URL"]) + private var apiUrl: String = "https://api.copilot.mobile.dev" + + + @Option(names = ["--apiKey"], description = ["[Beta] API key"]) + private var apiKey: String? = null + @CommandLine.Spec lateinit var commandSpec: CommandLine.Model.CommandSpec private val usedPorts = ConcurrentHashMap() private val logger = LoggerFactory.getLogger(TestCommand::class.java) - private val auth by lazy { - Auth(ApiClient("https://api.copilot.mobile.dev/v2")) - } private fun isWebFlow(): Boolean { if (flowFiles.isSingleFile) { @@ -207,19 +207,6 @@ class TestCommand : Callable { throw CliError("The config file ${configFile?.absolutePath} does not exist.") } - // TODO: Integrate with `maestro login` - // if (analyze) { - // if (auth.getCachedAuthToken() == null) { - // throw CliError(listOf( - // "❌ Login Required\n", - // "You need to sign in before using the --analyze option.", - // "Please run:", - // "`maestro login`\n", - // "After signing in, try running your command again." - // ).joinToString("\n").box()) - // } - // } - val executionPlan = try { WorkspaceExecutionPlanner.plan( input = flowFiles.map { it.toPath().toAbsolutePath() }.toSet(), @@ -317,7 +304,7 @@ class TestCommand : Callable { suites.mergeSummaries()?.saveReport() if (effectiveShards > 1) printShardsMessage(passed, total, suites) - if (analyze) TestAnalysisReporter().runAnalysis(debugOutputPath) + if (analyze) TestAnalysisReporter(apiUrl = apiUrl, apiKey = apiKey).runAnalysis(debugOutputPath) if (passed == total) 0 else 1 } diff --git a/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt index 53ff260ca5..6d55cd0d8c 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/HtmlInsightsAnalysisReporter.kt @@ -1,184 +1,23 @@ package maestro.cli.report -import kotlinx.html.a -import kotlinx.html.body -import kotlinx.html.div -import kotlinx.html.h1 -import kotlinx.html.h2 -import kotlinx.html.h3 -import kotlinx.html.h4 -import kotlinx.html.head -import kotlinx.html.html -import kotlinx.html.img -import kotlinx.html.lang -import kotlinx.html.li -import kotlinx.html.meta -import kotlinx.html.p -import kotlinx.html.pre -import kotlinx.html.script -import kotlinx.html.span -import kotlinx.html.stream.appendHTML -import kotlinx.html.style -import kotlinx.html.svg -import kotlinx.html.title -import kotlinx.html.ul -import kotlinx.html.unsafe -import maestro.ai.FlowFiles -import maestro.ai.Insight import java.nio.file.Files import java.nio.file.Path -import java.util.* class HtmlInsightsAnalysisReporter { fun report( - flowFiles: List, - insights: List, + html: String, outputDestination: Path ): Path { if (!Files.isDirectory(outputDestination)) { throw IllegalArgumentException("Output destination must be a directory") } - val htmlContent = buildInsightHtmlReport(flowFiles, insights) - val fileName = "Test_Report.html" + val fileName = "insights-report.html" val filePath = outputDestination.resolve(fileName) - Files.write(filePath, htmlContent.toByteArray()) + Files.write(filePath, html.toByteArray()) return filePath } - - private fun buildInsightHtmlReport(flowFiles: List, insights: List): String { - return buildString { - appendLine("") - appendHTML().html { - lang = "en" - head { - meta { charset = "UTF-8" } - meta { name = "viewport"; content = "width=device-width, initial-scale=1.0" } - title { +"Maestro Insights Report" } - script { src = "https://cdn.tailwindcss.com" } - style { - unsafe { - +""" - @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap'); - body { - font-family: 'Inter', sans-serif; - } - .image-carousel { - display: flex; - overflow-x: auto; - scroll-snap-type: x mandatory; - gap: 1rem; - padding: 1rem 0; - } - .image-carousel::-webkit-scrollbar { - display: none; - } - .carousel-item { - flex: 0 0 auto; - scroll-snap-align: start; - } - img { - max-height: 450px; - width: auto; - border-radius: 8px; - } - .file-link { - color: #2563eb; - text-decoration: underline; - margin-top: 0.5rem; - display: inline-block; - } - """ - } - } - } - body(classes = "bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-100") { - div(classes = "container mx-auto px-4 py-8") { - h1(classes = "text-4xl font-bold mb-8") { +"Maestro Insight Report" } - h2(classes = "text-2xl font-semibold mb-4") { +"Insights" } - div(classes = "grid grid-cols-1 md:grid-cols-2 gap-6") { - insights.groupBy { it.category }.forEach { (category, categoryInsights) -> - val emoji = when (category) { - "visual" -> "🎨" - "text" -> "📝" - "maestro" -> "🛠️" - else -> "🔍" - } - div(classes = "bg-white dark:bg-gray-800 rounded-lg shadow-lg overflow-hidden") { - div(classes = "p-6 h-full flex flex-col") { - div(classes = "flex items-center mb-4") { - span(classes = "text-4xl mr-3") { +emoji } - h3(classes = "text-2xl font-semibold") { +category.capitalize(Locale.ROOT) } - } - ul(classes = "space-y-4 flex-grow") { - categoryInsights.forEach { insight -> - li(classes = "flex items-start") { - svg(classes = "w-6 h-6 text-green-500 mr-2 flex-shrink-0") { - unsafe { - +""" - - """ - } - } - p(classes = "text-base") { +insight.reasoning } - } - } - } - } - } - } - } - flowFiles.forEach { fileGroup -> - h2(classes = "text-2xl font-semibold my-8") { +"Debug Artifacts" } - div(classes = "image-carousel") { - fileGroup.imageFiles.forEachIndexed { index, imageFile -> - div(classes = "carousel-item") { - img { - alt = "Image ${index + 1}" - src = imageFile.second.toString() - } - a(classes = "file-link", href = imageFile.second.toString(), target = "_blank") { +"Open Image ${index + 1}" } - } - } - } - div(classes = "grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6 my-4") { - fileGroup.jsonFiles.forEachIndexed { index, jsonFile -> - div(classes = "bg-white dark:bg-gray-800 rounded-lg shadow-lg overflow-hidden") { - div(classes = "p-4") { - h4(classes = "font-semibold text-lg mb-2") { +"JSON File ${index + 1}" } - pre(classes = "text-sm bg-gray-100 dark:bg-gray-700 p-2 rounded overflow-hidden") { - style = "max-height: 100px" - +String(jsonFile.first) - } - a(classes = "file-link", href = jsonFile.second.toString(), target = "_blank") { +"Open JSON File ${index + 1}" } - } - } - } - fileGroup.textFiles.forEachIndexed { index, textFile -> - div(classes = "bg-white dark:bg-gray-800 rounded-lg shadow-lg overflow-hidden") { - div(classes = "p-4") { - h4(classes = "font-semibold text-lg mb-2") { +"Text File ${index + 1}" } - pre(classes = "text-sm bg-gray-100 dark:bg-gray-700 p-2 rounded overflow-hidden") { - style = "max-height: 100px" - +String(textFile.first) - } - a(classes = "file-link", href = textFile.second.toString(), target = "_blank") { +"Open Text File ${index + 1}" } - } - } - } - } - } - } - } - } - } - } } diff --git a/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt b/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt index 2769161d4a..6f3afbaf8f 100644 --- a/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt @@ -1,53 +1,32 @@ package maestro.cli.util -import maestro.ai.AI - -import maestro.ai.Prediction import java.nio.file.Files import java.nio.file.Path import java.util.stream.Collectors -import kotlinx.coroutines.runBlocking -import maestro.ai.FlowFiles -import maestro.ai.Insight -import maestro.ai.openai.OpenAI -import maestro.cli.report.HtmlInsightsAnalysisReporter -import java.util.Locale - -class TestAnalysisReporter { +import maestro.cli.api.ApiClient +import maestro.cli.cloud.CloudInteractor - private fun initAI(): AI { - val apiKey = "API-KEY" - val modelName = "gpt-4o" +data class FlowFiles( + val imageFiles: List>, + val textFiles: List> +) - return OpenAI(apiKey = apiKey, defaultModel = modelName) +class TestAnalysisReporter(private val apiUrl: String, private val apiKey: String?) { + private val apiclient by lazy { + ApiClient(apiUrl) } - fun runAnalysis(debugOutputPath: Path) { - val filesByFlow = processFilesByFlowName(debugOutputPath) - if (filesByFlow.isEmpty()) { - PrintUtils.warn("No files found for analysis.") - return + fun runAnalysis(debugOutputPath: Path): Int { + val flowFiles = processFilesByFlowName(debugOutputPath) + if (flowFiles.isEmpty()) { + PrintUtils.warn("No screenshots or debug artifacts found for analysis.") + return 0; } - PrintUtils.info("\nAnalysing and generating insights...\n") - - val insights = generateInsights(filesByFlow) - val outputFilePath = HtmlInsightsAnalysisReporter().report(filesByFlow, insights, debugOutputPath) - val os = System.getProperty("os.name").lowercase(Locale.getDefault()) - - PrintUtils.message(listOf( - "To view the report, open the following link in your browser:", - "file:${if (os.contains("win")) "///" else "//"}${outputFilePath}\n", - "Analyze support is in Beta. We would appreciate your feedback!" - ).joinToString("\n")) - } - - private fun generateInsights(flowFiles: List): List = runBlocking { - val aiClient = initAI() - - Prediction.generateInsights( - aiClient = aiClient, + return CloudInteractor(apiclient).analyze( + apiKey = apiKey, flowFiles = flowFiles, + debugOutputPath = debugOutputPath ) } @@ -57,10 +36,9 @@ class TestAnalysisReporter { .collect(Collectors.toList()) return if (files.isNotEmpty()) { - val (imageFiles, jsonFiles, textFiles) = getFilesByType(files) + val (imageFiles, textFiles) = getDebugFiles(files) listOf( FlowFiles( - jsonFiles = jsonFiles, imageFiles = imageFiles, textFiles = textFiles ) @@ -70,28 +48,29 @@ class TestAnalysisReporter { } } - private fun getFilesByType(files: List): Triple>, List>, List>> { + private fun getDebugFiles(files: List): Pair>, List>> { val imageFiles = mutableListOf>() - val jsonFiles = mutableListOf>() val textFiles = mutableListOf>() files.forEach { filePath -> val content = Files.readAllBytes(filePath) - val fileName = filePath.fileName.toString() + val fileName = filePath.fileName.toString().lowercase() when { - fileName.endsWith(".png", true) || fileName.endsWith(".jpg", true) || fileName.endsWith(".jpeg", true) -> { + fileName.endsWith(".png") || fileName.endsWith(".jpg") || fileName.endsWith(".jpeg") -> { imageFiles.add(content to filePath) } - fileName.endsWith(".json", true) -> { - jsonFiles.add(content to filePath) + + fileName.startsWith("commands") -> { + textFiles.add(content to filePath) } - else -> { + + fileName == "maestro.log" -> { textFiles.add(content to filePath) } } } - return Triple(imageFiles, jsonFiles, textFiles) + return Pair(imageFiles, textFiles) } } From 60dbad146c36bd3d07a28dd0100ac0dc7eb8f4a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Takahashi?= Date: Thu, 23 Jan 2025 10:05:36 +0000 Subject: [PATCH 13/16] feat(login): Adjusts apiUrl option to kebab case --- maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt index 405a2e1295..a6a2520510 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/LoginCommand.kt @@ -26,7 +26,7 @@ class LoginCommand : Callable { @CommandLine.Mixin var showHelpMixin: ShowHelpMixin? = null - @Option(names = ["--apiUrl"], description = ["API base URL"]) + @Option(names = ["--api-url", "--apiUrl"], description = ["API base URL"]) private var apiUrl: String = "https://api.copilot.mobile.dev" private val auth by lazy { From bbd1cec8922193367f6dd529942f134f62c3a3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Takahashi?= Date: Thu, 23 Jan 2025 10:06:04 +0000 Subject: [PATCH 14/16] feat(test): Adjusts api url and key options to kebab case --- maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index b8580ceed1..ce44143870 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -164,11 +164,11 @@ class TestCommand : Callable { ) private var analyze: Boolean = false - @Option(names = ["--apiUrl"], description = ["[Beta] API base URL"]) + @Option(names = ["--api-url"], description = ["[Beta] API base URL"]) private var apiUrl: String = "https://api.copilot.mobile.dev" - @Option(names = ["--apiKey"], description = ["[Beta] API key"]) + @Option(names = ["--api-key"], description = ["[Beta] API key"]) private var apiKey: String? = null @CommandLine.Spec From 030da108a86a3e0fbc6ab2ef852adb367274f3ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Takahashi?= Date: Thu, 23 Jan 2025 11:06:10 +0000 Subject: [PATCH 15/16] style(analyze): Adjusts cli output messages and styles --- .../src/main/java/maestro/cli/cloud/CloudInteractor.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt index 6c5596228b..ad9a792bcb 100644 --- a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt @@ -504,7 +504,7 @@ class CloudInteractor( ): Int { val authToken = getAuthToken(apiKey) - PrintUtils.info("\uD83D\uDD0E Analyzing Flow(s)...\n") + PrintUtils.info("\n\uD83D\uDD0E Analyzing Flow(s)...") try { val response = client.analyze(authToken, flowFiles) @@ -518,7 +518,7 @@ class CloudInteractor( val outputFilePath = HtmlInsightsAnalysisReporter().report(response.output, debugOutputPath) val os = System.getProperty("os.name").lowercase(Locale.getDefault()) - PrintUtils.message( + PrintUtils.success( listOf( "To view the report, open the following link in your browser:", "file:${if (os.contains("win")) "///" else "//"}${outputFilePath}\n", From 820acab06d3d6b242254077f8f13ca9879ff7a7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Takahashi?= Date: Fri, 24 Jan 2025 12:05:42 +0000 Subject: [PATCH 16/16] feat(analyze): Adjusts Analysis Manager to maybe notify new feature --- maestro-cli/src/main/java/maestro/cli/App.kt | 5 +- .../main/java/maestro/cli/api/ApiClient.kt | 2 +- .../java/maestro/cli/cloud/CloudInteractor.kt | 2 +- .../java/maestro/cli/command/TestCommand.kt | 4 +- .../java/maestro/cli/insights/Insights.kt | 31 ---- .../cli/insights/TestAnalysisManager.kt | 142 ++++++++++++++++++ .../maestro/cli/util/TestAnalysisReporter.kt | 76 ---------- 7 files changed, 148 insertions(+), 114 deletions(-) delete mode 100644 maestro-cli/src/main/java/maestro/cli/insights/Insights.kt create mode 100644 maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt delete mode 100644 maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt diff --git a/maestro-cli/src/main/java/maestro/cli/App.kt b/maestro-cli/src/main/java/maestro/cli/App.kt index 056b5d974a..757c66ff06 100644 --- a/maestro-cli/src/main/java/maestro/cli/App.kt +++ b/maestro-cli/src/main/java/maestro/cli/App.kt @@ -33,7 +33,7 @@ import maestro.cli.command.StartDeviceCommand import maestro.cli.command.StudioCommand import maestro.cli.command.TestCommand import maestro.cli.command.UploadCommand -import maestro.cli.insights.Insights +import maestro.cli.insights.TestAnalysisManager import maestro.cli.update.Updates import maestro.cli.util.ChangeLogUtils import maestro.cli.util.ErrorReporter @@ -144,8 +144,7 @@ fun main(args: Array) { .execute(*args) DebugLogStore.finalizeRun() - - Insights.maybeNotifyInsights() + TestAnalysisManager.maybeNotify() val newVersion = Updates.checkForUpdates() if (newVersion != null) { diff --git a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt index 067c35b8be..4eefd03dc9 100644 --- a/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt +++ b/maestro-cli/src/main/java/maestro/cli/api/ApiClient.kt @@ -11,11 +11,11 @@ import com.github.michaelbull.result.Result import maestro.cli.CliError import maestro.cli.analytics.Analytics import maestro.cli.analytics.AnalyticsReport +import maestro.cli.insights.FlowFiles import maestro.cli.model.FlowStatus import maestro.cli.runner.resultview.AnsiResultView import maestro.cli.util.CiUtils import maestro.cli.util.EnvUtils -import maestro.cli.util.FlowFiles import maestro.cli.util.PrintUtils import maestro.utils.HttpClient import okhttp3.Interceptor diff --git a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt index ad9a792bcb..4f9ac5895f 100644 --- a/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/cloud/CloudInteractor.kt @@ -10,6 +10,7 @@ import maestro.cli.api.RobinUploadResponse import maestro.cli.api.UploadStatus import maestro.cli.auth.Auth import maestro.cli.device.Platform +import maestro.cli.insights.FlowFiles import maestro.cli.model.FlowStatus import maestro.cli.model.RunningFlow import maestro.cli.model.RunningFlows @@ -20,7 +21,6 @@ import maestro.cli.report.ReporterFactory import maestro.cli.util.EnvUtils import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.FileUtils.isZip -import maestro.cli.util.FlowFiles import maestro.cli.util.PrintUtils import maestro.cli.util.TimeUtils import maestro.cli.util.WorkspaceUtils diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index ce44143870..585b4c3916 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -43,7 +43,7 @@ import maestro.cli.session.MaestroSessionManager import maestro.cli.util.EnvUtils import maestro.cli.util.FileUtils.isWebFlow import maestro.cli.util.PrintUtils -import maestro.cli.util.TestAnalysisReporter +import maestro.cli.insights.TestAnalysisManager import maestro.cli.view.box import maestro.orchestra.error.ValidationError import maestro.orchestra.util.Env.withDefaultEnvVars @@ -304,7 +304,7 @@ class TestCommand : Callable { suites.mergeSummaries()?.saveReport() if (effectiveShards > 1) printShardsMessage(passed, total, suites) - if (analyze) TestAnalysisReporter(apiUrl = apiUrl, apiKey = apiKey).runAnalysis(debugOutputPath) + if (analyze) TestAnalysisManager(apiUrl = apiUrl, apiKey = apiKey).runAnalysis(debugOutputPath) if (passed == total) 0 else 1 } diff --git a/maestro-cli/src/main/java/maestro/cli/insights/Insights.kt b/maestro-cli/src/main/java/maestro/cli/insights/Insights.kt deleted file mode 100644 index cd72bc1483..0000000000 --- a/maestro-cli/src/main/java/maestro/cli/insights/Insights.kt +++ /dev/null @@ -1,31 +0,0 @@ -package maestro.cli.insights - -import maestro.cli.view.box - -/** - * The Insights helper for Maestro CLI. - * - Uses MAESTRO_CLI_INSIGHTS_NOTIFICATION_DISABLED env var to disable insights notifications - */ -object Insights { - - private const val DISABLE_INSIGHTS_ENV_VAR = "MAESTRO_CLI_INSIGHTS_NOTIFICATION_DISABLED" - private val disabled: Boolean - get() = System.getenv(DISABLE_INSIGHTS_ENV_VAR) == "true" - - fun maybeNotifyInsights() { - if (disabled) return - - println() - println( - listOf( - "Tryout our new Analyze with Ai feature.\n", - "See what's new:", - // TODO: Add final link to analyze with Ai Docs - "https://github.com/mobile-dev-inc/maestro/blob/main/CHANGELOG.md#blaaa", - "Analyze command:", - "maestro analyze android-flow.yaml | bash\n", - "To disable this notification, set $DISABLE_INSIGHTS_ENV_VAR environment variable to \"true\" before running Maestro." - ).joinToString("\n").box() - ) - } -} diff --git a/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt b/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt new file mode 100644 index 0000000000..2bd1d05f37 --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/insights/TestAnalysisManager.kt @@ -0,0 +1,142 @@ +package maestro.cli.insights + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors +import maestro.cli.api.ApiClient +import maestro.cli.cloud.CloudInteractor +import maestro.cli.util.EnvUtils +import maestro.cli.util.PrintUtils +import maestro.cli.view.box +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.readText +import kotlin.io.path.writeText + +data class FlowFiles( + val imageFiles: List>, + val textFiles: List> +) + +class TestAnalysisManager(private val apiUrl: String, private val apiKey: String?) { + private val apiclient by lazy { + ApiClient(apiUrl) + } + + fun runAnalysis(debugOutputPath: Path): Int { + val flowFiles = processFilesByFlowName(debugOutputPath) + if (flowFiles.isEmpty()) { + PrintUtils.warn("No screenshots or debug artifacts found for analysis.") + return 0; + } + + return CloudInteractor(apiclient).analyze( + apiKey = apiKey, + flowFiles = flowFiles, + debugOutputPath = debugOutputPath + ) + } + + private fun processFilesByFlowName(outputPath: Path): List { + val files = Files.walk(outputPath) + .filter(Files::isRegularFile) + .collect(Collectors.toList()) + + return if (files.isNotEmpty()) { + val (imageFiles, textFiles) = getDebugFiles(files) + listOf( + FlowFiles( + imageFiles = imageFiles, + textFiles = textFiles + ) + ) + } else { + emptyList() + } + } + + private fun getDebugFiles(files: List): Pair>, List>> { + val imageFiles = mutableListOf>() + val textFiles = mutableListOf>() + + files.forEach { filePath -> + val content = Files.readAllBytes(filePath) + val fileName = filePath.fileName.toString().lowercase() + + when { + fileName.endsWith(".png") || fileName.endsWith(".jpg") || fileName.endsWith(".jpeg") -> { + imageFiles.add(content to filePath) + } + + fileName.startsWith("commands") -> { + textFiles.add(content to filePath) + } + + fileName == "maestro.log" -> { + textFiles.add(content to filePath) + } + } + } + + return Pair(imageFiles, textFiles) + } + + /** + * The analytics system for Test Analysis. + * - Uses configuration from $XDG_CONFIG_HOME/maestro/analyze-analytics.json. + */ + companion object Analytics { + private const val DISABLE_INSIGHTS_ENV_VAR = "MAESTRO_CLI_INSIGHTS_NOTIFICATION_DISABLED" + private val disabled: Boolean + get() = System.getenv(DISABLE_INSIGHTS_ENV_VAR) == "true" + + private val analyticsStatePath: Path = EnvUtils.xdgStateHome().resolve("analyze-analytics.json") + + private val JSON = jacksonObjectMapper().apply { + registerModule(JavaTimeModule()) + enable(SerializationFeature.INDENT_OUTPUT) + } + + private val shouldNotNotify: Boolean + get() = disabled || analyticsStatePath.exists() && analyticsState.acknowledged + + private val analyticsState: AnalyticsState + get() = JSON.readValue(analyticsStatePath.readText()) + + fun maybeNotify() { + if (shouldNotNotify) return + + println( + listOf( + "Tryout our new Analyze with Ai feature.\n", + "See what's new:", + "> https://maestro.mobile.dev/cli/test-suites-and-reports#analyze", + "Analyze command:", + "$ maestro test flow-file.yaml --analyze | bash\n", + "To disable this notification, set $DISABLE_INSIGHTS_ENV_VAR environment variable to \"true\" before running Maestro." + ).joinToString("\n").box() + ) + ack(); + } + + private fun ack() { + val state = AnalyticsState( + acknowledged = true + ) + + val stateJson = JSON.writeValueAsString(state) + analyticsStatePath.parent.createDirectories() + analyticsStatePath.writeText(stateJson + "\n") + } + } +} + +@JsonIgnoreProperties(ignoreUnknown = true) +data class AnalyticsState( + val acknowledged: Boolean = false +) diff --git a/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt b/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt deleted file mode 100644 index 6f3afbaf8f..0000000000 --- a/maestro-cli/src/main/java/maestro/cli/util/TestAnalysisReporter.kt +++ /dev/null @@ -1,76 +0,0 @@ -package maestro.cli.util - -import java.nio.file.Files -import java.nio.file.Path -import java.util.stream.Collectors -import maestro.cli.api.ApiClient -import maestro.cli.cloud.CloudInteractor - -data class FlowFiles( - val imageFiles: List>, - val textFiles: List> -) - -class TestAnalysisReporter(private val apiUrl: String, private val apiKey: String?) { - private val apiclient by lazy { - ApiClient(apiUrl) - } - - fun runAnalysis(debugOutputPath: Path): Int { - val flowFiles = processFilesByFlowName(debugOutputPath) - if (flowFiles.isEmpty()) { - PrintUtils.warn("No screenshots or debug artifacts found for analysis.") - return 0; - } - - return CloudInteractor(apiclient).analyze( - apiKey = apiKey, - flowFiles = flowFiles, - debugOutputPath = debugOutputPath - ) - } - - private fun processFilesByFlowName(outputPath: Path): List { - val files = Files.walk(outputPath) - .filter(Files::isRegularFile) - .collect(Collectors.toList()) - - return if (files.isNotEmpty()) { - val (imageFiles, textFiles) = getDebugFiles(files) - listOf( - FlowFiles( - imageFiles = imageFiles, - textFiles = textFiles - ) - ) - } else { - emptyList() - } - } - - private fun getDebugFiles(files: List): Pair>, List>> { - val imageFiles = mutableListOf>() - val textFiles = mutableListOf>() - - files.forEach { filePath -> - val content = Files.readAllBytes(filePath) - val fileName = filePath.fileName.toString().lowercase() - - when { - fileName.endsWith(".png") || fileName.endsWith(".jpg") || fileName.endsWith(".jpeg") -> { - imageFiles.add(content to filePath) - } - - fileName.startsWith("commands") -> { - textFiles.add(content to filePath) - } - - fileName == "maestro.log" -> { - textFiles.add(content to filePath) - } - } - } - - return Pair(imageFiles, textFiles) - } -}