diff --git a/.github/workflows/test-e2e.yaml b/.github/workflows/test-e2e.yaml index bc525a603d..c6675563d0 100644 --- a/.github/workflows/test-e2e.yaml +++ b/.github/workflows/test-e2e.yaml @@ -165,7 +165,7 @@ jobs: - name: Run tests working-directory: ${{ github.workspace }}/e2e - timeout-minutes: 10 + timeout-minutes: 20 run: ./run_tests android - name: Stop screen recording of AVD diff --git a/e2e/run_tests b/e2e/run_tests index 8c80723238..0e3f09769e 100755 --- a/e2e/run_tests +++ b/e2e/run_tests @@ -10,15 +10,15 @@ command -v maestro >/dev/null 2>&1 || { echo "maestro is required" && exit 1; } ALL_PASS=true _h1() { - printf "=>\n=> $1\n=>\n" + printf "=>\n=> %s\n=>\n" "$1" } _h2() { - printf "==> [$1] $2\n" + printf "==> [%s] %s\n" "$1" "$2" } _h3() { - printf "==> [$1] [$2] => $3\n" + printf "==> [%s] [%s] => %s\n" "$1" "$2" "$3" } platform="${1:-}" diff --git a/e2e/workspaces/demo_app/commands/assertNotVisible.yaml b/e2e/workspaces/demo_app/commands/assertNotVisible.yaml new file mode 100644 index 0000000000..c55b0fe6cc --- /dev/null +++ b/e2e/workspaces/demo_app/commands/assertNotVisible.yaml @@ -0,0 +1,13 @@ +appId: com.example.example +--- + +- launchApp # For idempotence of sections + +- assertNotVisible: 'kwyjibo' + +- assertNotVisible: + text: 'kwyjibo' + +- assertNotVisible: + text: 'Form Test' + enabled: false \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/assertTrue.yaml b/e2e/workspaces/demo_app/commands/assertTrue.yaml new file mode 100644 index 0000000000..8b4b364f6e --- /dev/null +++ b/e2e/workspaces/demo_app/commands/assertTrue.yaml @@ -0,0 +1,12 @@ +appId: com.example.example +--- + +- launchApp # For idempotence of sections + +- assertTrue: ${"test" == "test"} + +- assertTrue: + condition: ${12 < 20} + +- assertTrue: + condition: ${THING == "five"} # Using the env at the top of the file \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/assertVisible.yaml b/e2e/workspaces/demo_app/commands/assertVisible.yaml new file mode 100644 index 0000000000..4a4a91cf2f --- /dev/null +++ b/e2e/workspaces/demo_app/commands/assertVisible.yaml @@ -0,0 +1,12 @@ +appId: com.example.example +--- + +- launchApp # For idempotence of sections + +- assertVisible: 'Form Test' + +- assertVisible: + text: 'Form Test' + +- assertVisible: + id: 'fabAddIcon' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/back.yaml b/e2e/workspaces/demo_app/commands/back.yaml new file mode 100644 index 0000000000..c937911a32 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/back.yaml @@ -0,0 +1,8 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Form Test' +- assertVisible: 'Login' +- back +- assertVisible: 'Form Test' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/copyTextFrom.yaml b/e2e/workspaces/demo_app/commands/copyTextFrom.yaml new file mode 100644 index 0000000000..19c8f3c5da --- /dev/null +++ b/e2e/workspaces/demo_app/commands/copyTextFrom.yaml @@ -0,0 +1,10 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: + id: 'fabAddIcon' + retryTapIfNoChange: false +- copyTextFrom: + text: '\d+' +- assertTrue: ${maestro.copiedText == '1'} \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/eraseText.yaml b/e2e/workspaces/demo_app/commands/eraseText.yaml new file mode 100644 index 0000000000..c3a79feff7 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/eraseText.yaml @@ -0,0 +1,18 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Form Test' +- tapOn: 'Email' +- inputText: 'foo' +- assertVisible: 'foo' +- eraseText +- assertNotVisible: 'foo' + +- inputText: 'testing' +- assertVisible: 'testing' +- eraseText: 3 +- assertNotVisible: 'testing' +- assertVisible: + text: 'test' + optional: true # FIXME: This still takes an extra character sometimes, even after #2123 diff --git a/e2e/workspaces/demo_app/commands/evalScript.yaml b/e2e/workspaces/demo_app/commands/evalScript.yaml new file mode 100644 index 0000000000..06d99f7088 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/evalScript.yaml @@ -0,0 +1,7 @@ +appId: com.example.example +--- + +- launchApp # For idempotence of sections + +- evalScript: ${output.test = 'foo'} +- assertTrue: ${output.test == 'foo'} \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/extendedWaitUntil.yaml b/e2e/workspaces/demo_app/commands/extendedWaitUntil.yaml new file mode 100644 index 0000000000..69c3081b65 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/extendedWaitUntil.yaml @@ -0,0 +1,13 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- extendedWaitUntil: + timeout: 10000 + visible: + text: 'Swipe Test' + +- extendedWaitUntil: + timeout: 100 + notVisible: + text: 'Non Existent Text' diff --git a/e2e/workspaces/demo_app/commands/hideKeyboard.yaml b/e2e/workspaces/demo_app/commands/hideKeyboard.yaml new file mode 100644 index 0000000000..cce913684a --- /dev/null +++ b/e2e/workspaces/demo_app/commands/hideKeyboard.yaml @@ -0,0 +1,11 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Form Test' +- tapOn: 'Email' +- assertVisible: + id: com.google.android.inputmethod.latin:id/key_pos_shift # The shift key on the Android keyboard +- hideKeyboard +- assertNotVisible: + id: com.google.android.inputmethod.latin:id/key_pos_shift \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/inputRandomEmail.yaml b/e2e/workspaces/demo_app/commands/inputRandomEmail.yaml new file mode 100644 index 0000000000..fd8738dd1f --- /dev/null +++ b/e2e/workspaces/demo_app/commands/inputRandomEmail.yaml @@ -0,0 +1,9 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Input Test' +- tapOn: + id: 'textInput' +- inputRandomEmail +- assertVisible: '.+@.+' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/inputRandomNumber.yaml b/e2e/workspaces/demo_app/commands/inputRandomNumber.yaml new file mode 100644 index 0000000000..56d85d5733 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/inputRandomNumber.yaml @@ -0,0 +1,18 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Input Test' +- tapOn: + id: 'textInput' +- inputRandomNumber +- assertVisible: + text: '\d{8}' # The default length is 8 + id: 'textInput' + +- eraseText +- inputRandomNumber: + length: 4 +- assertVisible: + text: '\d{4}' + id: 'textInput' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/inputRandomPersonName.yaml b/e2e/workspaces/demo_app/commands/inputRandomPersonName.yaml new file mode 100644 index 0000000000..1bcbc76ff2 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/inputRandomPersonName.yaml @@ -0,0 +1,11 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Input Test' +- tapOn: + id: 'textInput' +- inputRandomPersonName +- assertVisible: + text: '[A-Z][a-z]+ [A-Z][a-z]+' + id: 'textInput' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/inputRandomText.yaml b/e2e/workspaces/demo_app/commands/inputRandomText.yaml new file mode 100644 index 0000000000..d570f0dd54 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/inputRandomText.yaml @@ -0,0 +1,18 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Input Test' +- tapOn: + id: 'textInput' +- inputRandomText +- assertVisible: + text: '[a-z0-9]{8}' # The default length is 8 + id: 'textInput' + +- eraseText +- inputRandomText: + length: 4 +- assertVisible: + text: '[a-z0-9]{4}' + id: 'textInput' diff --git a/e2e/workspaces/demo_app/commands/inputText.yaml b/e2e/workspaces/demo_app/commands/inputText.yaml new file mode 100644 index 0000000000..8575a2a30b --- /dev/null +++ b/e2e/workspaces/demo_app/commands/inputText.yaml @@ -0,0 +1,9 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- tapOn: 'Input Test' +- tapOn: + id: 'textInput' +- inputText: 'foo' +- assertVisible: 'foo' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/killApp.yaml b/e2e/workspaces/demo_app/commands/killApp.yaml new file mode 100644 index 0000000000..a66be89fc5 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/killApp.yaml @@ -0,0 +1,7 @@ +appId: com.example.example +--- + +- launchApp +- assertVisible: 'Form Test' +- killApp +- assertNotVisible: 'Form Test' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/launchApp.yaml b/e2e/workspaces/demo_app/commands/launchApp.yaml new file mode 100644 index 0000000000..26ae4406c6 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/launchApp.yaml @@ -0,0 +1,14 @@ +appId: com.example.example +--- +- launchApp: + appId: com.example.example + clearState: true + clearKeychain: true + stopApp: true + permissions: + all: allow +- assertVisible: 'Form Test' + +- stopApp +- launchApp +- assertVisible: 'Form Test' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/pasteText.yaml b/e2e/workspaces/demo_app/commands/pasteText.yaml new file mode 100644 index 0000000000..e8f9a40452 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/pasteText.yaml @@ -0,0 +1,13 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- evalScript: ${maestro.copiedText = 'foo'} +- tapOn: 'Input Test' +- tapOn: + id: 'textInput' +- inputText: 'foo' +- copyTextFrom: + id: 'textInput' +- pasteText +- assertVisible: 'foofoo' \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/pressKey.yaml b/e2e/workspaces/demo_app/commands/pressKey.yaml new file mode 100644 index 0000000000..3f5d78d29e --- /dev/null +++ b/e2e/workspaces/demo_app/commands/pressKey.yaml @@ -0,0 +1,15 @@ +appId: com.example.example +--- +- launchApp + +- assertVisible: 'Form Test' +- pressKey: 'Home' +- assertNotVisible: 'Form Test' + +- launchApp + +- assertVisible: 'Form Test' +- tapOn: 'Form Test' +- assertNotVisible: 'Form Test' +- pressKey: 'Back' +- assertVisible: 'Form Test' diff --git a/e2e/workspaces/demo_app/commands/repeat.yaml b/e2e/workspaces/demo_app/commands/repeat.yaml new file mode 100644 index 0000000000..490474474a --- /dev/null +++ b/e2e/workspaces/demo_app/commands/repeat.yaml @@ -0,0 +1,19 @@ +appId: com.example.example +--- +- launchApp # For idempotence of sections + +- assertVisible: '0' +- repeat: + times: 3 + commands: + - tapOn: + id: 'fabAddIcon' +- assertVisible: '3' +- assertNotVisible: '0' + +- evalScript: ${output.counter = 0} +- repeat: + while: + true: ${output.counter < 3} + commands: + - evalScript: ${output.counter = output.counter + 1} \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/runFlow.yaml b/e2e/workspaces/demo_app/commands/runFlow.yaml new file mode 100644 index 0000000000..0e1c2b5ad9 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/runFlow.yaml @@ -0,0 +1,20 @@ +appId: com.example.example +--- + +# runFlow with file: isn't included since it's in the root flow + +- launchApp # For idempotence of sections + +- runFlow: + commands: + - evalScript: ${output.test = 'bar'} + - assertTrue: ${output.test == 'bar'} + - tapOn: + id: 'fabAddIcon' +- assertVisible: '1' + +- runFlow: + env: + THIS_THING: "six" + commands: + - assertTrue: ${THIS_THING == "six"} \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/runScript.js b/e2e/workspaces/demo_app/commands/runScript.js new file mode 100644 index 0000000000..a3c2c1ff46 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/runScript.js @@ -0,0 +1,5 @@ +if (THIS_THING == "six"){ + output.something = "foo" +} else { + output.something = "bar" +} \ No newline at end of file diff --git a/e2e/workspaces/demo_app/commands/runScript.yaml b/e2e/workspaces/demo_app/commands/runScript.yaml new file mode 100644 index 0000000000..32fdb650b9 --- /dev/null +++ b/e2e/workspaces/demo_app/commands/runScript.yaml @@ -0,0 +1,23 @@ +appId: com.example.example +--- + +- launchApp # For idempotence of sections + +- evalScript: ${output.something = 'baz'} + +- runScript: runScript.js +- assertTrue: ${output.something == 'bar'} + +- evalScript: ${output.something = 'baz'} + +- runScript: + file: runScript.js +- assertTrue: ${output.something == 'bar'} + +- evalScript: ${output.something = 'baz'} + +- runScript: + env: + THIS_THING: "six" + file: runScript.js +- assertTrue: ${output.something == 'foo'} diff --git a/e2e/workspaces/demo_app/commands_tour.yaml b/e2e/workspaces/demo_app/commands_tour.yaml new file mode 100644 index 0000000000..80b3ea1990 --- /dev/null +++ b/e2e/workspaces/demo_app/commands_tour.yaml @@ -0,0 +1,144 @@ +# This flow exercises as many commands as possible, using as many configurations as possible. +appId: com.example.example +tags: + - passing + - android # TODO: Make this iOS compatible (or skip platform-specific tests) +env: + THING: "five" + RUN_ONLY: "" # Set to a command name to run only that command +--- + +# TODO: addMedia + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "assertNotVisible"} + file: commands/assertNotVisible.yaml + label: assertNotVisible + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "assertTrue"} + file: commands/assertTrue.yaml + label: assertTrue + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "assertVisible"} + file: commands/assertVisible.yaml + label: assertVisible + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "back"} + file: commands/back.yaml + label: back + +# TODO: clearKeychain + +# TODO: clearState + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "copyTextFrom"} + file: commands/copyTextFrom.yaml + label: copyTextFrom + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "evalScript"} + file: commands/evalScript.yaml + label: evalScript + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "eraseText"} + file: commands/eraseText.yaml + label: eraseText + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "extendedWaitUntil"} + file: commands/extendedWaitUntil.yaml + label: extendedWaitUntil + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "hideKeyboard"} + file: commands/hideKeyboard.yaml + label: hideKeyboard + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "inputText"} + file: commands/inputText.yaml + label: inputText + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "inputRandomEmail"} + file: commands/inputRandomEmail.yaml + label: inputRandomEmail + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "inputRandomPersonName"} + file: commands/inputRandomPersonName.yaml + label: inputRandomPersonName + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "inputRandomPhoneNumber"} + file: commands/inputRandomNumber.yaml + label: inputRandomNumber + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "inputRandomText"} + file: commands/inputRandomText.yaml + label: inputRandomText + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "killApp"} + file: commands/killApp.yaml + label: killApp + optional: true # FIXME: Why is this failing? + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "launchApp"} + file: commands/launchApp.yaml + label: launchApp + +# TODO: openLink (probably after #2058) + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "pressKey"} + file: commands/pressKey.yaml + label: pressKey + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "pasteText"} + file: commands/pasteText.yaml + label: pasteText + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "repeat"} + file: commands/repeat.yaml + label: repeat + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "runFlow"} + file: commands/runFlow.yaml + label: runFlow + +- runFlow: + when: + true: ${RUN_ONLY == "" || RUN_ONLY == "runScript"} + file: commands/runScript.yaml + label: runScript diff --git a/e2e/workspaces/demo_app/fail_launchApp.yaml b/e2e/workspaces/demo_app/fail_launchApp.yaml new file mode 100644 index 0000000000..64125f6646 --- /dev/null +++ b/e2e/workspaces/demo_app/fail_launchApp.yaml @@ -0,0 +1,5 @@ +appId: com.nonexistent +tags: + - failing +--- +- launchApp diff --git a/e2e/workspaces/demo_app/fail_launchApp_nonDefault.yaml b/e2e/workspaces/demo_app/fail_launchApp_nonDefault.yaml new file mode 100644 index 0000000000..6cc9b05514 --- /dev/null +++ b/e2e/workspaces/demo_app/fail_launchApp_nonDefault.yaml @@ -0,0 +1,6 @@ +appId: com.example.example +tags: + - failing +--- +- launchApp: + appId: com.nonexistent diff --git a/e2e/workspaces/demo_app/fail_visible.yaml b/e2e/workspaces/demo_app/fail_visible.yaml new file mode 100644 index 0000000000..9b20f0d2d7 --- /dev/null +++ b/e2e/workspaces/demo_app/fail_visible.yaml @@ -0,0 +1,8 @@ +appId: com.example.example +tags: + - failing +--- +- launchApp: + clearState: true +- assertVisible: + id: non-existent-id diff --git a/e2e/workspaces/demo_app/fail_visible_extended.yaml b/e2e/workspaces/demo_app/fail_visible_extended.yaml new file mode 100644 index 0000000000..85dc35aaa8 --- /dev/null +++ b/e2e/workspaces/demo_app/fail_visible_extended.yaml @@ -0,0 +1,10 @@ +appId: com.example.example +tags: + - failing +--- +- launchApp: + clearState: true +- extendedWaitUntil: + visible: + id: non-existent-id + timeout: 100