PoC gui test automation

Create gui test suite with common setup. Create generic keywords for launching and closing any app with gui test automation. Create example test cases for chromium and firefox. Added test case for logout and login, and test for booting to login screen. Login is included in common setup before launching apps. Log out is added to the common teardown of GUI test suite. Gui tests are expected to be run from logged out state. Logout state at boot is checked as one test case. Graphical details and icons used in image recognition are copied from run-time ghaf gui-vm. Fixed also a bug in iperf test report message showing only tx results. Signed-off-by: Samuli Leivo <[email protected]>
tiiuae · Oct 23, 2024 · 63b02ef · 63b02ef
1 parent 25a03cd
commit 63b02ef
Show file tree

Hide file tree

Showing 10 changed files with 418 additions and 14 deletions.
diff --git a/Robot-Framework/lib/gui_testing.py b/Robot-Framework/lib/gui_testing.py
@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+from pyscreeze import locate, center
+import logging
+import subprocess
+
+
+def locate_image(image, confidence):
+    screenshot = "./screenshot.png"
+    image_box = locate(image, screenshot, confidence=confidence)
+    image_center = center(image_box)
+    logging.info(image_box)
+    logging.info(image_center)
+    image_center_in_mouse_coordinates = convert_resolution(image_center, screenshot)
+    logging.info(image_center_in_mouse_coordinates)
+    return image_center_in_mouse_coordinates
+
+def convert_resolution(coordinates, screenshot):
+    # Currently default screenshot image resolution is 1920x1200
+    # but ydotool mouse movement resolution was tested to be 960x600.
+    # Testing shows that this scaling ratio stays fixed even if changing the display resolution:
+    # ydotool mouse resolution changes in relation to display resolution.
+    # Hence we can use the hardcoded value.
+    scaling_factor = 2
+    mouse_coordinates = {
+        'x': int(coordinates[0] / scaling_factor),
+        'y': int(coordinates[1] / scaling_factor)
+    }
+    return mouse_coordinates
+
+def convert_app_icon(crop, background, input_file='icon.svg', output_file='icon.png'):
+    if background != "none":
+        subprocess.run(['magick', '-background', background, input_file, '-gravity', 'center', '-extent',
+                        '{}x{}'.format(crop, crop), output_file])
+    else:
+        subprocess.run(['magick', input_file, '-gravity', 'center', '-extent',
+                        '{}x{}'.format(crop, crop), output_file])
+    return
+
+def negate_app_icon(input_file, output_file):
+    subprocess.run(['magick', input_file, '-negate', output_file])
diff --git a/Robot-Framework/resources/common_keywords.resource b/Robot-Framework/resources/common_keywords.resource
@@ -19,8 +19,29 @@ Check that the application was started
     Should Not Be Empty  ${app_pids}  ${app_name} is not started
     Log To Console       ${app_name} is started
 
+Check that the application is not running
+    [Arguments]          ${app_name}  ${range}=2
+    ${pids}=  Set Variable  ${EMPTY}
+    FOR   ${i}   IN RANGE  ${range}
+        ${keyword_status}  ${pids}  Run Keyword And Ignore Error   Find pid by name    ${app_name}
+        ${status}    Run Keyword And Return Status   Should Be Empty  ${pids}
+        IF    ${status}    BREAK
+        Sleep   1
+    END
+    Should Be Empty   ${pids}   ${app_name} is still running
+    Log To Console    ${app_name} not running
+
 Check If Ping Fails
     [Documentation]  Check that ping is not getting response from host
-    # ${out}   Run and Return RC   ping ${DEVICE_IP_ADDRESS} -c 1
     ${result}  Run Process  ping  ${DEVICE_IP_ADDRESS}  -c1  timeout=1s
-    Should Not Be Equal   ${result.rc}  ${0}
+    Should Not Be Equal   ${result.rc}  ${0}
+
+Run journalctl recording
+    ${output}     Execute Command    journalctl > jrnl.txt
+    ${output}     Execute Command    nohup journalctl -f >> jrnl.txt 2>&1 &
+
+Log journctl
+    ${output}     Execute Command    cat jrnl.txt
+    Log           ${output}
+    @{pid}        Find pid by name   journalctl
+    Kill process  @{pid}
diff --git a/Robot-Framework/resources/gui_keywords.resource b/Robot-Framework/resources/gui_keywords.resource
@@ -0,0 +1,120 @@
+# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+*** Settings ***
+Resource            ../config/variables.robot
+Library             ../lib/gui_testing.py
+Library             Collections
+
+*** Variables ***
+
+${start_menu}       ./launcher.png
+
+*** Keywords ***
+
+GUI Log in
+    [Documentation]   Login and verify that task bar is available
+    Start ydotoold
+    Log To Console    Typing username and password to login
+    Type string and press enter  ${LOGIN}
+    Type string and press enter  ${PASSWORD}
+    Log To Console    Check if menu icon is available on desktop
+    Locate image on screen  ${start_menu}  0.95  15
+
+GUI Log out
+    [Documentation]   Log out and optionally verify that desktop is not available
+    [Arguments]        ${log_out_icon}=./logout.png
+    Start ydotoold
+    Get icon           ghaf-artwork  power.svg  crop=0  background=black
+    Locate and click   ./icon.png  0.95  5
+    Get icon           ghaf-artwork  logout.svg  crop=0  background=black
+    Locate and click   ./icon.png  0.95  5
+    Verify logout
+    Run Keyword If     ${LOGGED_IN_STATUS}  FAIL   Log out failed. Desktop is still running.
+
+Type string and press enter
+    [Arguments]   ${string}
+    Log To Console    Typing
+    Execute Command   ydotool type ${string}  sudo=True  sudo_password=${PASSWORD}
+    Log To Console    Pressing Enter
+    Execute Command   ydotool key -d 0 28:1 28:0  sudo=True  sudo_password=${PASSWORD}
+
+Locate image on screen
+    [Documentation]    Take a screenshot. Locate given image on the screenshot.
+    ...                Return center coordinates of the image in mouse coordinate system
+    [Arguments]        ${image_to_be_searched}  ${confidence}=0.999   ${iterations}=5
+    ${coordinates}=  Set Variable  ${EMPTY}
+    ${pass_status}=  Set Variable  FAIL
+    FOR   ${i}   IN RANGE  ${iterations}
+        Log To Console    Taking screenshot
+        Execute Command   rm screenshot.png
+        ${rc} =    Execute Command  grim screenshot.png  return_stdout=False  return_rc=${true}
+        IF  "${rc}" == "0"
+            SSHLibrary.Get File  screenshot.png   screenshot.png
+            Log To Console    Locating image ${image_to_be_searched} on screenshot
+            ${pass_status}  ${coordinates}  Run Keyword And Ignore Error  Locate image  ${image_to_be_searched}   ${confidence}
+        END
+        IF    $pass_status=='PASS'    BREAK
+        Sleep  0.5
+    END
+    IF    $pass_status=='FAIL'    FAIL  Image recognition failure: ${image_to_be_searched}
+    Log To Console    Coordinates: ${coordinates}
+    ${mouse_x}  Get From Dictionary   ${coordinates}  x
+    ${mouse_y}  Get From Dictionary   ${coordinates}  y
+    RETURN  ${mouse_x}  ${mouse_y}
+
+Locate and click
+    [Arguments]   ${image_to_be_searched}  ${confidence}=0.99  ${iterations}=5
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ${image_to_be_searched}   ${confidence}
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+
+Start ydotoold
+    [Documentation]    Start ydotool daemon if it is not already running.
+    ${ydotoold_state}=    Execute Command    sh -c 'ps aux | grep ydotoold | grep -v grep'
+    IF  $ydotoold_state == '${EMPTY}'
+        Log To Console    Starting ydotool daemon
+        Run Keyword And Ignore Error  Execute Command   -b /run/current-system/sw/bin/ydotoold --socket-path /tmp/.ydotool_socket  sudo=True  sudo_password=${PASSWORD}  timeout=3
+        ${ydotoold_state}=    Execute Command    sh -c 'ps aux | grep ydotoold | grep -v grep'
+        Should Not Be Empty  ${ydotoold_state}  failed to start ydotool daemon
+    ELSE
+        Log To Console    Check: ydotool daemon running
+    END
+
+Stop ydotoold
+    [Documentation]    Kill ydotool daemon
+    Log To Console    Stopping ydotool daemon
+    Execute Command   pkill ydotoold  sudo=True  sudo_password=${PASSWORD}
+
+Move cursor to corner
+    [Documentation]    Move the cursor to the upper left corner so that it will not block searching further gui screenshots
+    Log To Console    Moving cursor to corner from blocking further image detection
+    Start ydotoold
+    Execute Command   ydotool mousemove --absolute -x 50 -y 50  sudo=True  sudo_password=${PASSWORD}
+
+Verify logout
+    [Documentation]    Check that dekstop is not available by running 'grim' which should have return code 1 in this case
+    [Arguments]        ${iterations}=5
+    ${status}=  Set Variable  ${EMPTY}
+    FOR   ${i}   IN RANGE  ${iterations}
+        ${rc}=    Execute Command  grim check.png  return_stdout=False  return_rc=${true}
+        IF  "${rc}" == "1"
+            Set Global Variable  ${LOGGED_IN_STATUS}  ${False}
+            BREAK
+        ELSE
+            Set Global Variable  ${LOGGED_IN_STATUS}  ${True}
+        END
+        Sleep  1
+    END
+
+Get icon
+    [Documentation]    Copy icon svg file to test agent machine. Crop and convert the svg file to png.
+    [Arguments]        ${path}   ${icon_name}   ${crop}   ${background}=none   ${output_filename}=icon.png
+    IF  $path == "app"
+        SSHLibrary.Get File  ${APP_ICON_PATH}/${icon_name}   icon.svg
+    ELSE IF  $path == "ghaf-artwork"
+        SSHLibrary.Get File  ${ARTWORK_PATH}/${icon_name}   icon.svg
+    ELSE
+        SSHLibrary.Get File  ${path}/${icon_name}   icon.svg
+    END
+    Convert app icon  ${crop}   ${background}   input_file=icon.svg  output_file=${output_filename}
diff --git a/Robot-Framework/test-suites/bat-tests/__init__.robot b/Robot-Framework/test-suites/bat-tests/__init__.robot
@@ -5,6 +5,7 @@
 Documentation       BAT tests
 Resource            ../../resources/ssh_keywords.resource
 Resource            ../../resources/serial_keywords.resource
+Resource            ../../resources/common_keywords.resource
 Suite Setup         Common Setup
 Suite Teardown      Common Teardown
 
@@ -33,16 +34,6 @@ Common Teardown
     END
     Close All Connections
 
-Run journalctl recording
-    ${output}     Execute Command    journalctl > jrnl.txt
-    ${output}     Execute Command    nohup journalctl -f >> jrnl.txt 2>&1 &
-
-Log journctl
-    ${output}     Execute Command    cat jrnl.txt
-    Log           ${output}
-    @{pid}        Find pid by name   journalctl
-    Kill process  @{pid}
-
 Log versions
     ${ghaf_version}     Execute Command   ghaf-version
     Log to console      Ghaf version: ${ghaf_version}

diff --git a/Robot-Framework/test-suites/bat-tests/apps.robot b/Robot-Framework/test-suites/bat-tests/apps.robot
@@ -7,6 +7,9 @@ Force Tags          apps
 Resource            ../../resources/ssh_keywords.resource
 Resource            ../../config/variables.robot
 Resource            ../../resources/common_keywords.resource
+Library             ../../lib/gui_testing.py
+Library             Collections
+Library             BuiltIn
 Suite Teardown      Close All Connections
 
 

diff --git a/Robot-Framework/test-suites/gui-tests/__init__.robot b/Robot-Framework/test-suites/gui-tests/__init__.robot
@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+*** Settings ***
+Documentation       GUI tests
+Resource            ../../resources/ssh_keywords.resource
+Resource            ../../resources/serial_keywords.resource
+Resource            ../../resources/gui_keywords.resource
+Resource            ../../resources/common_keywords.resource
+Library             ../../lib/gui_testing.py
+Suite Setup         Common Setup
+Suite Teardown      Common Teardown
+
+
+*** Keywords ***
+
+Common Setup
+    Set Variables           ${DEVICE}
+    Run Keyword If          "${DEVICE_IP_ADDRESS}" == ""    Get ethernet IP address
+    ${port_22_is_available}     Check if ssh is ready on device   timeout=180
+    IF  ${port_22_is_available} == False
+        FAIL    Failed because port 22 of device was not available, tests can not be run.
+    END
+    Connect
+    IF  "Lenovo" in "${DEVICE}"
+        Verify service status   range=15  [email protected]  expected_status=active  expected_state=running
+        Connect to netvm
+        Connect to VM           ${GUI_VM}
+    END
+    Run journalctl recording
+    Save most common icons and paths to icons
+    Verify logout
+    Log To Console              LOGGED_IN_STATUS at start
+    Log To Console              ${LOGGED_IN_STATUS}
+    IF  ${LOGGED_IN_STATUS}
+        Log To Console          Already logged in. Skipping login at suite setup.
+    ELSE
+        Log To Console          Logging in
+        GUI Log in
+    END
+
+Common Teardown
+    Connect
+    IF  "Lenovo" in "${DEVICE}"
+        Connect to netvm
+        Connect to VM       ${GUI_VM}
+    END
+    GUI Log out
+    Log journctl
+    Close All Connections
+
+Save most common icons and paths to icons
+    [Documentation]         Save those icons by name which will be used in multiple test cases
+    ...                     Śave paths to icon packs in gui-vm nix store
+    ${adwaita}              Set Variable  /run/current-system/sw/share/icons/Adwaita
+    Log To Console          Saving path to icon-pack
+    ${app_icon_pack_path}   Execute Command   echo /nix/store/$(ls /nix/store | grep icon-pack | tail -1)
+    Set Global Variable     ${APP_ICON_PATH}  ${app_icon_pack_path}
+    Log To Console          ${APP_ICON_PATH}
+    Log To Console          Saving path to ghaf-artwork icons
+    ${ghaf_artwork_path}    Execute Command   echo /nix/store/$(ls /nix/store | grep ghaf-artwork | tail -1)/icons
+    Set Global Variable     ${ARTWORK_PATH}  ${ghaf_artwork_path}
+    Log To Console          ${ARTWORK_PATH}
+    Log To Console          Saving gui icons
+    Get icon                ghaf-artwork  launcher.svg  crop=0  background=black  output_filename=launcher.png
+    Get icon                ${adwaita}/symbolic/ui  window-close-symbolic.svg  crop=0  output_filename=window-close.png  background=white
+    Negate app icon         window-close.png  window-close-neg.png