diff --git a/Robot-Framework/lib/gui_testing.py b/Robot-Framework/lib/gui_testing.py new file mode 100644 index 0000000..325baa5 --- /dev/null +++ b/Robot-Framework/lib/gui_testing.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +from pyscreeze import locate, center +import logging +import subprocess + + +def locate_image(image, confidence): + screenshot = "./screenshot.png" + image_box = locate(image, screenshot, confidence=confidence) + image_center = center(image_box) + logging.info(image_box) + logging.info(image_center) + image_center_in_mouse_coordinates = convert_resolution(image_center, screenshot) + logging.info(image_center_in_mouse_coordinates) + return image_center_in_mouse_coordinates + +def convert_resolution(coordinates, screenshot): + # Currently default screenshot image resolution is 1920x1200 + # but ydotool mouse movement resolution was tested to be 960x600. + # Testing shows that this scaling ratio stays fixed even if changing the display resolution: + # ydotool mouse resolution changes in relation to display resolution. + # Hence we can use the hardcoded value. + scaling_factor = 2 + mouse_coordinates = { + 'x': int(coordinates[0] / scaling_factor), + 'y': int(coordinates[1] / scaling_factor) + } + return mouse_coordinates + +def convert_app_icon(crop, background, input_file='icon.svg', output_file='icon.png'): + if background != "none": + subprocess.run(['magick', '-background', background, input_file, '-gravity', 'center', '-extent', + '{}x{}'.format(crop, crop), output_file]) + else: + subprocess.run(['magick', input_file, '-gravity', 'center', '-extent', + '{}x{}'.format(crop, crop), output_file]) + return + +def negate_app_icon(input_file, output_file): + subprocess.run(['magick', input_file, '-negate', output_file]) diff --git a/Robot-Framework/resources/common_keywords.resource b/Robot-Framework/resources/common_keywords.resource index 2cac671..514b1ba 100644 --- a/Robot-Framework/resources/common_keywords.resource +++ b/Robot-Framework/resources/common_keywords.resource @@ -19,8 +19,29 @@ Check that the application was started Should Not Be Empty ${app_pids} ${app_name} is not started Log To Console ${app_name} is started +Check that the application is not running + [Arguments] ${app_name} ${range}=2 + ${pids}= Set Variable ${EMPTY} + FOR ${i} IN RANGE ${range} + ${keyword_status} ${pids} Run Keyword And Ignore Error Find pid by name ${app_name} + ${status} Run Keyword And Return Status Should Be Empty ${pids} + IF ${status} BREAK + Sleep 1 + END + Should Be Empty ${pids} ${app_name} is still running + Log To Console ${app_name} not running + Check If Ping Fails [Documentation] Check that ping is not getting response from host - # ${out} Run and Return RC ping ${DEVICE_IP_ADDRESS} -c 1 ${result} Run Process ping ${DEVICE_IP_ADDRESS} -c1 timeout=1s - Should Not Be Equal ${result.rc} ${0} \ No newline at end of file + Should Not Be Equal ${result.rc} ${0} + +Run journalctl recording + ${output} Execute Command journalctl > jrnl.txt + ${output} Execute Command nohup journalctl -f >> jrnl.txt 2>&1 & + +Log journctl + ${output} Execute Command cat jrnl.txt + Log ${output} + @{pid} Find pid by name journalctl + Kill process @{pid} \ No newline at end of file diff --git a/Robot-Framework/resources/gui_keywords.resource b/Robot-Framework/resources/gui_keywords.resource new file mode 100644 index 0000000..039f4e5 --- /dev/null +++ b/Robot-Framework/resources/gui_keywords.resource @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Resource ../config/variables.robot +Library ../lib/gui_testing.py +Library Collections + +*** Variables *** + +${start_menu} ./launcher.png + +*** Keywords *** + +GUI Log in + [Documentation] Login and verify that task bar is available + Start ydotoold + Log To Console Typing username and password to login + Type string and press enter ${LOGIN} + Type string and press enter ${PASSWORD} + Log To Console Check if menu icon is available on desktop + Locate image on screen ${start_menu} 0.95 15 + +GUI Log out + [Documentation] Log out and optionally verify that desktop is not available + [Arguments] ${log_out_icon}=./logout.png + Start ydotoold + Get icon ghaf-artwork power.svg crop=0 background=black + Locate and click ./icon.png 0.95 5 + Get icon ghaf-artwork logout.svg crop=0 background=black + Locate and click ./icon.png 0.95 5 + Verify logout + Run Keyword If ${LOGGED_IN_STATUS} FAIL Log out failed. Desktop is still running. + +Type string and press enter + [Arguments] ${string} + Log To Console Typing + Execute Command ydotool type ${string} sudo=True sudo_password=${PASSWORD} + Log To Console Pressing Enter + Execute Command ydotool key -d 0 28:1 28:0 sudo=True sudo_password=${PASSWORD} + +Locate image on screen + [Documentation] Take a screenshot. Locate given image on the screenshot. + ... Return center coordinates of the image in mouse coordinate system + [Arguments] ${image_to_be_searched} ${confidence}=0.999 ${iterations}=5 + ${coordinates}= Set Variable ${EMPTY} + ${pass_status}= Set Variable FAIL + FOR ${i} IN RANGE ${iterations} + Log To Console Taking screenshot + Execute Command rm screenshot.png + ${rc} = Execute Command grim screenshot.png return_stdout=False return_rc=${true} + IF "${rc}" == "0" + SSHLibrary.Get File screenshot.png screenshot.png + Log To Console Locating image ${image_to_be_searched} on screenshot + ${pass_status} ${coordinates} Run Keyword And Ignore Error Locate image ${image_to_be_searched} ${confidence} + END + IF $pass_status=='PASS' BREAK + Sleep 0.5 + END + IF $pass_status=='FAIL' FAIL Image recognition failure: ${image_to_be_searched} + Log To Console Coordinates: ${coordinates} + ${mouse_x} Get From Dictionary ${coordinates} x + ${mouse_y} Get From Dictionary ${coordinates} y + RETURN ${mouse_x} ${mouse_y} + +Locate and click + [Arguments] ${image_to_be_searched} ${confidence}=0.99 ${iterations}=5 + ${mouse_x} ${mouse_y} Locate image on screen ${image_to_be_searched} ${confidence} + Execute Command ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y} sudo=True sudo_password=${PASSWORD} + Execute Command ydotool click 0xC0 sudo=True sudo_password=${PASSWORD} + +Start ydotoold + [Documentation] Start ydotool daemon if it is not already running. + ${ydotoold_state}= Execute Command sh -c 'ps aux | grep ydotoold | grep -v grep' + IF $ydotoold_state == '${EMPTY}' + Log To Console Starting ydotool daemon + Run Keyword And Ignore Error Execute Command -b /run/current-system/sw/bin/ydotoold --socket-path /tmp/.ydotool_socket sudo=True sudo_password=${PASSWORD} timeout=3 + ${ydotoold_state}= Execute Command sh -c 'ps aux | grep ydotoold | grep -v grep' + Should Not Be Empty ${ydotoold_state} failed to start ydotool daemon + ELSE + Log To Console Check: ydotool daemon running + END + +Stop ydotoold + [Documentation] Kill ydotool daemon + Log To Console Stopping ydotool daemon + Execute Command pkill ydotoold sudo=True sudo_password=${PASSWORD} + +Move cursor to corner + [Documentation] Move the cursor to the upper left corner so that it will not block searching further gui screenshots + Log To Console Moving cursor to corner from blocking further image detection + Start ydotoold + Execute Command ydotool mousemove --absolute -x 50 -y 50 sudo=True sudo_password=${PASSWORD} + +Verify logout + [Documentation] Check that dekstop is not available by running 'grim' which should have return code 1 in this case + [Arguments] ${iterations}=5 + ${status}= Set Variable ${EMPTY} + FOR ${i} IN RANGE ${iterations} + ${rc}= Execute Command grim check.png return_stdout=False return_rc=${true} + IF "${rc}" == "1" + Set Global Variable ${LOGGED_IN_STATUS} ${False} + BREAK + ELSE + Set Global Variable ${LOGGED_IN_STATUS} ${True} + END + Sleep 1 + END + +Get icon + [Documentation] Copy icon svg file to test agent machine. Crop and convert the svg file to png. + [Arguments] ${path} ${icon_name} ${crop} ${background}=none ${output_filename}=icon.png + IF $path == "app" + SSHLibrary.Get File ${APP_ICON_PATH}/${icon_name} icon.svg + ELSE IF $path == "ghaf-artwork" + SSHLibrary.Get File ${ARTWORK_PATH}/${icon_name} icon.svg + ELSE + SSHLibrary.Get File ${path}/${icon_name} icon.svg + END + Convert app icon ${crop} ${background} input_file=icon.svg output_file=${output_filename} diff --git a/Robot-Framework/test-suites/bat-tests/__init__.robot b/Robot-Framework/test-suites/bat-tests/__init__.robot index 5a670be..c3bd93a 100644 --- a/Robot-Framework/test-suites/bat-tests/__init__.robot +++ b/Robot-Framework/test-suites/bat-tests/__init__.robot @@ -5,6 +5,7 @@ Documentation BAT tests Resource ../../resources/ssh_keywords.resource Resource ../../resources/serial_keywords.resource +Resource ../../resources/common_keywords.resource Suite Setup Common Setup Suite Teardown Common Teardown @@ -33,16 +34,6 @@ Common Teardown END Close All Connections -Run journalctl recording - ${output} Execute Command journalctl > jrnl.txt - ${output} Execute Command nohup journalctl -f >> jrnl.txt 2>&1 & - -Log journctl - ${output} Execute Command cat jrnl.txt - Log ${output} - @{pid} Find pid by name journalctl - Kill process @{pid} - Log versions ${ghaf_version} Execute Command ghaf-version Log to console Ghaf version: ${ghaf_version} diff --git a/Robot-Framework/test-suites/bat-tests/apps.robot b/Robot-Framework/test-suites/bat-tests/apps.robot index d8cb716..21cc8b3 100644 --- a/Robot-Framework/test-suites/bat-tests/apps.robot +++ b/Robot-Framework/test-suites/bat-tests/apps.robot @@ -7,6 +7,9 @@ Force Tags apps Resource ../../resources/ssh_keywords.resource Resource ../../config/variables.robot Resource ../../resources/common_keywords.resource +Library ../../lib/gui_testing.py +Library Collections +Library BuiltIn Suite Teardown Close All Connections diff --git a/Robot-Framework/test-suites/gui-tests/__init__.robot b/Robot-Framework/test-suites/gui-tests/__init__.robot new file mode 100644 index 0000000..1535269 --- /dev/null +++ b/Robot-Framework/test-suites/gui-tests/__init__.robot @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Documentation GUI tests +Resource ../../resources/ssh_keywords.resource +Resource ../../resources/serial_keywords.resource +Resource ../../resources/gui_keywords.resource +Resource ../../resources/common_keywords.resource +Library ../../lib/gui_testing.py +Suite Setup Common Setup +Suite Teardown Common Teardown + + +*** Keywords *** + +Common Setup + Set Variables ${DEVICE} + Run Keyword If "${DEVICE_IP_ADDRESS}" == "" Get ethernet IP address + ${port_22_is_available} Check if ssh is ready on device timeout=180 + IF ${port_22_is_available} == False + FAIL Failed because port 22 of device was not available, tests can not be run. + END + Connect + IF "Lenovo" in "${DEVICE}" + Verify service status range=15 service=microvm@gui-vm.service expected_status=active expected_state=running + Connect to netvm + Connect to VM ${GUI_VM} + END + Run journalctl recording + Save most common icons and paths to icons + Verify logout + Log To Console LOGGED_IN_STATUS at start + Log To Console ${LOGGED_IN_STATUS} + IF ${LOGGED_IN_STATUS} + Log To Console Already logged in. Skipping login at suite setup. + ELSE + Log To Console Logging in + GUI Log in + END + +Common Teardown + Connect + IF "Lenovo" in "${DEVICE}" + Connect to netvm + Connect to VM ${GUI_VM} + END + GUI Log out + Log journctl + Close All Connections + +Save most common icons and paths to icons + [Documentation] Save those icons by name which will be used in multiple test cases + ... Śave paths to icon packs in gui-vm nix store + ${adwaita} Set Variable /run/current-system/sw/share/icons/Adwaita + Log To Console Saving path to icon-pack + ${app_icon_pack_path} Execute Command echo /nix/store/$(ls /nix/store | grep icon-pack | tail -1) + Set Global Variable ${APP_ICON_PATH} ${app_icon_pack_path} + Log To Console ${APP_ICON_PATH} + Log To Console Saving path to ghaf-artwork icons + ${ghaf_artwork_path} Execute Command echo /nix/store/$(ls /nix/store | grep ghaf-artwork | tail -1)/icons + Set Global Variable ${ARTWORK_PATH} ${ghaf_artwork_path} + Log To Console ${ARTWORK_PATH} + Log To Console Saving gui icons + Get icon ghaf-artwork launcher.svg crop=0 background=black output_filename=launcher.png + Get icon ${adwaita}/symbolic/ui window-close-symbolic.svg crop=0 output_filename=window-close.png background=white + Negate app icon window-close.png window-close-neg.png diff --git a/Robot-Framework/test-suites/gui-tests/gui_apps.robot b/Robot-Framework/test-suites/gui-tests/gui_apps.robot new file mode 100644 index 0000000..e07abcf --- /dev/null +++ b/Robot-Framework/test-suites/gui-tests/gui_apps.robot @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Documentation Testing launching applications via GUI +Force Tags gui +Resource ../../resources/ssh_keywords.resource +Resource ../../config/variables.robot +Resource ../../resources/common_keywords.resource +Resource ../../resources/gui_keywords.resource +Library ../../lib/gui_testing.py +Suite Teardown Close All Connections + + +*** Variables *** + +@{app_pids} ${EMPTY} +${start_menu} ./launcher.png + + +*** Test Cases *** + +Start and close chromium via GUI on LenovoX1 + [Documentation] Start Chromium via GUI test automation and verify related process started + ... Close Chromium via GUI test automation and verify related process stopped + [Tags] SP-T41 lenovo-x1 + Get icon app chromium.svg crop=30 + Start app via GUI on LenovoX1 chromium-vm chromium + Close app via GUI on LenovoX1 chromium-vm chromium ./window-close-neg.png + +Start and close Firefox via GUI on Orin AGX + [Documentation] Passing this test requires that display is connected to the target device + ... Start Firefox via GUI test automation and verify related process started + ... Close Firefox via GUI test automation and verify related process stopped + [Tags] SP-T41 orin-agx + Get icon app firefox.svg crop=30 + Start app via GUI on Orin AGX firefox + Close app via GUI on Orin AGX firefox + +*** Keywords *** + +Start app via GUI on LenovoX1 + [Documentation] Start Application via GUI test automation and verify related process started + [Arguments] ${app-vm} + ... ${app} + ... ${launch_icon}=./icon.png + + Connect + Verify service status range=15 service=microvm@${app-vm}.service expected_status=active expected_state=running + Connect to netvm + Connect to VM ${GUI_VM} + Check if ssh is ready on vm ${app-vm} + + Start ydotoold + + Log To Console Going to click the app menu icon + Locate and click ${start_menu} 0.95 5 + Log To Console Going to click the application launch icon + Locate and click ${launch_icon} 0.95 5 + + Connect to VM ${app-vm} + Check that the application was started ${app} 10 + + [Teardown] Run Keywords Connect to VM ${GUI_VM} + ... AND Move cursor to corner + +Close app via GUI on LenovoX1 + [Documentation] Close Application via GUI test automation and verify related process stopped + [Arguments] ${app-vm} + ... ${app} + ... ${close_button}=./window-close.png + + Connect to netvm + Connect to VM ${app-vm} + Check that the application was started ${app} + Connect to VM ${GUI_VM} + Start ydotoold + + Log To Console Going to click the close button of the application window + Locate and click ${close_button} 0.85 5 + + Connect to VM ${app-vm} + Check that the application is not running ${app} 5 + + # In case closing the app via GUI failed + [Teardown] Run Keywords Kill process @{app_pids} + ... AND Connect to VM ${GUI_VM} + ... AND Move cursor to corner + ... AND Stop ydotoold + +Start app via GUI on Orin AGX + [Documentation] Start Application via GUI test automation and verify related process started + ... Only for ghaf builds where desktop is running on ghaf-host + [Arguments] ${app}=firefox + ... ${launch_icon}=../gui-ref-images/${app}/launch_icon.png + + Connect + + Start ydotoold + + Log To Console Going to click the app menu icon + Locate and click ${start_menu} 0.95 5 + Log To Console Going to click the application launch icon + Locate and click ${launch_icon} 0.95 5 + + Check that the application was started ${app} 10 + + [Teardown] Run Keywords Move cursor to corner + +Close app via GUI on Orin AGX + [Documentation] Close Application via GUI test automation and verify related process stopped + ... Only for ghaf builds where desktop is running on ghaf-host + [Arguments] ${app}=firefox + ... ${close_button}=../gui-ref-images/${app}/close_button.png + + Connect + Check that the application was started ${app} + Start ydotoold + + Log To Console Going to click the close button of the application window + Locate and click ${close_button} 0.999 5 + + Check that the application is not running ${app} 5 + + # In case closing the app via GUI failed + [Teardown] Run Keywords Kill process @{app_pids} + ... AND Move cursor to corner + ... AND Stop ydotoold diff --git a/Robot-Framework/test-suites/gui-tests/gui_login.robot b/Robot-Framework/test-suites/gui-tests/gui_login.robot new file mode 100644 index 0000000..3d76951 --- /dev/null +++ b/Robot-Framework/test-suites/gui-tests/gui_login.robot @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Documentation Testing logout and login via GUI +Force Tags gui +Resource ../../resources/ssh_keywords.resource +Resource ../../config/variables.robot +Resource ../../resources/gui_keywords.resource +Suite Teardown Close All Connections + + +*** Test Cases *** + +Boot to login screen + [Documentation] Check that device booted to login screen + [Tags] bat lenovo-x1 SP-T2 + Run Keyword If ${LOGGED_IN_STATUS} FAIL Desktop was detected at setup. Device failed to boot to login screen. + +Log out and log in + [Tags] bat lenovo-x1 SP-T149 + Connect + IF "Lenovo" in "${DEVICE}" + Connect to netvm + Connect to VM ${GUI_VM} + END + GUI Log out + GUI Log in \ No newline at end of file diff --git a/Robot-Framework/test-suites/performance-tests/network.robot b/Robot-Framework/test-suites/performance-tests/network.robot index dce116f..fe28523 100644 --- a/Robot-Framework/test-suites/performance-tests/network.robot +++ b/Robot-Framework/test-suites/performance-tests/network.robot @@ -37,7 +37,7 @@ TCP speed test ${fail_msg}= Set Variable TX:\n${add_msg} END IF "${statistics_rx}[flag]" == "-1" - ${add_msg} Create fail message ${statistics_tx} + ${add_msg} Create fail message ${statistics_rx} ${fail_msg}= Set Variable ${fail_msg}\nRX:\n${add_msg} END IF "${statistics_tx}[flag]" == "-1" or "${statistics_rx}[flag]" == "-1" @@ -50,7 +50,7 @@ TCP speed test ${pass_msg}= Set Variable TX:\n${add_msg} END IF "${statistics_rx}[flag]" == "1" - ${add_msg} Create improved message ${statistics_tx} + ${add_msg} Create improved message ${statistics_rx} ${pass_msg}= Set Variable ${pass_msg}\nRX:\n${add_msg} END IF "${statistics_tx}[flag]" == "1" or "${statistics_rx}[flag]" == "1" diff --git a/flake.nix b/flake.nix index 3f82f55..d02fd8a 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,8 @@ devShell = pkgs.mkShell { buildInputs = with pkgs; [ iperf + file + imagemagick (python3.withPackages (ps: with ps; [ robotframework @@ -56,6 +58,8 @@ python-kasa pytz pandas + pyscreeze + python3Packages.opencv4 ])) ]; };