diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 0000000..a99bb8f --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,63 @@ +A Robot Framework tests suite for automating the validation of the +meta-wpe-image images. + +# Installation + +We have the `install-requirements.sh` and `podman-compose.sh` scripts in the +project.The first one is a convenient script for installing the Podman +requirements. The second, is a wrapper for execute the podman-compose command +but with the environment variables defined in the setup-env.sh. + +``` sh +./install-requirements.sh +cp setup-env-local.sh.sample setup-env-local.sh # Use an editor for adapt the content +``` + +A sample environment setup file (`setup-env-local.sh.sample`) is provided to +guide the initial configuration. It sets the variables for the test board and +network configurations adapted to your environment. + +## How It Works + +To set up the testing environment, run: + +```sh +./podman-compose.sh up --force-recreate --always-recreate-deps --build -d -t 4 +``` + +Once the environment is running, you can trigger the tests with the +`./run-tests.sh` launcher: + +```sh +./run-tests.sh +``` +### Services Setup + +The `./podman-compose.sh up` command initializes the following services: + +- **webserver**: Runs an NGINX container, exposing port **8008** for HTTP + requests. +- **robot**: Runs a Python-based container configured for executing tests + using the Robot Framework. + +### Running Tests + +To execute the tests, use: + +```sh +./run-tests.sh [options] +``` + +Options: + +- `--force-recreate` : Recreate and build containers before running tests. +- `--help` : Display the help message for available options. + +### Stopping the Containers + +To stop the Podman containers, use: + +```sh +./podman-compose.sh down -t 4 +``` + diff --git a/.ci/conf/nginx.conf b/.ci/conf/nginx.conf new file mode 100644 index 0000000..1ffbdc8 --- /dev/null +++ b/.ci/conf/nginx.conf @@ -0,0 +1,35 @@ +server { + listen 8008; + listen [::]:8008; + server_name localhost; + + #access_log /var/log/nginx/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + location /robot_framework/html/ { + root /; + autoindex on; + } + + location /tests_results/ { + root /; + autoindex on; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page + # /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + +} + + diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml new file mode 100644 index 0000000..7bf1186 --- /dev/null +++ b/.ci/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3' +services: + webserver: + image: nginx:latest + network_mode: host + volumes: + - ./conf/nginx.conf:/etc/nginx/conf.d/default.conf + - ./robot_framework/html:/robot_framework/html + - ./tests_results:/tests_results + robot: + build: docker/robot/ + network_mode: host + volumes: + - .:/app diff --git a/.ci/docker/robot/Dockerfile b/.ci/docker/robot/Dockerfile new file mode 100644 index 0000000..c1f9cf4 --- /dev/null +++ b/.ci/docker/robot/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3-slim + +WORKDIR /app + +CMD ["robot_framework/init-robot.sh"] diff --git a/.ci/install-requirements.sh b/.ci/install-requirements.sh new file mode 100755 index 0000000..dc86c3e --- /dev/null +++ b/.ci/install-requirements.sh @@ -0,0 +1,37 @@ +#! /bin/sh -e + +# Identify the Linux distribution +if [ -f /etc/os-release ]; then + . /etc/os-release +else + echo "Distribution identification file /etc/os-release is missing." + exit 1 +fi + +# Function to install podman-compose on Fedora +install_fedora() { + echo "Installing podman-compose on Fedora..." + sudo yum install -y podman-compose pycodestyle python3-pyflakes shellcheck +} + +# Function to install podman-compose on Debian or Ubuntu +install_debian_ubuntu() { + echo "Installing podman-compose on $NAME..." + sudo apt update + sudo apt install -y podman-compose pycodestyle pyflakes3 shellcheck +} + +# Installation process based on the identified distribution +case $ID in + fedora) + install_fedora + ;; + ubuntu | debian) + install_debian_ubuntu + ;; + *) + echo "Your distribution ($ID) is not supported by this script." + exit 2 + ;; +esac + diff --git a/.ci/podman-compose.sh b/.ci/podman-compose.sh new file mode 100755 index 0000000..7e87b75 --- /dev/null +++ b/.ci/podman-compose.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +set -e + +if [ ! -e ./setup-env.sh ] +then + echo "Please, create a ./setup-env.sh to run this command" + exit 1 +fi + +. ./setup-env.sh +exec podman-compose "$@" diff --git a/.ci/prepare-board.sh b/.ci/prepare-board.sh new file mode 100755 index 0000000..b4b7f2f --- /dev/null +++ b/.ci/prepare-board.sh @@ -0,0 +1,30 @@ +#! /bin/bash + +set -e + +BASEPATH="$(dirname "$(readlink -f "$0")")" + +SETUPENV="${BASEPATH}/setup-env.sh" + +if [ ! -e "${SETUPENV}" ] +then + echo "Please, create a ${SETUPENV} to run this command" + exit 1 +fi + +# shellcheck source=./setup-env.sh +. "${SETUPENV}" + +sshi() { + ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "root@${TEST_BOARD_IP}" "$@" +} + +scpi() { + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r "$@" "root@${TEST_BOARD_IP}": +} + +pushd "${BASEPATH}" || exit 1 +scpi scripts +popd || exit 1 + +sshi "/usr/bin/kill-demo || true" diff --git a/.ci/robot_framework/html/bbb_sunflower_1080p_30fps_normal.mp4 b/.ci/robot_framework/html/bbb_sunflower_1080p_30fps_normal.mp4 new file mode 100644 index 0000000..b9acbba --- /dev/null +++ b/.ci/robot_framework/html/bbb_sunflower_1080p_30fps_normal.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c56cd32f709d1da3b93302381736b26ffb0da916b5bde950134b863a93ea5e7 +size 276134947 diff --git a/.ci/robot_framework/html/vertical_scroll.html b/.ci/robot_framework/html/vertical_scroll.html new file mode 100644 index 0000000..792f44e --- /dev/null +++ b/.ci/robot_framework/html/vertical_scroll.html @@ -0,0 +1,101 @@ + + + + + + Alternating Color Blocks + + + + + +
+ +
1
+
2
+
3
+
4
+
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+
50
+
51
+
52
+
53
+
54
+
55
+
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+
64
+
+ + + + diff --git a/.ci/robot_framework/html/video_fps.html b/.ci/robot_framework/html/video_fps.html new file mode 100644 index 0000000..4ccfc55 --- /dev/null +++ b/.ci/robot_framework/html/video_fps.html @@ -0,0 +1,64 @@ + + + + + + FPS Display Video + + + + + + + + +
FPS: 0
+ + + + + diff --git a/.ci/robot_framework/images/pinch-gesture.png b/.ci/robot_framework/images/pinch-gesture.png new file mode 100644 index 0000000..65a819d Binary files /dev/null and b/.ci/robot_framework/images/pinch-gesture.png differ diff --git a/.ci/robot_framework/images/zoom-gesture.png b/.ci/robot_framework/images/zoom-gesture.png new file mode 100644 index 0000000..ddac1b3 Binary files /dev/null and b/.ci/robot_framework/images/zoom-gesture.png differ diff --git a/.ci/robot_framework/init-robot.sh b/.ci/robot_framework/init-robot.sh new file mode 100755 index 0000000..c4e4161 --- /dev/null +++ b/.ci/robot_framework/init-robot.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +set -e + +apt-get update -y +apt-get install -y git ssh \ + build-essential \ + imagemagick tesseract-ocr ghostscript libdmtx0b libzbar0 + +pip install virtualenv + +virtualenv .venv_robot_framework +. ./.venv_robot_framework/bin/activate + +# Install Robot depends +pip install gitpython pygithub \ + robotframework \ + robotframework-doctestlibrary \ + robotframework-retryfailed \ + robotframework-seleniumlibrary \ + robotframework-sshlibrary + +pushd robot_framework/html/ +if [ ! -d "rbyers" ]; then + git clone https://github.com/RByers/rbyers.github.io.git rbyers +fi +popd + +tail -f /dev/null diff --git a/.ci/robot_framework/libs/TestUtils.py b/.ci/robot_framework/libs/TestUtils.py new file mode 100755 index 0000000..849527e --- /dev/null +++ b/.ci/robot_framework/libs/TestUtils.py @@ -0,0 +1,94 @@ +import argparse +import os +import paramiko +import subprocess +import sys +import time +from multiprocessing import Process +from selenium import webdriver + + +class TestUtils: + """Robot Framework library for interacting with remote hosts and running + tests via SSH and WebDriver.""" + + def __init__(self): + self.driver = None + + def print_envvar(starts_with=""): + test_args_env_vars = {key: value for key, value in os.environ.items() if + key.startswith(starts_with)} + + # Sort the dictionary by key + sorted_test_args_env_vars = dict(sorted(test_args_env_vars.items())) + + # Print the variables in a well-tabulated format + for key, value in sorted_test_args_env_vars.items(): + print(f"{key:<30} '{value}'") + + def ssh_command(self, ip, command, quiet=False, debug=False): + """Run SSH command.""" + if debug: + print(f"COMMAND: {ip} {command}", file=sys.stderr) + username = 'root' + password = None + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + client.connect(ip, username=username, password=None) + except paramiko.ssh_exception.SSHException as e: + if not password: + client.get_transport().auth_none(username) + else: + raise e + stdin, stdout, stderr = client.exec_command(command) + + output = stdout.read().decode('utf-8').strip() + error = stderr.read().decode('utf-8').strip() + + # Ensure command completes before closing the session + stdout.channel.recv_exit_status() + + client.close() + + return output, error + + def ssh_command_in_background(self, ip, command): + """Run SSH command in the background without closing the SSH + connection.""" + process = Process(target=self.ssh_command, args=(ip, command)) + process.start() + + def ssh_force_kill(self, ip, text): + """Force kill all related process.""" + print(f"RUN: Killing all '{text}' related processes ...") + command = f"pgrep {text} && pgrep {text} | xargs kill -9 || echo '{text} not running'" + return self.ssh_command(ip, command, quiet=True) + + def ssh_reboot_force_reboot(self, ip): + command("(echo b > /proc/sysrq-trigger) /dev/null &") + self.ssh_command_in_background(ip, command) + + def ssh_reboot_wait_for_reboot(self, ip, timeout=60): + start_time = time.time() + while True: + try: + self.ssh_command(ip, "true") + print("Host is back online.") + break + except Exception: + print("Host is still down...") + if time.time() - start_time > timeout: + print("Timeout reached, stopping wait.") + break + time.sleep(5) + + def ssh_webdriver_remote_start(self, ip, port): + command = ( + f"WPEWebDriver --host={ip} " + f"--port={port} --host-all" + ) + self.ssh_command_in_background(ip, command) + + def ssh_webdriver_remote_stop(self, ip): + self.ssh_force_kill(ip, "WPEWebDriver") diff --git a/.ci/robot_framework/run-robot.sh b/.ci/robot_framework/run-robot.sh new file mode 100755 index 0000000..93e59ef --- /dev/null +++ b/.ci/robot_framework/run-robot.sh @@ -0,0 +1,34 @@ +#! /bin/bash + +set -e + +. ./.venv_robot_framework/bin/activate + +SETUPENV="./setup-env.sh" + +if [ ! -e "${SETUPENV}" ] +then + echo "${SETUPENV} not found in the current path (${PWD})" + exit 1 +fi + +# shellcheck source=./setup-env.sh +. ${SETUPENV} + +DATE=$(date +%Y%m%d_%H%M%S) +mkdir -p "tests_results/${DATE}_robot_${TESTS_RESULTS}/" + +rm -rf tests_results/robot +ln -s "${DATE}_robot_${TESTS_RESULTS}" tests_results/robot +cd "tests_results/${DATE}_robot_${TESTS_RESULTS}/" + +# Copy the setup-env files. +cp ../../setup-env*sh . + +exec robot --name "WPE image tests" \ + --consolewidth 158 \ + --exclude skip \ + --skiponfailure ignoreonfail \ + --listener RetryFailed:2 \ + ../../robot_framework/tests/tests_*.robot + diff --git a/.ci/robot_framework/tests/keywords_common.robot b/.ci/robot_framework/tests/keywords_common.robot new file mode 100644 index 0000000..b1d3ab9 --- /dev/null +++ b/.ci/robot_framework/tests/keywords_common.robot @@ -0,0 +1,56 @@ +*** Settings *** +Library Collections +Library OperatingSystem +Library ../libs/TestUtils.py + +*** Keywords *** +Create WPEWebKitOptions + [Arguments] ${binary_name} ${binary_path} @{other_params} + + ${wpe_options} = Evaluate sys.modules['selenium.webdriver'].WPEWebKitOptions() sys, selenium.webdriver + ${wpe_options.binary_location} Set Variable ${binary_path} + FOR ${param} IN @{other_params} + Call Method ${wpe_options} add_argument ${param} + END + Call Method ${wpe_options} set_capability browserName ${binary_name} + RETURN ${wpe_options} + +Get Remote CPU Load + ${TEST_BOARD_IP} Get Environment Variable TEST_BOARD_IP + ${stdout}= SSH Command ${TEST_BOARD_IP} uptime | awk -F'load average:' '{print $2}' | awk -F',' '{print $1}' + ${value}= Evaluate float(${stdout}[0]) + RETURN ${value} + +Get Remote Memory Used + ${TEST_BOARD_IP} Get Environment Variable TEST_BOARD_IP + ${stdout}= SSH Command ${TEST_BOARD_IP} free -m | grep Mem | awk '{print $3}' + ${value}= Evaluate float(${stdout}[0]) + RETURN ${value} + +Prepare Board + ${rc} ${output}= Run And Return Rc And Output ${PREPARE_BOARD_SCRIPT} + Should Be Equal As Integers ${rc} 0 msg=Prepare Board command failed with non-zero exit status + Log output: ${output} + +Webdriver Remote Start + ${TEST_BOARD_IP} Get Environment Variable TEST_BOARD_IP + ${TEST_BOARD_WEBDRIVER_PORT} Get Environment Variable TEST_BOARD_WEBDRIVER_PORT + + # Force kill previous launchers + SSH Webdriver Remote Stop ${TEST_BOARD_IP} + SSH Force Kill ${TEST_BOARD_IP} cog + + SSH Webdriver Remote Start ${TEST_BOARD_IP} ${TEST_BOARD_WEBDRIVER_PORT} + Sleep 5 + + ${wpe_options} = Create WPEWebKitOptions cog /usr/bin/cog-fdo-exported-wayland --maximized --automation + Create Webdriver Remote command_executor=${TEST_BOARD_IP}:${TEST_BOARD_WEBDRIVER_PORT} options=${wpe_options} + +Webdriver Remote Stop + ${TEST_BOARD_IP} Get Environment Variable TEST_BOARD_IP + ${TEST_BOARD_WEBDRIVER_PORT} Get Environment Variable TEST_BOARD_WEBDRIVER_PORT + + Close All Browsers + SSH Webdriver Remote Stop ${TEST_BOARD_IP} + SSH Force Kill ${TEST_BOARD_IP} cog + diff --git a/.ci/robot_framework/tests/keywords_touch_events.robot b/.ci/robot_framework/tests/keywords_touch_events.robot new file mode 100644 index 0000000..1c8e01a --- /dev/null +++ b/.ci/robot_framework/tests/keywords_touch_events.robot @@ -0,0 +1,66 @@ +*** Variables *** +${SCROLL_POSITION} 300 +${SWIPE_POSITION} 1000 +${SWIPE_THRESHOLD} 150 +${SWIPE_WAIT} 5 + +${BASELINE_IMAGES_PATH} /app/robot_framework/images/ +${PINCH_GESTURE_IMAGE} pinch-gesture.png +${ZOOM_GESTURE_IMAGE} zoom-gesture.png + +*** Settings *** +Library DocTest.VisualTest +Library OperatingSystem +Library ../libs/TestUtils.py + +Resource variables.robot +Resource keywords_common.robot + +*** Keywords *** +Check Browser Touch Events Using Uinput + ${TEST_BOARD_IP} Get Environment Variable TEST_BOARD_IP + ${TEST_WEBSERVER_IP} Get Environment Variable TEST_WEBSERVER_IP + ${TEST_WEBSERVER_PORT} Get Environment Variable TEST_WEBSERVER_PORT + ${PAGE} Set Variable http://${TEST_WEBSERVER_IP}:${TEST_WEBSERVER_PORT}/robot_framework/html/vertical_scroll.html + ${PAGE2} Set Variable http://${TEST_WEBSERVER_IP}:${TEST_WEBSERVER_PORT}/robot_framework/html/rbyers/paint.html + + Go to ${PAGE} + Capture Page Screenshot + + # Scroll + SSH Command ${TEST_BOARD_IP} /root/scripts/touch-one-finger-gesture.py --duration 5 --steps 40 --delay-on-touch-up 0 100 500 100 200 + Capture Page Screenshot + ${scroll_position}= Execute JavaScript return window.pageYOffset; + Should Be Equal As Numbers ${scroll_position} ${SCROLL_POSITION} + + SSH Command ${TEST_BOARD_IP} /root/scripts/touch-one-finger-gesture.py --duration 5 --steps 40 --delay-on-touch-up 0 100 200 100 500 + Capture Page Screenshot + ${scroll_position}= Execute JavaScript return window.pageYOffset; + Should Be Equal As Numbers ${scroll_position} 0 + + # Swipe + SSH Command ${TEST_BOARD_IP} /root/scripts/touch-one-finger-gesture.py --duration 0.1 --steps 40 --delay-on-touch-up 0 100 500 100 200 + Capture Page Screenshot + Sleep ${SWIPE_WAIT} + ${scroll_position}= Execute JavaScript return window.pageYOffset; + ${swipe_upper_position}= Set Variable ${SWIPE_POSITION} + ${SWIPE_THRESHOLD} + Should Be True ${scroll_position} > ${SWIPE_POSITION} and ${scroll_position} < ${swipe_upper_position} + + SSH Command ${TEST_BOARD_IP} /root/scripts/touch-one-finger-gesture.py --duration 0.1 --steps 40 --delay-on-touch-up 0 100 200 100 500 + Capture Page Screenshot + Sleep ${SWIPE_WAIT} + ${scroll_position}= Execute JavaScript return window.pageYOffset; + Should Be True ${scroll_position} >= 0 and ${scroll_position} < ${SWIPE_THRESHOLD} + + # Multitouch: Pinch + Go to ${PAGE2} + SSH Command ${TEST_BOARD_IP} /root/scripts/touch-two-fingers-gesture.py --duration 2 --steps 40 900 200 900 500 900 800 900 500 + Capture Page Screenshot ${PINCH_GESTURE_IMAGE} + Compare Images ${BASELINE_IMAGES_PATH}/${PINCH_GESTURE_IMAGE} ${PINCH_GESTURE_IMAGE} + + # Multitouch: Zoom + Go to ${PAGE2} + SSH Command ${TEST_BOARD_IP} /root/scripts/touch-two-fingers-gesture.py --duration 2 --steps 40 900 500 900 200 900 500 900 800 + Capture Page Screenshot ${ZOOM_GESTURE_IMAGE} + Compare Images ${BASELINE_IMAGES_PATH}/${ZOOM_GESTURE_IMAGE} ${ZOOM_GESTURE_IMAGE} + diff --git a/.ci/robot_framework/tests/tests_000_common.robot b/.ci/robot_framework/tests/tests_000_common.robot new file mode 100644 index 0000000..5da0826 --- /dev/null +++ b/.ci/robot_framework/tests/tests_000_common.robot @@ -0,0 +1,12 @@ +*** Settings *** +Test Timeout 600 seconds + +Resource variables.robot +Resource keywords_common.robot + +*** Test Cases *** +Setup + ${TEST_BOARD_SETUP_SKIP} Get Environment Variable TEST_BOARD_SETUP_SKIP default=no + Run Keyword If '${TEST_BOARD_SETUP_SKIP}' != 'yes' + ... Prepare Board + diff --git a/.ci/robot_framework/tests/tests_010_input_events.robot b/.ci/robot_framework/tests/tests_010_input_events.robot new file mode 100644 index 0000000..5bb70bc --- /dev/null +++ b/.ci/robot_framework/tests/tests_010_input_events.robot @@ -0,0 +1,20 @@ +*** Settings *** +Test Timeout 60 seconds + +Suite Setup Webdriver Remote Start +Suite Teardown Webdriver Remote Stop + +Library Collections +Library DocTest.VisualTest +Library OperatingSystem +Library SeleniumLibrary +Library ../libs/TestUtils.py + +Resource variables.robot +Resource keywords_touch_events.robot + +*** Test Cases *** +Test Check Browser Touch Events Using Uinput + [Tags] ignoreonfail + Check Browser Touch Events Using Uinput + diff --git a/.ci/robot_framework/tests/tests_015_video.robot b/.ci/robot_framework/tests/tests_015_video.robot new file mode 100644 index 0000000..fb910d9 --- /dev/null +++ b/.ci/robot_framework/tests/tests_015_video.robot @@ -0,0 +1,42 @@ +*** Settings *** + +Test Timeout 60 seconds + +Suite Setup Webdriver Remote Start +Suite Teardown Webdriver Remote Stop + +Library SeleniumLibrary + +Resource variables.robot +Resource keywords_common.robot + +*** Keywords *** +Get FPS Value + ${fps_text}= Get Text id=fps + ${fps}= Convert To Number ${fps_text.split(":")[1].strip()} + RETURN ${fps} + +*** Test Cases *** +Verify Full HD 30 FPS + ${TEST_WEBSERVER_IP} Get Environment Variable TEST_WEBSERVER_IP + ${TEST_WEBSERVER_PORT} Get Environment Variable TEST_WEBSERVER_PORT + ${PAGE} Set Variable http://${TEST_WEBSERVER_IP}:${TEST_WEBSERVER_PORT}/robot_framework/html/video_fps.html + + Go to ${PAGE} + Sleep 20 seconds + + ${memory_used}= Get Remote Memory Used + Log Memory used: ${memory_used} + + ${cpu_load}= Get Remote CPU Load + Log CPU load: ${cpu_load} + + ${fps} Get FPS Value + Log FPS value: ${fps} + + Capture Page Screenshot + + Should Be True ${fps} > ${VIDEO_30_FPS_THRESHOLD_FPS} + Should Be True ${cpu_load} < ${VIDEO_30_FPS_THRESHOLD_CPU_LOAD} + Should Be True ${memory_used} < ${VIDEO_30_FPS_THRESHOLD_MEMORY_USED} + diff --git a/.ci/robot_framework/tests/tests_020_motionmark.robot b/.ci/robot_framework/tests/tests_020_motionmark.robot new file mode 100644 index 0000000..f6fe9de --- /dev/null +++ b/.ci/robot_framework/tests/tests_020_motionmark.robot @@ -0,0 +1,49 @@ +*** Variables *** +${URL} https://browserbench.org/MotionMark1.2/ +${RUN_BENCHMARK_BUTTON} xpath=//*[@id="intro"]/div[2]/button +${SCORE_SELECTOR} xpath=//*[@id="results"]/div[2]/div[1]/div[1] +${TEST_AGAIN_BUTTON} xpath=//button[contains(@onclick, 'benchmarkController.startBenchmark()') and contains(text(), 'Test Again')] + +*** Settings *** + +Test Timeout 900 seconds + +Suite Setup Webdriver Remote Start +Suite Teardown Webdriver Remote Stop + +Library SeleniumLibrary + +Resource variables.robot +Resource keywords_common.robot + +*** Keywords *** +Capture Images Until Test Completion + [Documentation] Captures a screenshot each time a new test section loads until the "Test Again" button appears. + + ${index}= Set Variable 1 + WHILE "True" + Sleep 20s + Capture Page Screenshot motionmark_test_${index}.png + ${index}= Evaluate ${index} + 1 + + Run Keyword And Ignore Error Element Should Be Visible ${TEST_AGAIN_BUTTON} + ${is_test_again_visible}= Run Keyword And Return Status Element Should Be Visible ${TEST_AGAIN_BUTTON} + Run Keyword If ${is_test_again_visible} Exit For Loop + END + +*** Test Cases *** +Run MotionMark Benchmark And Validate Score + [Documentation] Loads MotionMark benchmark, runs it, waits for the score, and validates. + + Go to ${URL} + Wait Until Page Contains Element ${RUN_BENCHMARK_BUTTON} + Click Element ${RUN_BENCHMARK_BUTTON} + + Capture Images Until Test Completion + + Wait Until Page Contains Element ${TEST_AGAIN_BUTTON} timeout=600s + Capture Page Screenshot + ${score}= Get Text ${SCORE_SELECTOR} + Log MotionMark Score : ${score} + Should Be True ${score} > ${MOTIONMARK_MIN_SCORE} + diff --git a/.ci/robot_framework/tests/variables.robot b/.ci/robot_framework/tests/variables.robot new file mode 100644 index 0000000..7677d47 --- /dev/null +++ b/.ci/robot_framework/tests/variables.robot @@ -0,0 +1,9 @@ +*** Variables *** +${PREPARE_BOARD_SCRIPT} ../../prepare-board.sh + +${MOTIONMARK_MIN_SCORE} 90 + +${VIDEO_30_FPS_THRESHOLD_CPU_LOAD} 3 +${VIDEO_30_FPS_THRESHOLD_MEMORY_USED} 850 +${VIDEO_30_FPS_THRESHOLD_FPS} 28 + diff --git a/.ci/run-tests.sh b/.ci/run-tests.sh new file mode 100755 index 0000000..35a577c --- /dev/null +++ b/.ci/run-tests.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +# Function to display help +show_help() { + echo "Usage: $0 [options]" + echo "Options:" + echo " --force-recreate Recreate and build containers before running tests" + echo " --help Show this help message" +} + +# Check arguments +force_recreate=false + +for arg in "$@"; do + case $arg in + --force-recreate) + force_recreate=true + ;; + --help) + show_help + exit 0 + ;; + *) + echo "Unknown option: $arg" + show_help + exit 1 + ;; + esac +done + +# Run podman-compose only if --force-recreate is specified +if [ "$force_recreate" = true ]; then + ./podman-compose.sh up --force-recreate --always-recreate-deps --build -d -t 4 > /dev/null 2>&1 +fi + +# Run the test script +podman exec -ti meta-wpe-image-tests_robot_1 ./robot_framework/run-robot.sh + + diff --git a/.ci/scripts/touch-one-finger-gesture.py b/.ci/scripts/touch-one-finger-gesture.py new file mode 100755 index 0000000..8a818e9 --- /dev/null +++ b/.ci/scripts/touch-one-finger-gesture.py @@ -0,0 +1,76 @@ +#! /usr/bin/python3 + +import argparse +import uinput +import time + +DEVICE_READY_TIMEOUT = 3 +DISPLAY_WIDTH = 1920 +DISPLAY_HEIGHT = 1080 + + +def simulate_scroll(device, start_x, start_y, end_x, end_y, + duration=0.5, steps=10, delay_on_touch_up=0): + """Simulate a swipe gesture from (start_x, start_y) to (end_x, end_y)""" + + # Calculate step increments for smooth motion + if steps > 0: + step_x = (end_x - start_x) // steps + step_y = (end_y - start_y) // steps + delay = 1.0 * duration / steps # Time delay between each step + + # Start touch + device.emit(uinput.ABS_X, start_x, syn=False) + device.emit(uinput.ABS_Y, start_y, syn=False) + device.emit(uinput.BTN_TOUCH, 1) # Press touch + device.syn() + + # Move in steps to simulate dragging + for i in range(steps): + device.emit(uinput.ABS_X, start_x + step_x * i, syn=False) + device.emit(uinput.ABS_Y, start_y + step_y * i, syn=False) + device.syn() + time.sleep(delay) + + # End touch + time.sleep(delay_on_touch_up) + device.emit(uinput.ABS_X, end_x, syn=False) + device.emit(uinput.ABS_Y, end_y, syn=False) + device.emit(uinput.BTN_TOUCH, 0) # Release touch + device.syn() + + +def main(): + parser = argparse.ArgumentParser(description="Simulate a swipe gesture.") + parser.add_argument("start_x", type=int, + help="Starting X position of the swipe.") + parser.add_argument("start_y", type=int, + help="Starting Y position of the swipe.") + parser.add_argument("end_x", type=int, + help="Ending X position of the swipe.") + parser.add_argument("end_y", type=int, + help="Ending Y position of the swipe.") + parser.add_argument("--duration", type=float, default=0.5, + help="Duration of the swipe in seconds.") + parser.add_argument("--steps", type=int, default=10, + help="Number of steps for the swipe.") + parser.add_argument("--delay-on-touch-up", type=float, default=0, + help="Delay on touch up.") + + args = parser.parse_args() + + # Create a device that can emit touch events + device = uinput.Device([ + uinput.ABS_X + (0, DISPLAY_WIDTH, 0, 0), + uinput.ABS_Y + (0, DISPLAY_HEIGHT, 0, 0), + uinput.BTN_TOUCH, + ]) + + time.sleep(DEVICE_READY_TIMEOUT) + + simulate_scroll(device, args.start_x, args.start_y, args.end_x, args.end_y, + args.duration, args.steps, args.delay_on_touch_up) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/touch-two-fingers-gesture.py b/.ci/scripts/touch-two-fingers-gesture.py new file mode 100755 index 0000000..2f07720 --- /dev/null +++ b/.ci/scripts/touch-two-fingers-gesture.py @@ -0,0 +1,137 @@ +#!/usr/bin/python3 + +import argparse +import uinput +import time + +DEVICE_READY_TIMEOUT = 3 +DISPLAY_WIDTH = 1920 +DISPLAY_HEIGHT = 1080 + + +def simulate_two_finger_gesture(device, + start_x1, start_y1, end_x1, end_y1, + start_x2, start_y2, end_x2, end_y2, + duration=0.5, steps=10): + """ Simulate a two-finger scroll from (start_x1, start_y1) and + (start_x2, start_y2) to (end_x1, end_y1) and (end_x2, end_y2) + """ + + tracking_id1 = 1 + tracking_id2 = 2 + + # Calculate step increments for smooth motion + step_x1 = (end_x1 - start_x1) // steps + step_y1 = (end_y1 - start_y1) // steps + step_x2 = (end_x2 - start_x2) // steps + step_y2 = (end_y2 - start_y2) // steps + delay = duration / steps # Time delay between each step + + # Start touch for both fingers + device.emit(uinput.ABS_MT_TRACKING_ID, tracking_id1, syn=False) + device.emit(uinput.ABS_MT_POSITION_X, start_x1, syn=False) + device.emit(uinput.ABS_MT_POSITION_Y, start_y1, syn=False) + device.emit(uinput.ABS_MT_SLOT, 1, syn=False) + + device.emit(uinput.ABS_MT_TRACKING_ID, tracking_id2, syn=False) + device.emit(uinput.ABS_MT_POSITION_X, start_x2, syn=False) + device.emit(uinput.ABS_MT_POSITION_Y, start_y2, syn=False) + device.emit(uinput.ABS_MT_SLOT, 0, syn=False) + + device.emit(uinput.BTN_TOUCH, 1) # Press touch for first finger + device.emit(uinput.ABS_X, start_x1, syn=False) + device.emit(uinput.ABS_Y, start_y1, syn=False) + device.syn() + + # Move both fingers in steps to simulate dragging + for i in range(steps): + # Update first finger position + device.emit(uinput.ABS_MT_SLOT, 1, syn=False) + device.emit(uinput.ABS_MT_POSITION_X, start_x1 + step_x1 * i, + syn=False) + device.emit(uinput.ABS_MT_POSITION_Y, start_y1 + step_y1 * i, + syn=False) + device.emit(uinput.ABS_X, start_x1 + step_x1 * i, syn=False) + device.emit(uinput.ABS_Y, start_y1 + step_y1 * i, syn=False) + + # Update second finger position + device.emit(uinput.ABS_MT_SLOT, 0, syn=False) + device.emit(uinput.ABS_MT_POSITION_X, start_x2 + step_x2 * i, + syn=False) + device.emit(uinput.ABS_MT_POSITION_Y, start_y2 + step_y2 * i, + syn=False) + + device.syn() + + # Wait between steps to simulate smooth scroll + time.sleep(delay) + + # End touch for both fingers + device.emit(uinput.ABS_MT_SLOT, 1, syn=False) + device.emit(uinput.ABS_MT_POSITION_X, end_x1, syn=False) + device.emit(uinput.ABS_MT_POSITION_Y, end_y1, syn=False) + device.emit(uinput.ABS_X, end_x1, syn=False) + device.emit(uinput.ABS_Y, end_y1, syn=False) + device.emit(uinput.ABS_MT_TRACKING_ID, -1, syn=False) + device.emit(uinput.BTN_TOUCH, 0) # Release touch for first finger + device.syn() + + device.emit(uinput.ABS_MT_SLOT, 0, syn=False) + device.emit(uinput.ABS_MT_POSITION_X, end_x2, syn=False) + device.emit(uinput.ABS_MT_POSITION_Y, end_y2, syn=False) + device.emit(uinput.ABS_X, end_x2, syn=False) + device.emit(uinput.ABS_Y, end_y2, syn=False) + device.emit(uinput.ABS_MT_TRACKING_ID, -1, syn=False) + device.emit(uinput.BTN_TOUCH, 0) # Release touch for second finger + device.syn() + + +def main(): + parser = argparse.ArgumentParser(description="Simulate a swipe gesture.") + parser.add_argument("start_x1", type=int, + help="Starting X1 position of the swipe.") + parser.add_argument("start_y1", type=int, + help="Starting Y1 position of the swipe.") + parser.add_argument("end_x1", type=int, + help="Ending X1 position of the swipe.") + parser.add_argument("end_y1", type=int, + help="Ending Y1 position of the swipe.") + parser.add_argument("start_x2", type=int, + help="Starting X2 position of the swipe.") + parser.add_argument("start_y2", type=int, + help="Starting Y2 position of the swipe.") + parser.add_argument("end_x2", type=int, + help="Ending X2 position of the swipe.") + parser.add_argument("end_y2", type=int, + help="Ending Y2 position of the swipe.") + parser.add_argument("--duration", type=float, default=0.5, + help="Duration of the swipe in seconds.") + parser.add_argument("--steps", type=int, default=10, + help="Number of steps for the swipe.") + + args = parser.parse_args() + + # Create a device that can emit touch events + device = uinput.Device([ + uinput.ABS_MT_TRACKING_ID + (0, 65535, 0, 0), + uinput.ABS_MT_POSITION_X + (0, DISPLAY_WIDTH, 0, 0), + uinput.ABS_MT_POSITION_Y + (0, DISPLAY_HEIGHT, 0, 0), + uinput.ABS_X + (0, DISPLAY_WIDTH, 0, 0), + uinput.ABS_Y + (0, DISPLAY_HEIGHT, 0, 0), + uinput.ABS_MT_SLOT + (0, 10, 0, 0), + uinput.BTN_TOUCH, + ]) + + time.sleep(DEVICE_READY_TIMEOUT) + + # Simulate a two-finger scroll gesture + simulate_two_finger_gesture(device, + args.start_x1, args.start_y1, + args.end_x1, args.end_y1, + args.start_x2, args.start_y2, + args.end_x2, args.end_y2, + args.duration, args.steps) + + +if __name__ == "__main__": + main() diff --git a/.ci/setup-env-local.sh.sample b/.ci/setup-env-local.sh.sample new file mode 100644 index 0000000..e45a33f --- /dev/null +++ b/.ci/setup-env-local.sh.sample @@ -0,0 +1,10 @@ +#!/bin/sh + +# export TEST_BOARD_SETUP_SKIP="yes" + +# export TEST_BOARD_IP="192.168.1.105" +# export TEST_BOARD_NAME="rpi5" + +# export TEST_WEBSERVER_IP="192.168.1.92" +# export TEST_WEBSERVER_PORT="8008" + diff --git a/.ci/setup-env.sh b/.ci/setup-env.sh new file mode 100755 index 0000000..0d020fa --- /dev/null +++ b/.ci/setup-env.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +################################################################################ +# Environment variables definitions +################################################################################# + +# export TEST_BOARD_SETUP_SKIP="yes" + +export TEST_BOARD_WEBDRIVER_PORT="8888" + +export TEST_BOARD_IP="192.168.1.105" +export TEST_BOARD_NAME="rpi5" + +export TEST_WEBSERVER_IP="192.168.1.92" +export TEST_WEBSERVER_PORT="8008" + +################################################################################ +# Load local setup +################################################################################# + +# XXX: Get the basepath from the environment +SETUPENVLOCAL="setup-env-local.sh" +APPBASEPATH="/app" +APPSETUPENVLOCAL="${APPBASEPATH}/${SETUPENVLOCAL}" + +if [ -f "${APPSETUPENVLOCAL}" ]; then + # shellcheck source=./setup-env.sh + . "${APPSETUPENVLOCAL}" +elif [ -f "${SETUPENVLOCAL}" ]; then + # shellcheck source=./setup-env.sh + . "./${SETUPENVLOCAL}" +else + echo "WARNING: Not ${APPSETUPENVLOCAL} nor ${SETUPENVLOCAL} found" +fi + diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b90b5b9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.ci/robot_framework/html/bbb_sunflower_1080p_30fps_normal.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/.github/scripts/run-all-sanatizers b/.github/scripts/run-all-sanatizers new file mode 100755 index 0000000..2bb1897 --- /dev/null +++ b/.github/scripts/run-all-sanatizers @@ -0,0 +1,19 @@ +#!/bin/bash + +# Navigate to the directory containing the sanitizer scripts +SANATIZERS_DIR="$(dirname $(realpath $0))" + +# Initialize an exit status variable +exit_status=0 + +# Loop through each sanitizer script and execute it +for script in $SANATIZERS_DIR/sanatizer-*; do + "$script" + # Check the exit status of the script + if [ $? -ne 0 ]; then + echo "Error: $script failed" + exit_status=1 + fi +done + +exit $exit_status diff --git a/.github/scripts/sanatizer-pycodestyle b/.github/scripts/sanatizer-pycodestyle new file mode 100755 index 0000000..76402e2 --- /dev/null +++ b/.github/scripts/sanatizer-pycodestyle @@ -0,0 +1,18 @@ +#! /bin/bash + +set -e + +scripts=$(find . -executable -type f ! -path '*/\.*' -exec grep -lE '^#! *(|/usr/bin/env +|/bin/|/usr/bin/)(python|python3)' {} +) +echo "Running pycodestyle on the following scripts:" +echo "$scripts" +errors=0 +for script in $scripts; do + if ! pycodestyle "$script"; then + errors=$((errors + 1)) + fi +done +if [ "$errors" -ne 0 ]; then + echo "pycodestyle found issues." + exit 1 +fi +echo "pycodestyle passed successfully." diff --git a/.github/scripts/sanatizer-pyflake8 b/.github/scripts/sanatizer-pyflake8 new file mode 100755 index 0000000..de8f859 --- /dev/null +++ b/.github/scripts/sanatizer-pyflake8 @@ -0,0 +1,18 @@ +#! /bin/bash + +set -e + +scripts=$(find . -executable -type f ! -path '*/\.*' -exec grep -lE '^#! *(|/usr/bin/env +|/bin/|/usr/bin/)(python|python3)' {} +) +echo "Running pyflakes3 on the following scripts:" +echo "$scripts" +errors=0 +for script in $scripts; do + if ! pyflakes3 "$script"; then + errors=$((errors + 1)) + fi +done +if [ "$errors" -ne 0 ]; then + echo "pyflakes3 found issues." + exit 1 +fi +echo "pyflakes3 passed successfully." diff --git a/.github/scripts/sanatizer-shellcheck b/.github/scripts/sanatizer-shellcheck new file mode 100755 index 0000000..65b11b9 --- /dev/null +++ b/.github/scripts/sanatizer-shellcheck @@ -0,0 +1,24 @@ +#! /bin/bash + +set -e + +scripts=$(find . -executable -type f ! -path '*/\.*' -exec grep -lE '^#! *(|/usr/bin/env +|/bin/|/usr/bin/)(sh|bash|dash)' {} +) +echo "Running ShellCheck on the following scripts:" +echo "$scripts" +errors=0 + +# Force the creation of the setup-env.sh and so +touch ./setup-env.sh +mkdir -p ./.venv_robot_framework/bin +touch ./.venv_robot_framework/bin/activate + +for script in $scripts; do + if ! shellcheck -x "$script"; then + errors=$((errors + 1)) + fi +done +if [ "$errors" -ne 0 ]; then + echo "ShellCheck found issues." + exit 1 +fi +echo "ShellCheck passed successfully." diff --git a/.github/workflows/sanatizers.yml b/.github/workflows/sanatizers.yml new file mode 100644 index 0000000..e0936ee --- /dev/null +++ b/.github/workflows/sanatizers.yml @@ -0,0 +1,19 @@ +name: Sanatizers + +on: + pull_request: + types: [synchronize, opened, reopened] + +jobs: + sanatizers: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install requirements + run: sudo apt-get install python3-flake8 python3-pycodestyle shellcheck + + - name: Run sanatizers + run: ./.github/scripts/run-all-sanatizers + diff --git a/.gitignore b/.gitignore index cb055f9..4f804ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,19 @@ __pycache__ +*.bak *.pyc *.pyo *.swp +*.tmp *.orig *.rej *~ + +# Python venv +venv* +.env +.virtualenv +.venv_robot_framework + +.ci/robot_framework/html/rbyers +.ci/setup-env-local.sh +.ci/tests_results/