From 9ebdd72b46e9a41d914953622ce6f70175e2e2b8 Mon Sep 17 00:00:00 2001 From: Mariia_Azbeleva Date: Mon, 17 Jul 2023 17:07:45 +0300 Subject: [PATCH] Add logging and additional checks 1. Add more log messages to console 2. Add saving and printing of versions ang journalctl 3. Move version tests to another suite 4. Add checking of port 22 of netvm Signed-off-by: Mariia Azbeleva --- .../resources/serial_keywords.resource | 20 +++++- .../resources/ssh_keywords.resource | 64 ++++++++++++++++--- .../test-suites/bat-tests/__init__.robot | 40 ++++++++++++ .../test-suites/{ => bat-tests}/apps.robot | 5 +- .../test-suites/{ => bat-tests}/netvm.robot | 22 +++++-- .../test-suites/bat-tests/others.robot | 26 ++++++++ Robot-Framework/test-suites/boot_test.robot | 27 +++----- 7 files changed, 168 insertions(+), 36 deletions(-) create mode 100644 Robot-Framework/test-suites/bat-tests/__init__.robot rename Robot-Framework/test-suites/{ => bat-tests}/apps.robot (79%) rename Robot-Framework/test-suites/{ => bat-tests}/netvm.robot (91%) create mode 100644 Robot-Framework/test-suites/bat-tests/others.robot diff --git a/Robot-Framework/resources/serial_keywords.resource b/Robot-Framework/resources/serial_keywords.resource index 82a42a7..f7c3bd7 100644 --- a/Robot-Framework/resources/serial_keywords.resource +++ b/Robot-Framework/resources/serial_keywords.resource @@ -6,15 +6,18 @@ Library SerialLibrary encoding=ascii Resource ../config/variables.robot Library BuiltIn Library String +Library AdvancedLogging Library ../lib/output_parser.py *** Keywords *** Open Serial Port + [Arguments] ${timeout}=1.0 Add Port ${SERIAL_PORT} ... baudrate=115200 ... bytesize=8 ... parity=N ... stopbits=1 + ... timeout=${timeout} Check Serial Connection [Documentation] Check if device is available by serial @@ -55,8 +58,7 @@ Log In To Ghaf OS Verify Systemctl status via serial [Arguments] ${range}=60 [Documentation] Check is systemctl running with given loop ${range} - Open Serial Port - Log In To Ghaf OS + Connect via serial ${start_time}= Get Time epoch FOR ${i} IN RANGE ${range} Write Data systemctl status${\n} @@ -71,7 +73,21 @@ Verify Systemctl status via serial Log To Console Systemctl status is ${status} RETURN END + Sleep 1 END ${diff}= Evaluate int(time.time()) - int(${start_time}) FAIL Systemctl is not running after ${diff} sec! Status is ${status} [Teardown] Delete All Ports + +Connect via serial + [Arguments] ${timeout}=1.0 + Open Serial Port timeout=${timeout} + Log In To Ghaf OS + +Save log + Connect via serial timeout=60.0 + Write Data sh -c 'journalctl | tee jrnl.txt | cat'${\n} + ${output} = SerialLibrary.Read Until ghaf@ghaf-host: + Write Advanced Testlog journalctl.log ${output} + Log ${output} + [Teardown] Delete All Ports diff --git a/Robot-Framework/resources/ssh_keywords.resource b/Robot-Framework/resources/ssh_keywords.resource index c90a84d..927aeb7 100644 --- a/Robot-Framework/resources/ssh_keywords.resource +++ b/Robot-Framework/resources/ssh_keywords.resource @@ -21,15 +21,22 @@ Ping Host Check Network Availability [Arguments] ${host} ${expected_result}=True ${range}=5 + Log To Console Checking network ${host} availability, expected: ${expected_result} Set Global Variable ${is_available} False FOR ${i} IN RANGE ${range} Write ping ${host} -c 1 TRY Read Until 1 received Set Global Variable ${is_available} True + Log To Console ${host} is available BREAK EXCEPT - CONTINUE + IF ${expected_result} == False + Log To Console ${host} is unavailable + BREAK + ELSE + CONTINUE + END END END IF ${is_available} != ${expected_result} @@ -67,20 +74,28 @@ Verify Systemctl status [Teardown] Close Connection Start Chromium + Log To Console ${\n}Starting Chromium ${command}= Set Variable chromium-browser --enable-features=UseOzonePlatform --ozone-platform=wayland ${command_output}= Execute Command nohup ${command} > output.log 2>&1 & Is process started [Arguments] ${pid} - ${rc}= Execute Command ps -p ${pid} return_rc=True - ${status}= Run Keyword And Return Status Should Be Equal As Integers ${rc} 0 + ${rc}= Execute Command ps -p ${pid} return_rc=True + ${status}= Run Keyword And Return Status Should Be Equal As Integers ${rc[1]} 0 + IF ${status} + Log To Console Process ${pid} exists + ELSE + Log To Console Process ${pid} doesn't exist + END [Return] ${status} Find pid by name - [Arguments] ${proc_name} - ${output}= Execute Command ps aux + [Arguments] ${proc_name} + Log To Console Looking for pids of the proccess ${proc_name} + ${output}= Execute Command sh -c 'ps aux | grep "${proc_name}" | grep -v grep' + Log ${output} @{pids}= Find Pid ${output} ${proc_name} - Log To Console Process: ${proc_name}\n PIDs: ${pids} + Log To Console Found PIDs: ${pids} [Return] @{pids} Kill process @@ -92,6 +107,8 @@ Kill process ${ps_exists}= Is Process Started ${pid} IF ${ps_exists} Sleep 0.5 + ELSE + BREAK END END IF ${ps_exists} FAIL Process ${pid} haven't stopped @@ -102,10 +119,11 @@ Verify service status [Arguments] ${range}=1 ${service}=${EMPTY} ${expected_status}=active ${expected_state}=running FOR ${i} IN RANGE ${range} ${output}= Execute Command systemctl status ${service} + Log ${output} ${result}= Get Service Status ${output} ${status} ${state} Get Service Status ${output} END - Log To Console Service status: ${result} + Log To Console ${\n}systemctl status ${service} ${result} Should Be True '${status}' == '${expected_status}' Expected ${expected_status} but in fact ${status} Should Be True '${state}' == '${expected_state}' Expected ${expected_state} but in fact ${state} [Return] ${status} ${state} @@ -136,9 +154,17 @@ Wait until NetVM service started Create file [Documentation] Create file with given path & name, requires existing ssh connection [Arguments] ${file_name} + Log To Console Creating file ${file_name} Execute Command touch ${file_name} sudo=True sudo_password=${password} SSHLibrary.File Should Exist ${file_name} +Check file doesn't exist + [Documentation] Check file doesn't exist + [Arguments] ${file_name} + Log To Console Check if file ${file_name} exists + SSHLibrary.File Should Not Exist ${file_name} + Log To Console File ${file_name} doesn't exist + Verify Ghaf Version Format [Documentation] Check that ghaf-version contains version number in the format:"dd.dd", ... date of commit in format yyyymmdd and 7 symbols of hash commit @@ -170,5 +196,27 @@ Get Version [Arguments] ${whose} ${output} ${rc} Execute Command ${whose}-version return_rc=True Should Be Equal As Integers ${rc} 0 Couldn't get ghaf version, command return code + Log To Console ${whose}-version: ${output} ${major} ${minor} ${date} ${commit} ${name} Parse Version ${output} - [Return] ${major} ${minor} ${date} ${commit} ${name} \ No newline at end of file + [Return] ${major} ${minor} ${date} ${commit} ${name} + +Save log + ${output} Execute Command journalctl > jrnl.txt${\n} + ${output} Execute Command cat jrnl.txt${\n} + Log ${output} + +Check if ssh is ready on netvm + [Arguments] ${timeout}=15 + ${is_ready} = Set Variable False + FOR ${i} IN RANGE ${timeout} + Write nc -zvw3 192.168.101.1 22 + ${output} Read Until ghaf-host + ${status} Run Keyword And Return Status + ... Should Contain ${output} Connection to ${netvm_ip} 22 port [tcp/ssh] succeeded! + IF ${status} + ${is_ready} = Set Variable True + BREAK + END + Sleep 1 + END + IF ${status} == False FAIL Port 22 of NetVM is not ready after ${timeout} diff --git a/Robot-Framework/test-suites/bat-tests/__init__.robot b/Robot-Framework/test-suites/bat-tests/__init__.robot new file mode 100644 index 0000000..6a9c93f --- /dev/null +++ b/Robot-Framework/test-suites/bat-tests/__init__.robot @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2022-2023 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +#Documentation +#Force Tags +Resource ../../resources/ssh_keywords.resource +Resource ../../config/variables.robot +Suite Setup Common Setup +Suite Teardown Common Teardown + + +*** Keywords *** + +Common Setup + Set Variables ${DEVICE} + Connect + Log versions + Run journalctl recording + +Common Teardown + Connect + Log journctl + Close All Connections + +Run journalctl recording + ${output} Execute Command journalctl > jrnl.txt + ${output} Execute Command nohup journalctl -f >> jrnl.txt 2>&1 & + +Log journctl + ${output} Execute Command cat jrnl.txt + Log ${output} + @{pid} Find pid by name journalctl + Kill process @{pid} + +Log versions + ${ghaf_version} Execute Command ghaf-version + Log Ghaf version: ${ghaf_version} + ${nixos_version} Execute Command nixos-version + Log Nixos version: ${nixos_version} diff --git a/Robot-Framework/test-suites/apps.robot b/Robot-Framework/test-suites/bat-tests/apps.robot similarity index 79% rename from Robot-Framework/test-suites/apps.robot rename to Robot-Framework/test-suites/bat-tests/apps.robot index 3278be9..e94a0db 100644 --- a/Robot-Framework/test-suites/apps.robot +++ b/Robot-Framework/test-suites/bat-tests/apps.robot @@ -4,9 +4,8 @@ *** Settings *** Documentation Testing launching applications Force Tags apps -Resource ../resources/ssh_keywords.resource -Resource ../config/variables.robot -Suite Setup Set Variables ${DEVICE} +Resource ../../resources/ssh_keywords.resource +Resource ../../config/variables.robot Suite Teardown Close All Connections *** Test Cases *** diff --git a/Robot-Framework/test-suites/netvm.robot b/Robot-Framework/test-suites/bat-tests/netvm.robot similarity index 91% rename from Robot-Framework/test-suites/netvm.robot rename to Robot-Framework/test-suites/bat-tests/netvm.robot index 65af6f4..23f6199 100644 --- a/Robot-Framework/test-suites/netvm.robot +++ b/Robot-Framework/test-suites/bat-tests/netvm.robot @@ -4,9 +4,8 @@ *** Settings *** Documentation Testing Network VM Force Tags netvm -Resource ../resources/ssh_keywords.resource -Resource ../config/variables.robot -Suite Setup Set Variables ${DEVICE} +Resource ../../resources/ssh_keywords.resource +Resource ../../config/variables.robot Suite Teardown Close All Connections @@ -38,6 +37,7 @@ Wifi passthrought into NetVM ... Connect to ghaf host AND Connect to netvm via tunnel Configure wifi ${netvm} ${SSID} ${wifi_pswd} Check Network Availability ${netwotk_ip} expected_result=True + Log To Console Switch connection to Ghaf Host Switch Connection ${ghaf_host} Check Network Availability ${netwotk_ip} expected_result=False [Teardown] Remove Wifi configuration @@ -45,7 +45,7 @@ Wifi passthrought into NetVM NetVM stops and starts successfully [Documentation] Verify that NetVM stops properly and starts after that [Tags] bat SP-T52 - [Setup] Run Keywords Connect to ghaf host AND Verify service status service=${netvm_service} + [Setup] Connect to ghaf host Restart NetVM [Teardown] Run Keywords Start NetVM if dead AND Close All Connections @@ -58,8 +58,10 @@ NetVM is wiped after restarting Create file /etc/test.txt Switch Connection ${ghaf_host} Restart NetVM + Check Network Availability ${netvm_ip} expected_result=True range=5 Connect to netvm via tunnel - SSHLibrary.File Should Not Exist /etc/test.txt + Log To Console Create if created file still exists + Check file doesn't exist /etc/test.txt [Teardown] Run Keywords Close tunnel ${ghaf_host} AND Close All Connections @@ -72,19 +74,23 @@ Restart NetVM Stop NetVM Sleep ${delay} Start NetVM + Check if ssh is ready on netvm Create tunnel [Documentation] Set up forwarding from a local port through a tunneled connection to NetVM ${command}= Set Variable ssh -o StrictHostKeyChecking=no -L ${DEVICE_IP_ADDRESS}:${local_port}:${NETVM_IP}:22 ${NETVM_IP} -fN ${output}= Execute Command ${command} + @{pid}= Find pid by name ${command} Close tunnel [Documentation] Check if tunnel to netvm exists and kill the process [Arguments] ${ghaf_host} Switch Connection ${ghaf_host} + Log to Console ${\n}Check if there are existing tunnels to NetVM ${command}= Set Variable ssh -o StrictHostKeyChecking=no -L ${DEVICE_IP_ADDRESS}:${local_port}:${NETVM_IP}:22 ${NETVM_IP} -fN @{pid}= Find pid by name ${command} IF @{pid} != @{EMPTY} + Log to Console Close existing tunnels with pids: @{pid} Kill process @{pid} sig=9 END @@ -106,13 +112,14 @@ Connect to netvm in console Connect to netvm via tunnel [Documentation] Create tunnel using port ${local_port}, connect to netvm directly from test run machine, ... allow using standard SSHLibrary commands, like 'Execute Command' - Log To Console Configuring tunnel... Switch connection ${ghaf_host} Close tunnel ${ghaf_host} + Log To Console Configuring tunnel... Write ssh-keygen -R ${netvm_ip} Copy keys Clear iptables rules Create tunnel + Log To Console Connecting to NEtVM via tunnel ${connection}= Connect IP=${DEVICE_IP_ADDRESS} PORT=${local_port} target_output=ghaf@netvm Set Global Variable ${netvm} ${connection} [Return] ${netvm} @@ -139,6 +146,7 @@ Clear iptables rules Execute Command iptables -F sudo=True sudo_password=${PASSWORD} Login into NetVM + Log To Console Login into NetVM Write ssh-keygen -R ${netvm_ip} Write ssh ${LOGIN}@${netvm_ip} ${output}= Read delay=0.5s @@ -160,11 +168,13 @@ Login into NetVM Configure wifi [Arguments] ${netvm} ${SSID} ${passw} Switch Connection ${netvm} + Log To Console Configuring Wifi Execute Command sh -c "wpa_passphrase ${SSID} ${passw} > /etc/wpa_supplicant.conf" sudo=True sudo_password=${PASSWORD} Execute Command systemctl restart wpa_supplicant.service sudo=True sudo_password=${PASSWORD} Remove Wifi configuration Switch Connection ${netvm} + Log To Console Removing Wifi configuration Execute Command rm /etc/wpa_supplicant.conf sudo=True sudo_password=${PASSWORD} Execute Command systemctl restart wpa_supplicant.service sudo=True sudo_password=${PASSWORD} [Teardown] Close All Connections diff --git a/Robot-Framework/test-suites/bat-tests/others.robot b/Robot-Framework/test-suites/bat-tests/others.robot new file mode 100644 index 0000000..c0fbed4 --- /dev/null +++ b/Robot-Framework/test-suites/bat-tests/others.robot @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2022-2023 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Documentation Common system tests +Resource ../../resources/ssh_keywords.resource + +*** Test Cases *** + +Test ghaf version format + [Documentation] Test getting Ghaf version and verify its format: + ... Expected format: major.minor.yyyymmdd.commit_hash + [Tags] bat SP-T59 + [Setup] Connect + Verify Ghaf Version Format + [Teardown] Close All Connections + +Test nixos version format + [Documentation] Test getting Nixos version and verify its format: + ... Expected format: major.minor.yyyymmdd.commit_hash (name) + [Tags] bat SP-T60 + [Setup] Connect + Verify Nixos Version Format + [Teardown] Close All Connections + + diff --git a/Robot-Framework/test-suites/boot_test.robot b/Robot-Framework/test-suites/boot_test.robot index 4f33416..a59e44d 100644 --- a/Robot-Framework/test-suites/boot_test.robot +++ b/Robot-Framework/test-suites/boot_test.robot @@ -26,32 +26,25 @@ Verify booting after restart by power ELSE Log To Console The device started END - IF "${CONNECTION_TYPE}" == "ssh" Verify Systemctl status ELSE IF "${CONNECTION_TYPE}" == "serial" Verify Systemctl status via serial END - -Test ghaf version format - [Documentation] Test getting Ghaf version and verify its format: - ... Expected format: major.minor.yyyymmdd.commit_hash - [Tags] bat SP-T59 - [Setup] Connect - Verify Ghaf Version Format - [Teardown] Close All Connections - -Test nixos version format - [Documentation] Test getting Nixos version and verify its format: - ... Expected format: major.minor.yyyymmdd.commit_hash (name) - [Tags] bat SP-T60 - [Setup] Connect - Verify Nixos Version Format - [Teardown] Close All Connections + [Teardown] Teardown *** Keywords *** +Teardown + IF "${CONNECTION_TYPE}" == "ssh" + Run Keyword If Test Failed ssh_keywords.Save log + ELSE IF "${CONNECTION_TYPE}" == "serial" + Run Keyword If Test Failed serial_keywords.Save log + END + Close All Connections + Delete All Ports + Check If Device Is Up [Arguments] ${range}=20 ${start_time}= Get Time epoch