From fb22b07bc0b62d0e82b7239b5adb7d6d380f02d6 Mon Sep 17 00:00:00 2001 From: Mariia_Azbeleva Date: Tue, 15 Aug 2023 14:59:43 +0300 Subject: [PATCH] Add performance tests Signed-off-by: Mariia Azbeleva --- Robot-Framework/config/variables.robot | 15 +- .../lib/PerformanceDataProcessing.py | 196 ++++++++++++++++++ Robot-Framework/lib/output_parser.py | 59 ++++++ .../resources/ssh_keywords.resource | 13 ++ .../test-suites/performance/performance.robot | 93 +++++++++ requirements | 1 + 6 files changed, 364 insertions(+), 13 deletions(-) create mode 100644 Robot-Framework/lib/PerformanceDataProcessing.py create mode 100644 Robot-Framework/test-suites/performance/performance.robot diff --git a/Robot-Framework/config/variables.robot b/Robot-Framework/config/variables.robot index f2c59d7..b65edd0 100644 --- a/Robot-Framework/config/variables.robot +++ b/Robot-Framework/config/variables.robot @@ -4,18 +4,6 @@ *** Settings *** Library OperatingSystem -*** Variables *** -${DEVICE} ${DEVICE} - -${SERIAL_PORT} ${EMPTY} -${DEVICE_IP_ADDRESS} ${EMPTY} -${SOCKET_IP_ADDRESS} ${EMPTY} -${LOGIN} ${EMPTY} -${PASSWORD} ${EMPTY} -${PLUG_USERNAME} ${EMPTY} -${PLUG_PASSWORD} ${EMPTY} -${netvm_service} ${EMPTY} - *** Keywords *** Set Variables @@ -25,7 +13,8 @@ Set Variables Set Global Variable ${SERIAL_PORT} ${config['addresses']['${DEVICE}']['serial_port']} Set Global Variable ${DEVICE_IP_ADDRESS} ${config['addresses']['${DEVICE}']['device_ip_address']} Set Global Variable ${SOCKET_IP_ADDRESS} ${config['addresses']['${DEVICE}']['socket_ip_address']} - Set Global Variable ${netvm_service} microvm@netvm.service + Set Global Variable ${NETVM_SERVICE} microvm@netvm.service + Set Global Variable ${THREADS_NUMBER} ${config['addresses']['${DEVICE}']['threads']} Read Config [Arguments] ${file_path}=../config/test_config.json diff --git a/Robot-Framework/lib/PerformanceDataProcessing.py b/Robot-Framework/lib/PerformanceDataProcessing.py new file mode 100644 index 0000000..dce5521 --- /dev/null +++ b/Robot-Framework/lib/PerformanceDataProcessing.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: 2022-2023 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +import csv +import os +import matplotlib.pyplot as plt +import logging +from robot.api.deco import keyword + + +class PerformanceDataProcessing: + + def __init__(self, device): + # Initialize the instance variable with the global variable value + self.data_dir = "../../../Performance_test_results/" + self.device = device + + def _write_to_csv(self, test_name, data): + file_path = os.path.join(self.data_dir, f"{self.device}_{test_name}.csv") + logging.info(f"Writing data to {file_path}") + with open(file_path, 'a', newline='') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(data) + + @keyword + def write_cpu_to_csv(self, test_name, build_number, cpu_data): + data = [build_number, + cpu_data['cpu_events_per_second'], + cpu_data['min_latency'], + cpu_data['avg_latency'], + cpu_data['max_latency'], + cpu_data['cpu_events_per_thread'], + cpu_data['cpu_events_per_thread_stddev'], + self.device] + self._write_to_csv(test_name, data) + + @keyword + def write_mem_to_csv(self, test_name, build_number, mem_data): + data = [build_number, + mem_data['operations_per_second'], + mem_data['data_transfer_speed'], + mem_data['min_latency'], + mem_data['avg_latency'], + mem_data['max_latency'], + mem_data['avg_events_per_thread'], + mem_data['events_per_thread_stddev'], + self.device] + self._write_to_csv(test_name, data) + + @keyword + def read_cpu_csv_and_plot(self, test_name): + build_numbers = [] + cpu_events_per_second = [] + min_latency = [] + avg_latency = [] + max_latency = [] + cpu_events_per_thread = [] + cpu_events_per_thread_stddev = [] + + with open(f"{self.data_dir}{self.device}_{test_name}.csv", 'r') as csvfile: + csvreader = csv.reader(csvfile) + logging.info("Reading data from csv file...") + for row in csvreader: + if row[7] == self.device: + build_numbers.append(str(row[0])) + cpu_events_per_second.append(float(row[1])) + min_latency.append(float(row[2])) + avg_latency.append(float(row[3])) + max_latency.append(float(row[4])) + cpu_events_per_thread.append(float(row[5])) + cpu_events_per_thread_stddev.append(float(row[6])) + + plt.figure(figsize=(20, 10)) + plt.set_loglevel('WARNING') + + # Plot 1: CPU Events per Second + plt.subplot(3, 1, 1) + plt.ticklabel_format(axis='y', style='plain') + plt.plot(build_numbers, cpu_events_per_second, marker='o', linestyle='-', color='b') + plt.title('CPU Events per Second', loc='right', fontweight="bold") + plt.ylabel('CPU Events per Second') + plt.grid(True) + plt.xticks(build_numbers) + + # Plot 2: CPU Events per Thread + plt.subplot(3, 1, 2) + plt.ticklabel_format(axis='y', style='plain') + plt.plot(build_numbers, cpu_events_per_thread, marker='o', linestyle='-', color='b') + plt.title('CPU Events per Thread', loc='right', fontweight="bold") + plt.ylabel('CPU Events per Thread') + plt.grid(True) + plt.xticks(build_numbers) + # Create line chart with error bars on the same subplot + plt.errorbar(build_numbers, cpu_events_per_thread, + yerr=cpu_events_per_thread_stddev, + capsize=4) + + # Plot 3: Latency + plt.subplot(3, 1, 3) + plt.ticklabel_format(axis='y', style='plain') + plt.plot(build_numbers, avg_latency, marker='o', linestyle='-', color='b', label='Avg') + plt.ylabel('Avg Latency (ms)') + plt.legend(loc='upper left') + plt.xlabel('Build Number') + plt.twinx() + plt.plot(build_numbers, max_latency, marker='o', linestyle='-', color='r', label='Max') + plt.plot(build_numbers, min_latency, marker='o', linestyle='-', color='g', label='Min') + plt.ylabel('Max/Min Latency (ms)') + plt.legend(loc='upper right') + plt.title('Latency', loc='right', fontweight="bold") + plt.grid(True) + plt.xticks(build_numbers) + + plt.suptitle(f'{test_name} ({self.device})', fontsize=16, fontweight='bold') + + plt.tight_layout() + plt.savefig(f'../test-suites/{self.device}_{test_name}.png') # Save the plot as an image file + + @keyword + def read_mem_csv_and_plot(self, test_name): + build_numbers = [] + operations_per_second = [] + data_transfer_speed = [] + min_latency = [] + avg_latency = [] + max_latency = [] + avg_events_per_thread = [] + events_per_thread_stddev = [] + + with open(f"{self.data_dir}{self.device}_{test_name}.csv", 'r') as csvfile: + csvreader = csv.reader(csvfile) + logging.info("Reading data from csv file...") + for row in csvreader: + if row[8] == self.device: + build_numbers.append(str(row[0])) + operations_per_second.append(float(row[1])) + data_transfer_speed.append(float(row[2])) + min_latency.append(float(row[3])) + avg_latency.append(float(row[4])) + max_latency.append(float(row[5])) + avg_events_per_thread.append(float(row[6])) + events_per_thread_stddev.append(float(row[7])) + + plt.figure(figsize=(20, 10)) + plt.set_loglevel('WARNING') + + # Plot 1: Operations Per Second + plt.subplot(3, 1, 1) + plt.ticklabel_format(axis='y', style='sci', useMathText=True) + plt.plot(build_numbers, operations_per_second, marker='o', linestyle='-', color='b') + plt.title('Operations per Second', loc='right', fontweight="bold") + plt.ylabel('Operations per Second') + plt.grid(True) + plt.xticks(build_numbers) + + # Plot 2: Data Transfer Speed + plt.subplot(3, 1, 2) + plt.ticklabel_format(axis='y', style='plain') + plt.plot(build_numbers, data_transfer_speed, marker='o', linestyle='-', color='b') + plt.title('Data Transfer Speed', loc='right', fontweight="bold") + plt.ylabel('Data Transfer Speed (MiB/sec)') + plt.grid(True) + plt.xticks(build_numbers) + + # Plot 3: Latency + plt.subplot(3, 1, 3) + plt.ticklabel_format(axis='y', style='plain') + plt.plot(build_numbers, avg_latency, marker='o', linestyle='-', color='b', label='Avg') + plt.ylabel('Avg Latency (ms)') + plt.legend(loc='upper left') + plt.grid(True) + plt.xlabel('Build Number') + plt.twinx() + plt.plot(build_numbers, max_latency, marker='o', linestyle='-', color='r', label='Max') + plt.plot(build_numbers, min_latency, marker='o', linestyle='-', color='g', label='Min') + plt.ylabel('Max/Min Latency (ms)') + plt.legend(loc='upper right') + plt.title('Latency', loc='right', fontweight="bold") + plt.xticks(build_numbers) + + plt.suptitle(f'{test_name} ({self.device})', fontsize=16, fontweight='bold') + + plt.tight_layout() + plt.savefig(f'../test-suites/{self.device}_{test_name}.png') # Save the plot as an image file + + @keyword + def save_cpu_data(self, test_name, build_number, cpu_data): + + self.write_cpu_to_csv(test_name, build_number, cpu_data) + self.read_cpu_csv_and_plot(test_name) + + @keyword + def save_memory_data(self, test_name, build_number, cpu_data): + + self.write_mem_to_csv(test_name, build_number, cpu_data) + self.read_mem_csv_and_plot(test_name) diff --git a/Robot-Framework/lib/output_parser.py b/Robot-Framework/lib/output_parser.py index 3aa6edb..bfb36c2 100644 --- a/Robot-Framework/lib/output_parser.py +++ b/Robot-Framework/lib/output_parser.py @@ -48,3 +48,62 @@ def verify_date_format(date_string): datetime.strptime(date_string, '%Y%m%d') except ValueError: raise Exception("Wrong date format in version date field") + +def parse_cpu_results(output): + def extract_value(pattern, output): + match = re.search(pattern, output) + if match: + return match.group(1) + else: + raise Exception(f"Couldn't parse result of the test with pattern: {pattern}") + + output = re.sub(r'\033\[.*?m', '', output) # remove colors from serial console output + + cpu_events_per_second = extract_value(r'events per second:\s*([.\d]+)', output) + min_latency = extract_value(r'min:\s+([.\d]+)', output) + max_latency = extract_value(r'max:\s+([.\d]+)', output) + avg_latency = extract_value(r'avg:\s+([.\d]+)', output) + cpu_events_per_thread = extract_value(r'events \(avg\/stddev\):\s+([.\d]+)', output) + cpu_events_per_thread_stddev = extract_value(r'events \(avg\/stddev\):\s+[.\d]+\/([.\d]+)', output) + + cpu_data = { + 'cpu_events_per_second': cpu_events_per_second, + 'min_latency': min_latency, + 'max_latency': max_latency, + 'avg_latency': avg_latency, + 'cpu_events_per_thread': cpu_events_per_thread, + 'cpu_events_per_thread_stddev': cpu_events_per_thread_stddev + } + + return cpu_data + +def parse_memory_results(output): + def extract_value(pattern, output): + match = re.search(pattern, output) + if match: + return match.group(1) + else: + raise Exception(f"Couldn't parse result of the test with pattern: {pattern}") + + output = re.sub(r'\033\[.*?m', '', output) # remove colors from serial console output + + operations_per_second = extract_value(r'Total operations:\s*\d+ \(([.\d]+) per second', output) + data_transfer_speed = extract_value(r'\(([.\d]+) MiB\/sec\)', output) + min_latency = extract_value(r'min:\s+([.\d]+)', output) + max_latency = extract_value(r'max:\s+([.\d]+)', output) + avg_latency = extract_value(r'avg:\s+([.\d]+)', output) + avg_events_per_thread = extract_value(r'events \(avg\/stddev\):\s+([.\d]+)', output) + events_per_thread_stddev = extract_value(r'events \(avg\/stddev\):\s+[.\d]+\/([.\d]+)', output) + + mem_data = { + 'operations_per_second': operations_per_second, + 'data_transfer_speed': data_transfer_speed, + 'min_latency': min_latency, + 'max_latency': max_latency, + 'avg_latency': avg_latency, + 'avg_events_per_thread': avg_events_per_thread, + 'events_per_thread_stddev': events_per_thread_stddev + } + + return mem_data + diff --git a/Robot-Framework/resources/ssh_keywords.resource b/Robot-Framework/resources/ssh_keywords.resource index 8326a8a..fc4ffc7 100644 --- a/Robot-Framework/resources/ssh_keywords.resource +++ b/Robot-Framework/resources/ssh_keywords.resource @@ -223,3 +223,16 @@ Check if ssh is ready on netvm Sleep 1 END IF ${status} == False FAIL Port 22 of NetVM is not ready after ${timeout} + +Install sysbench tool + ${command_output}= Execute Command nix-env --query --installed + ${not_installed} = Run Keyword And Return Status Should Not Contain ${command_output} sysbench + IF ${not_installed} + Execute Command nix-env -i sysbench + ${command_output}= Execute Command nix-env --query --installed + Log To Console ${\n}Installed packages:${\n}${command_output} + Should Contain ${command_output} sysbench sysbench tool was not installed + Log To Console sysbench tool was succesfully installed + ELSE + Log To Console ${\n}sysbench tool was already installed + END diff --git a/Robot-Framework/test-suites/performance/performance.robot b/Robot-Framework/test-suites/performance/performance.robot new file mode 100644 index 0000000..6768739 --- /dev/null +++ b/Robot-Framework/test-suites/performance/performance.robot @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2022-2023 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Documentation Gathering performance data +Force Tags performance +Resource ../../resources/ssh_keywords.resource +Resource ../../config/variables.robot +Library ../../lib/output_parser.py +Library ../../lib/PerformanceDataProcessing.py ${DEVICE} +Suite Setup Common Setup +Suite Teardown Close All Connections + +*** Test Cases *** + +CPU One thread test + [Documentation] Run a CPU benchmark using Sysbench with a duration of 10 seconds and a SINGLE thread. + ... The benchmark records to csv CPU events per second, events per thread, and latency data. + ... Create visual plots to represent these metrics comparing to previous tests. + [Tags] performance cpu SP-T67-1 + ${output} Execute Command sysbench cpu --time=10 --threads=1 --cpu-max-prime=20000 run + Log ${output} + &{cpu_data} Parse Cpu Results ${output} + Save Cpu Data ${TEST NAME} ${buildID} ${cpu_data} + Log CPU Plot HTML + +CPU multimple threads test + [Documentation] Run a CPU benchmark using Sysbench with a duration of 10 seconds and MULTIPLE threads. + ... The benchmark records to csv CPU events per second, events per thread, and latency data. + ... Create visual plots to represent these metrics comparing to previous tests. + [Tags] performance cpu SP-T67-2 + ${output} Execute Command sysbench cpu --time=10 --threads=${threads_number} --cpu-max-prime=20000 run + Log ${output} + &{cpu_data} Parse Cpu Results ${output} + Save Cpu Data ${TEST NAME} ${buildID} ${cpu_data} + Log CPU Plot HTML + +Memory Read One thread test + [Documentation] Run a memory benchmark using Sysbench for 60 seconds with a SINGLE thread. + ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread, + ... and Latency for READ operations. + ... Create visual plots to represent these metrics comparing to previous tests. + [Tags] performance cpu SP-T67-3 + ${output} Execute Command sysbench memory --time=60 --memory-oper=read --threads=1 run + Log ${output} + &{cpu_data} Parse Memory Results ${output} + Save Memory Data ${TEST NAME} ${buildID} ${cpu_data} + Log Mem Plot HTML + +Memory Write One thread test + [Documentation] Run a memory benchmark using Sysbench for 60 seconds with a SINGLE thread. + ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread, + ... and Latency for WRITE operations. + ... Create visual plots to represent these metrics comparing to previous tests. + [Tags] performance cpu SP-T67-4 + ${output} Execute Command sysbench memory --time=60 --memory-oper=write --threads=1 run + Log ${output} + &{cpu_data} Parse Memory Results ${output} + Save Memory Data ${TEST NAME} ${buildID} ${cpu_data} + Log Mem Plot HTML + +Memory Read multimple threads test + [Documentation] Run a memory benchmark using Sysbench for 60 seconds with MULTIPLE threads. + ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread, + ... and Latency for READ operations. + ... Create visual plots to represent these metrics comparing to previous tests. + [Tags] performance cpu SP-T67-5 + ${output} Execute Command sysbench memory --time=60 --memory-oper=read --threads=${threads_number} run + Log ${output} + &{cpu_data} Parse Memory Results ${output} + Save Memory Data ${TEST NAME} ${buildID} ${cpu_data} + Log Mem Plot HTML + +Memory Write multimple threads test + [Documentation] Run a memory benchmark using Sysbench for 60 seconds with MULTIPLE threads. + ... The benchmark records Operations Per Second, Data Transfer Speed, Average Events per Thread, + ... and Latency for WRITE operations. + ... Create visual plots to represent these metrics comparing to previous tests. + [Tags] performance cpu SP-T67-6 + ${output} Execute Command sysbench memory --time=60 --memory-oper=write --threads=${threads_number} run + Log ${output} + &{cpu_data} Parse Memory Results ${output} + Save Memory Data ${TEST NAME} ${buildID} ${cpu_data} + Log Mem Plot HTML + + + +*** Keywords *** + +Common Setup + Set Variables ${DEVICE} + Connect + Install sysbench tool diff --git a/requirements b/requirements index f237bf3..449e49a 100644 --- a/requirements +++ b/requirements @@ -6,3 +6,4 @@ robotframework==5.0.1 PyP100==0.1.2 python-kasa~=0.5.1 robotframework-retryfailed==0.2.0 +matplotlib==3.7.2