From 92570140e84cc51d1551f36b6c57490dce14411a Mon Sep 17 00:00:00 2001 From: Artiom Divak Date: Mon, 20 Nov 2023 12:36:15 +0200 Subject: [PATCH] Added finish step script to make html from info files Running script when WITH_COVERAGE is 1. This script will run after all tests and will get an html report for the coverage from all the info files. Fixes: https://github.com/eclipse-bluechi/bluechi/issues/397 Signed-off-by: Artiom Divak --- tests/README.md | 22 +++++++++++ tests/bluechi_test/container.py | 1 + tests/containers/integration-test-base | 6 +++ tests/containers/integration-test-local | 6 --- tests/plans/tier0.fmf | 5 +++ tests/scripts/create_coverage_report.py | 51 +++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 tests/scripts/create_coverage_report.py diff --git a/tests/README.md b/tests/README.md index 3efd5b2547..23efcfaa9a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -86,6 +86,21 @@ tmt run -v -eWITH_VALGRIND=1 If `valgrind` detects a memory leak in a test, the test will fail, and the logs will be found in the test `data` directory. +## Running integration tests with code coverage report + +To produce code coverage report during integration tests run set WITH_COVERAGE environment variable as follows: + +```shell +tmt run -v -eWITH_COVERAGE=1 +``` + +After the integration tests will finish the code coverage html result can be found in 'res' dir inside the tmt execution result directory for example + +`/var/tmp/tmt/run-001/plans/tier0/report/default-0/report` + +To Build RPM with coverage the WITH_COVERAGE=1 option have to be added to the build. +For an example with skipper please refer to 'Running integration tests with local BlueChi build'. + ## Running integration tests with local BlueChi build In order to run integration tests for your local BlueChi build, you need have BlueChi RPM packages built from your source @@ -108,6 +123,13 @@ cd ~/bluechi skipper make rpm ``` +To Build RPM locally with coverage rpm please run: + +```shell +cd ~/bluechi +skipper make rpm WITH_COVERAGE=1 +``` + When done it's required to create DNF repository from those RPMs: ```shell diff --git a/tests/bluechi_test/container.py b/tests/bluechi_test/container.py index 97dcc3795d..ceb4eec384 100644 --- a/tests/bluechi_test/container.py +++ b/tests/bluechi_test/container.py @@ -85,6 +85,7 @@ def gather_coverage(self, data_coverage_dir: str) -> None: result, output = self.container.exec_run( f"geninfo {gcda_file_location} -b {src_file_location} -o {coverage_file_name}") + self.get_file(f"/{coverage_file_name}", data_coverage_dir) def cleanup(self): diff --git a/tests/containers/integration-test-base b/tests/containers/integration-test-base index b264bebea6..9a4132cb0c 100644 --- a/tests/containers/integration-test-base +++ b/tests/containers/integration-test-base @@ -1,5 +1,11 @@ FROM quay.io/centos/centos:stream9 +RUN dnf install -y epel-release + +RUN dnf install --nodocs \ + lcov \ + -y + RUN dnf upgrade --refresh -y --nodocs && \ dnf install --nodocs \ policycoreutils-python-utils \ diff --git a/tests/containers/integration-test-local b/tests/containers/integration-test-local index eeee2d1422..27cdfbd929 100644 --- a/tests/containers/integration-test-local +++ b/tests/containers/integration-test-local @@ -4,12 +4,6 @@ RUN mkdir -p /tmp/bluechi-rpms COPY ./bluechi-rpms /tmp/bluechi-rpms -RUN dnf install -y epel-release - -RUN dnf install --nodocs \ - lcov \ - -y - RUN dnf install --repo bluechi-rpms \ --repofrompath bluechi-rpms,file:///tmp/bluechi-rpms/ \ --nogpgcheck \ diff --git a/tests/plans/tier0.fmf b/tests/plans/tier0.fmf index 59d9fd723b..5edf4ee599 100644 --- a/tests/plans/tier0.fmf +++ b/tests/plans/tier0.fmf @@ -22,3 +22,8 @@ execute: how: tmt report: how: junit +finish: + - name: Create coverage report + how: shell + script: | + pytest -svv scripts/create_coverage_report.py \ No newline at end of file diff --git a/tests/scripts/create_coverage_report.py b/tests/scripts/create_coverage_report.py new file mode 100644 index 0000000000..cb2ac873c3 --- /dev/null +++ b/tests/scripts/create_coverage_report.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +import os +from typing import Dict +from bluechi_test.util import read_file +import subprocess + +from bluechi_test.test import BluechiTest +from bluechi_test.container import BluechiControllerContainer, BluechiNodeContainer +from bluechi_test.config import BluechiControllerConfig, BluechiNodeConfig + + +def merge_all_info_files(start_path): + add_merged_file = "" + all_bluchi_coverage_files = subprocess.check_output( + f"find {start_path} -name bluechi-coverage", shell=True).decode() + all_bluchi_coverage_files = all_bluchi_coverage_files.split("\n") + all_bluchi_coverage_files.pop() + for bluechi_coverage_dir in all_bluchi_coverage_files: + for path in os.listdir(bluechi_coverage_dir): + if "info" in path: + info_file = os.path.join(bluechi_coverage_dir, path) + if os.path.isfile(info_file): + if os.stat(info_file).st_size != 0: + os.system( + f'lcov --add-tracefile {info_file} {add_merged_file} -o merged.info > /dev/null') + add_merged_file = "-a merged.info" + + +def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer]): + path_to_info_files = os.environ["TMT_TREE"].split('/tree')[0] + "/execute/data/" + path_to_tests_results = os.environ["TMT_TREE"].split('/tree')[0] + "/report/default-0" + merge_file_name = "merged.info" + merge_dir = "/tmp" + report_dir_name = "/report" + + merge_all_info_files(path_to_info_files) + content = read_file(merge_file_name) + ctrl.create_file(merge_dir, merge_file_name, content) + ctrl.exec_run(f"genhtml {merge_dir}/{merge_file_name} --output-directory={report_dir_name}") + ctrl.get_file(report_dir_name, path_to_tests_results) + + +def test_create_coverag_report( + bluechi_test: BluechiTest, + bluechi_ctrl_default_config: BluechiControllerConfig, + bluechi_node_default_config: BluechiNodeConfig): + + if (os.getenv("WITH_COVERAGE") == "1"): + bluechi_test.set_bluechi_controller_config(bluechi_ctrl_default_config) + bluechi_test.run(exec)