Robot tests EXTN

.github/workflows/robot-tests.yml → .github/workflows/e2e_test.yml

@@ -1,4 +1,4 @@
-name: Robot Tests
+name: E2E Tests
 
 on:
   pull_request:
@@ -83,79 +83,32 @@ jobs:
           pip install --upgrade pip
           pip install -r test/requirements.txt
 
-    - name: Download reports windows amd64
-      uses: actions/download-artifact@v4
-      with:
-        name: reports-win-amd64
-        path: reports-win-amd64
-
-    - name: Download reports windows arm64
-      uses: actions/download-artifact@v4
-      with:
-        name: reports-win-arm64
-        path: reports-win-arm64
-
-    - name: Download reports linux amd64
-      uses: actions/download-artifact@v4
-      with:
-        name: reports-lin-amd64
-        path: reports-lin-amd64
-
-    - name: Download reports linux arm64
-      uses: actions/download-artifact@v4
-      with:
-        name: reports-lin-arm64
-        path: reports-lin-arm64
-
-    - name: Download reports darwin amd64
-      uses: actions/download-artifact@v4
-      with:
-        name: reports-mac-amd64
-        path: reports-mac-amd64
-
     - name: Download reports darwin arm64
       uses: actions/download-artifact@v4
       with:
-        name: reports-mac-arm64
-        path: reports-mac-arm64
+        path: artifacts
+        pattern: reports-*
 
     - name: Consolidate robot test results
-      continue-on-error: true
+      working-directory: artifacts
       run: |
-        python -m robot.rebot --name Collective_Robot_Results --outputdir Collective_Robot_Results --output output.xml \
-          ./reports-win-amd64/output.xml ./reports-win-amd64/output.xml \
-          ./reports-lin-arm64/output.xml ./reports-lin-arm64/output.xml \
+        python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
+          ./reports-win-amd64/output.xml ./reports-win-arm64/output.xml \
+          ./reports-lin-amd64/output.xml ./reports-lin-arm64/output.xml \
           ./reports-mac-amd64/output.xml ./reports-mac-arm64/output.xml
 
-    - name: Convert xml to junit xml
+    - name: Generate Summary report
       if: always()
       run: |
-        python -m robot.rebot -x test_results_junit.xml Collective_Robot_Results/output.xml
+        python ./test/lib/execution_summary.py artifacts/collective_robot_results/output.xml
 
-    - name: Publish Test Results
+    - name: Print E2E Report
       if: always()
-      uses: EnricoMi/publish-unit-test-result-action@v2
-      with:
-        report_individual_runs: true
-        files: "test_results_junit.xml"
-
-    - name: HTML Preview
-      if: always()
-      id: html_preview
-      uses: pavi2410/html-preview-action@v2
-      with:
-        html_file: 'Collective_Robot_Results/log.html'
+      run: cat summary_report.md >> $GITHUB_STEP_SUMMARY
 
     - name: Archieve consolidated test results
       if: always()
       uses: actions/upload-artifact@v4
       with:
         name: consolidated-reports
-        path: Collective_Robot_Results
-
-    - name: Send report to commit
-      if: always()
-      uses: joonvena/[email protected]
-      with:
-        report_path: Collective_Robot_Results
-        gh_access_token: ${{ secrets.GITHUB_TOKEN }}
+        path: artifacts/collective_robot_results

.gitignore

@@ -1 +1,7 @@
-build
+# Directories
+/build*/
+**/build
+**/__pycache__
+
+# Files
+**/stdout.txt

test/lib/elf_compare.py

@@ -64,7 +64,7 @@ def compare_elf_files(self):
         for context in context_list:
             logger.info('Processing context: %s.%s+%s' % (context.project, context.build, context.target))
             path = self.get_elf_file_path(self.cbuildgen_out_dir, context)
-            logger.debug('Path: %s' % path)
+            logger.info('Path: %s' % path)
             if path != '':
                 cbuildgen_elf_file = os.path.join(self.cbuildgen_out_dir, path)
                 cbuild2cmake_elf_file = os.path.join(self.cbuild2cmake_out_dir, path)
@@ -77,6 +77,7 @@ def compare_elf_files(self):
 
     def get_elf_file_path(self, basePath, context: Context):
         file_extns = [".axf", ".elf"]
+        logger.info('basePath: %s' % basePath)
         for extn in file_extns:
             filePath = os.path.join(basePath, context.project, context.target, context.build, context.project+extn)
             if os.path.exists(filePath):

test/lib/execution_summary.py

@@ -0,0 +1,99 @@
+# This script is designed to process the output.xml file generated by Robot
+# Framework, extract the test statuses, and create a markdown file based
+# on the parsed information.
+# 
+# execution_summary.py <path to input output.xml> [output file summary.md]
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+from robot.api import ExecutionResult, ResultVisitor
+from robot.result.model import TestCase
+from robot.result.executionresult import Result 
+
+class ResultVisitorEx(ResultVisitor):
+    def __init__(self, markdown_file='summary_report.md'):
+        self.failed_tests = {}
+        self.passed_tests = {}
+        self.skipped_tests = {}
+        self.markdown_file = markdown_file
+
+        # Remove existing markdown file if it exists
+        if os.path.exists(markdown_file):
+            os.remove(markdown_file)
+
+    def visit_test(self, test: TestCase):
+        tags = ", ".join(test.tags)
+        duration = "{:.2f} s".format(test.elapsed_time.total_seconds())
+        status = (test.name, test.message, duration, test.parent.name)
+        test_status = self.failed_tests if test.status == 'FAIL' else (
+            self.passed_tests if test.status == 'PASS' else self.skipped_tests)
+        if tags not in test_status:
+            test_status[tags] = []
+        test_status[tags].append(status)
+
+    def end_result(self, result: Result):
+        with open(self.markdown_file, "w") as f:
+            f.write("# Robot Framework Report\n\n")
+            self.__write_test_env(f)
+            f.write("## Summary\n\n")
+            f.write("|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|\n")
+            f.write("|:----:|:----:|:-----:|:---:|\n")
+            f.write(f"|{result.statistics.total.passed}|{result.statistics.total.failed}|{result.statistics.total.skipped}|{result.statistics.total.total}|\n")
+            self.__write_test_section(f, self.passed_tests, "Passed Tests", "|Tag|Test|:clock1030: Duration|Suite|\n")
+            self.__write_test_section(f, self.failed_tests, "Failed Tests", "|Tag|Test|Message|:clock1030: Duration|Suite|\n")
+            self.__write_test_section(f, self.skipped_tests, "Skipped Tests", "|Tag|Test|Suite|\n")
+
+    def __write_test_env(self, file):
+        tool_dict = {"cbuild": shutil.which("cbuild"),
+                    "cpackget": shutil.which("cpackget"),
+                    "csolution": shutil.which("csolution")}
+
+        file.write("## Test Environment\n\n")
+        for tool, path in tool_dict.items():
+            version = "unknown"
+            version_cmd = f"{tool} -V"
+            try:
+                output = subprocess.run(version_cmd, shell=True, check=True, text=True, capture_output=True)
+                version_match = re.search(r"(\d+\.\d+\.\d+.*) \(C\)", output.stdout)
+                if version_match:
+                    version = version_match.group(1)
+            except subprocess.CalledProcessError:
+                pass
+
+            tool_info = f"- **{tool}**: `version {version}, {path}`"
+            file.write(f"{tool_info}\n")
+        file.write(f"- **CMSIS_PACK_ROOT**: `{os.getenv('CMSIS_PACK_ROOT')}`\n\n")
+
+
+    def __write_test_section(self, file, test_dict, section_header, table_header):
+        if len(test_dict) != 0:
+            file.write(f"\n## {section_header}\n\n")
+            file.write(table_header)
+            tokens = table_header.split('|')
+            table_sep = "|:-"
+            num_req_sep = len(tokens) - 2
+            for i in range(num_req_sep):
+                table_sep += "--:|" 
+                if i != (num_req_sep - 1):
+                    table_sep += ":-" 
+            file.write(table_sep + "\n")
+            for key, value in test_dict.items():
+                for name, msg, duration, suite in value:
+                    if section_header.startswith("Pass"):
+                        file.write(f"|{key}|{name}|{duration}|{suite}|\n")
+                    elif section_header.startswith("Fail"):
+                        file.write(f"|{key}|{name}|{msg}|{duration}|{suite}|\n")
+                    elif section_header.startswith("Skip"):
+                        file.write(f"|{key}|{name}|{suite}|\n")
+
+if __name__ == '__main__':
+    # Get output and markdown file paths from command line arguments
+    output_file = sys.argv[1] if len(sys.argv) > 1 else "output.xml"
+    markdown_file = sys.argv[2] if len(sys.argv) > 2 else "summary_report.md"
+
+    # Parse the Robot Framework output file and generate the summary report
+    result = ExecutionResult(output_file)
+    result.visit(ResultVisitorEx(markdown_file))

test/resources/global.resource

@@ -1,4 +1,6 @@
 *** Variables ***
 ${RESOURCES}        ${CURDIR}      # Path to the resource directory
 ${Build}            build          # Directory name where example build files are copied
-${Data}             data           # Directory name of test data source
+${Data}             data           # Directory name of test data source
+${Pass}             ${0}
+${Fail}             ${1}

test/resources/utils.resource

@@ -3,6 +3,7 @@ Documentation    A collection of commonly used keywords across multiple test sui
 Library          Collections
 Library          Process
 Library          OperatingSystem
+Library          ..${/}lib${/}utils.py
 Resource         global.resource
 
 
@@ -28,8 +29,10 @@ Run Program
     [Arguments]                 ${exe_path}      ${input_File}    @{args}
     ${result}    Run Process    ${exe_path}      ${input_File}    @{args}
     ...                         shell=True       stdout=${CURDIR}/stdout.txt
-    Log Many                    StdOut: ${result.stdout}    Return Code: ${result.rc}
-    RETURN                      ${result.rc}
+
+    ${return_code}=             Set Variable If    ${result.rc} == ${0}    ${0}    ${1}
+    Log Many                    StdOut: ${result.stdout}    Return Code: ${return_code}
+    RETURN                      ${return_code}
 
 Run cbuild
     [Documentation]                Run cbuild with specified arguments
@@ -52,32 +55,28 @@ Remove Directory with Content
     Remove Directory                ${target_dir}      recursive=${True}
     Wait Until Removed              ${target_dir}      timeout=5 seconds
 
-# Checkout GitHub Repository
-#     [Documentation]                 Checkout github repository
-#     [Arguments]                     ${github_repo_url}    ${dest_path}
-#     ${result}=    Run Process       git    clone          ${github_repo_url}    ${dest_path}
-#     Log                             ${result.stdout}
-#     Log                             ${result.stderr}
-#     Should Be Equal As Integers     ${result.rc}          ${0}
-
-Run Project With cbuild2cmake
-    [Arguments]     ${input_file}    ${args}=@{EMPTY}
-    ${ex_args}=     Append Additional Arguments     ${args}    --cbuild2cmake
-    ${result}=      Run cbuild        ${input_file}      ${ex_args}
-    RETURN          ${result}
+Checkout GitHub Repository
+    [Documentation]                 Checkout github repository
+    [Arguments]                     ${github_repo_url}    ${dest_path}
+    ${result}=    Run Process       git    clone          ${github_repo_url}    ${dest_path}
+    Log                             ${result.stdout}
+    Log                             ${result.stderr}
+    Should Be Equal As Integers     ${result.rc}          ${0}
 
 Run Project with cbuildgen
-    [Arguments]    ${input_file}      ${example_name}    ${args}=@{EMPTY}
-    ${ex_args}=     Append Additional Arguments     ${args}    --output    ${TEST_DATA_DIR}${/}${example_name}${/}out_dir
-    ${result}=      Run cbuild        ${input_file}      ${ex_args}
-    RETURN          ${result}
+    [Arguments]         ${input_file}       ${expect}    ${args}=@{EMPTY}
+    ${parent_path}=    Get Parent Directory Path    ${input_file}
+    ${ex_args}=         Append Additional Arguments     ${args}    --output    ${parent_path}${/}out_dir
+    ${rc}=              Run cbuild          ${input_file}      ${ex_args}
+    Should Be Equal     ${rc}               ${expect}          msg=Unexpected status returned by cbuildgen execution
+    RETURN              ${rc}
 
-Validate Build Status
-    [Arguments]         ${rc_cbuildgen}        ${rc_cbuild2cmake}    ${rc_expected}
-    Should Be Equal     ${rc_cbuildgen}        ${rc_expected}        msg=Unexpected status returned by cbuildgen execution
-    Should Be Equal     ${rc_cbuild2cmake}     ${rc_expected}        msg=Unexpected status returned by cbuild2cmake execution
-    ${status_match}=    Evaluate    ${rc_cbuildgen} == ${rc_cbuild2cmake} and ${rc_cbuildgen} == ${rc_expected}
-    RETURN              ${status_match}
+Run Project With cbuild2cmake
+    [Arguments]         ${input_file}     ${expect}          ${args}=@{EMPTY}
+    ${ex_args}=         Append Additional Arguments          ${args}    --cbuild2cmake
+    ${result}=          Run cbuild        ${input_file}      ${ex_args}
+    Should Be Equal     ${result}         ${expect}          msg=Unexpected status returned by cbuild2cmake execution
+    RETURN              ${result}
 
 Append Additional Arguments
     [Arguments]                  ${list}    @{values}