diff --git a/Dockerfile b/Dockerfile index 7b43229..a2e8acc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,4 +12,4 @@ COPY export_to_markdown.py / # Code file to execute when the docker container starts up (`entrypoint.sh`) WORKDIR /github/workspace -ENTRYPOINT ["bash", "-x", "/entrypoint.sh"] +ENTRYPOINT ["bash", "/entrypoint.sh"] diff --git a/README.md b/README.md index af26cff..d876174 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Splunk AppInspect action -This action runs Splunk's AppInspect CLI against a provided a directory of a Splunk App. -It fails if the result contains any failures. +This action runs Splunk's AppInspect CLI against a provided directory of Splunk App. +It fails if the result contains any failures or manual checks are not vetted. The (json) result will be written to the file specified with [`result-file`](#result-file). This can be uploaded for later viewing to use in another step/job using [`actions/upload-artifact@v2`](https://github.com/marketplace/actions/upload-a-build-artifact). @@ -29,35 +29,58 @@ Appinspect tags to exclude `required`: `false` -### `app_vetting` -Path to app vetting yaml file. Used only if `manual` in `included_tags` +### `appinspect_manual_checks` +Path to file which contains list of manual checks -`default`: `.app-vetting.yaml` +`required`: `false` +`default`: `.appinspect.manualcheck.yaml` + +### `appinspect_expected_failures` +Path to file which contains list of expected appinspect failures + +`required`: `false` +`default`: `.appinspect.expect.yaml` ### `manual_check_markdown` -Path for generated file with markdown for manual checks. Used only if `manual` in `included_tags` +Path to generated file with markdown for manual checks +`required`: `false` `default`: `manual_check_markdown.txt` +### `appinspect_expected_failures` +Path to generated file with markdown for expected appinspect failures + +`required`: `false` +`default`: `expected_failure_markdown.txt` + ## Outputs ### `status`: `pass|fail` -## Using manual tag -Running `appinspect-cli-action` with `manual` tag in `included_tags` detects checks that need to be verified manually and tests if all of them were already reviewed - if not the action will fail. ### Manual checks review -To see checks to be verified inspect the `result_file` from `appinspect-cli-action` run with manual tag. Verify manual checks and mark them as reviewed by adding them one by one into `.app-vetting.yaml`, ex: +To see checks to be verified, inspect the `result_file` from `appinspect-cli-action`. Verify manual checks and mark them as reviewed by adding them one by one into `.appinspect.manualcheck.yaml`, ex: ```yml name_of_manual_check_1: comment: 'your comment' name_of_manual_check_2: comment: 'your comment' ``` -please note that names of validated manual checks should be aligned with those from `result_file` and your comment can't be empty. +Please note that names of validated manual checks should be aligned with those from `result_file` and your comment can't be empty. + +### Failure checks review +To mark Failures as expected, add them into `.appinspect.expect.yaml` with proper comment containing ticket id of ADDON/APPCERT project associated with the exception, ex: +```yml +name_of_exception_1: + comment: 'ADDON-123: your comment' +name_of_exception_2: + comment: 'APPCERT-123: your comment' +``` +Please note that your comment can't be empty, it must include ticket id of ADDON/APPCERT project associated with the exception and the names of exceptions should be aligned with those from `result_file`. + ### Running the job -When `appinspect-cli-action` is called with `manual` tag, it scans the package with Splunk's AppInspect CLI and searches for manual checks. In the next step, action compares `results_file` with `.app-vetting.yaml` if any check wasn't reviewed and isn't in `.app-vetting.yaml` then the job fails. +When `appinspect-cli-action` is called, it scans the package with Splunk's AppInspect CLI. If there are any failures observed then action compares `results_file` with `.appinspect.expect.yaml`. If that failure isn't present in `.appinspect.expect.yaml` or it does not contain an appropriate comment(containing ADDON/APPCERT ticket id associated with the exception) then the job fails with proper failure reason. In the next step, action compares `results_file` with `.appinspect.manualcheck.yaml`. If any manual check wasn't reviewed by addon developer and isn't in `.appinspect.manualcheck.yaml` then the job fails. ## Example usage @@ -66,20 +89,26 @@ When `appinspect-cli-action` is called with `manual` tag, it scans the package w with: app_path: 'test' ``` -### Downloading manual checks markdown -If the comparison is successful then a markdown consisting a table with manual check names and comments is generated. It can be uploaded to artifacts. +### Downloading markdowns +If the comparison is successful then a markdown consisting a table with check names and comments is generated. It can be uploaded to artifacts. ```yml - uses: actions/checkout@v2 - uses: splunk/appinspect-cli-action@v1.3 with: app_path: 'test' - included_tags: manual + included_tags: {appinspect-tags-to-include} manual_check_markdown: manual_check_markdown.txt + expected_failure_markdown: expected_failure_markdown.txt - name: upload-manual-check-markodown uses: actions/upload-artifact@v2 with: name: manual_check_markdown.txt path: manual_check_markdown.txt +- name: upload-expected_failure-markodown + uses: actions/upload-artifact@v2 + with: + name: expected_failure_markdown.txt + path: expected_failure_markdown.txt ``` The markdown is ready to paste into confluence, by: `Edit -> Insert more content -> Markup`, change insert type to `Markdown` and paste the contents of the file. diff --git a/action.yml b/action.yml index 50069d1..d7849eb 100644 --- a/action.yml +++ b/action.yml @@ -1,9 +1,9 @@ # action.yml name: "Splunk AppInspect" -description: "Run Splunk App insect on a Splunk app directory." +description: "Run Splunk App inspect on a Splunk app directory." inputs: app_path: - description: "path to the application directory to be inspected" + description: "Path to the application directory to be inspected" default: build/splunkbase result_file: description: "json result file name" @@ -14,17 +14,25 @@ inputs: excluded_tags: description: "Tags to exclude" required: false - app_vetting: - description: "Path to app vetting yaml file" + appinspect_manual_checks: + description: "Path to file which contains list of manual checks" required: false - default: ".app-vetting.yaml" + default: ".appinspect.manualcheck.yaml" + appinspect_expected_failures: + description: "Path to file which contains list of expected appinspect failures" + required: false + default: ".appinspect.expect.yaml" manual_check_markdown: - description: "Path for generated file with markdown for manual checks and exceptions" + description: "Path for generated file with markdown for manual checks" required: false default: "manual_check_markdown.txt" + expected_failure_markdown: + description: "Path for generated file with markdown for expected appinspect failures" + required: false + default: "expected_failure_markdown.txt" outputs: status: description: "value is success/fail based on app inspect result" runs: using: "docker" - image: "docker://ghcr.io/splunk/appinspect-cli-action/appinspect-cli-action:v1.5.1" + image: "Dockerfile" diff --git a/compare_checks.py b/compare_checks.py index 3282988..22f23e3 100644 --- a/compare_checks.py +++ b/compare_checks.py @@ -1,5 +1,6 @@ import json import os +import re import sys from typing import List @@ -26,18 +27,27 @@ class BCOLORS: UNDERLINE = "\033[4m" +def validate_comment(vetting_data): + checks = [] + ticket_id = re.compile(r"((?i)(ADDON|APPCERT)-[0-9]+)") + for check, info in vetting_data.items(): + if not re.search(ticket_id, info.get("comment")): + checks.append(check) + return checks + + def compare( check_type: str, - vetting_file: str = ".app-vetting.yaml", + vetting_file: str = ".appinspect.manualcheck.yaml", appinspect_result_file: str = "appinspect_output.json", ) -> List[str]: """ Compares checks from vetting file and appinspect result file. A lot prints are added to make it easier for users to create proper vetting_file and understand errors - :param vetting_file: path to yaml file with verified manual checks + :param vetting_file: path to file with varified list of checks :param appinspect_result_file: path to Splunk's AppInspect CLI result file - :return: list of non matching tests between vetting_file and appinspect_result_file or not commented ones + :return: list of non matching tests between vetting_file and appinspect_result_file or not commented ones or checks with inappropriate comment """ if not os.path.isfile(appinspect_result_file): raise FileNotFoundError( @@ -89,28 +99,38 @@ def compare( print( f"{BCOLORS.FAIL}{BCOLORS.BOLD}Please see appinspect report for more detailed description about {check_type} checks and review them accordingly.{BCOLORS.ENDC}" ) + checks_with_no_id = [] + if check_type == "failure": + checks_with_no_id = validate_comment(vetting_data) + if checks_with_no_id: + print( + f"{BCOLORS.FAIL}{BCOLORS.BOLD}There are some checks which require comment with proper ticket id in {vetting_file}. Below checks are not commented with required ticket id in" + f" {vetting_file}:{BCOLORS.ENDC}" + ) + for check in checks_with_no_id: + print(f"{BCOLORS.FAIL}{BCOLORS.BOLD}\t{check}{BCOLORS.ENDC}") - return new_checks + not_commented + return new_checks + not_commented + checks_with_no_id def get_checks_from_appinspect_result( path: str, result: str = "manual_check" ) -> List[str]: """ - Returns manual checks from appinspect json result file + Returns checks from appinspect json result file :param path: path to json result file :return: list of checks in string format """ - manual_checks = [] + checks = [] with open(path) as f: appinspect_results = json.load(f) for report in appinspect_results["reports"]: for group in report["groups"]: for check in group["checks"]: if check["result"] == result: - manual_checks.append(check["name"]) - return manual_checks + checks.append(check["name"]) + return checks def main(): diff --git a/entrypoint.sh b/entrypoint.sh index b1d41f0..b792dc5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -42,23 +42,25 @@ python3 /reporter.py $INPUT_RESULT_FILE exit_code=$? echo "::endgroup::" +exit_code_failure_check=$exit_code if [ $exit_code != 0 ]; then echo "::group::failure_checks" - python3 /compare_checks.py $INPUT_APP_VETTING $INPUT_RESULT_FILE "failure" - exit_code=$? + python3 /compare_checks.py $INPUT_APPINSPECT_EXPECTED_FAILURES $INPUT_RESULT_FILE "failure" + exit_code_failure_check=$? echo "::endgroup::" fi -if [[ "$INPUT_INCLUDED_TAGS" == *"manual"* ]] && [ $exit_code == 0 ]; then - echo "::group::manual_checks" - python3 /compare_checks.py $INPUT_APP_VETTING $INPUT_RESULT_FILE "manual_check" - exit_code=$? - if [ $exit_code == 0 ]; then - echo "successful comparison, generating markdown" - echo "/export_to_markdown.py $INPUT_APP_VETTING $INPUT_MANUAL_CHECK_MARKDOWN" - python3 /export_to_markdown.py $INPUT_APP_VETTING $INPUT_MANUAL_CHECK_MARKDOWN - fi +echo "::group::manual_checks" +python3 /compare_checks.py $INPUT_APPINSPECT_MANUAL_CHECKS $INPUT_RESULT_FILE "manual_check" +exit_code_manual_check=$? +echo "::endgroup::" + +if [ $exit_code_failure_check == 0 ] && [ $exit_code_manual_check == 0 ] ; then + echo "::group::generate_markdown" + echo "successful comparison, generating markdown" + python3 /export_to_markdown.py $INPUT_APPINSPECT_MANUAL_CHECKS $INPUT_MANUAL_CHECK_MARKDOWN + python3 /export_to_markdown.py $INPUT_APPINSPECT_EXPECTED_FAILURES $INPUT_EXPECTED_FAILURE_MARKDOWN echo "::endgroup::" fi -exit "$exit_code" +exit "$(($exit_code_failure_check || $exit_code_manual_check))" diff --git a/export_to_markdown.py b/export_to_markdown.py index d76c96e..d31feef 100644 --- a/export_to_markdown.py +++ b/export_to_markdown.py @@ -9,13 +9,13 @@ - + """ CHECK_MARKDOWN_TEMPLATE = """ - """ @@ -30,28 +30,28 @@ class ExportToMarkdown: Based on app vetting file generates file with markdown consisting names of validated checks and comments. """ - def __init__(self, manual_checks_path, markdown_output_path): - self.manual_checks_path = manual_checks_path + def __init__(self, checks_path, markdown_output_path): + self.checks_path = checks_path self.markdown_output_path = markdown_output_path - self.manual_checks = None + self.checks = None def __call__(self): - self._load_manual_checks() + self._load_checks() self._create_output_markup() - def _load_manual_checks(self): - with open(self.manual_checks_path) as vetting_data: - self.manual_checks = yaml.safe_load(vetting_data) - if self.manual_checks is None: - self.manual_checks = {} + def _load_checks(self): + with open(self.checks_path) as vetting_data: + self.checks = yaml.safe_load(vetting_data) + if self.checks is None: + self.checks = {} def _create_output_markup(self): with open(self.markdown_output_path, "w") as output: output.write(MARKDOWN_START) - for manual_check, check_attributes in self.manual_checks.items(): + for check, check_attributes in self.checks.items(): output.write( CHECK_MARKDOWN_TEMPLATE.format( - manual_check=manual_check, comment=check_attributes["comment"] + check=check, comment=check_attributes["comment"] ) ) output.write(MARKDOWN_END) @@ -59,7 +59,7 @@ def _create_output_markup(self): def main(): ExportToMarkdown( - manual_checks_path=APP_VETTING_PATH, markdown_output_path=MARKDOWN_OUTPUT_PATH + checks_path=APP_VETTING_PATH, markdown_output_path=MARKDOWN_OUTPUT_PATH )() diff --git a/reporter.py b/reporter.py index 7d1a2ed..64cd015 100644 --- a/reporter.py +++ b/reporter.py @@ -1,7 +1,32 @@ import json import os import sys -from pprint import pprint + +import tabulate + + +class BCOLORS: + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + BOLD = "\033[1m" + + +def format_result(result): + restructured_result = [ + "success", + "manual_check", + "not_applicable", + "skipped", + "warning", + "error", + "failure", + ] + row = [[result[x] for x in restructured_result]] + print(tabulate.tabulate(row, restructured_result)) def main(args): @@ -11,26 +36,32 @@ def main(args): if "summary" in result and "failure" in result["summary"]: failures = result["summary"]["failure"] if failures == 0: - print("App Inspect Passed!") + print(f"{BCOLORS.BOLD}{BCOLORS.OKGREEN}App Inspect Passed!") if "warning" in result["summary"] and result["summary"]["warning"]: - print("Warning List:") + print(f"{BCOLORS.OKBLUE}Warning List:") for group in result["reports"][0]["groups"]: for check in group["checks"]: if check["result"] == "warning": + print(f'{BCOLORS.WARNING} {check["name"]}') for msg in check["messages"]: print(msg["message"]) - pprint(result["summary"]) + print(f"{BCOLORS.OKBLUE}{BCOLORS.BOLD} SUMMARY") + format_result(result["summary"]) with open(os.environ["GITHUB_OUTPUT"], "a") as fh: print("status=pass", file=fh) else: - print(f"App Inspect returned {failures} failures.") + print( + f"{BCOLORS.BOLD}{BCOLORS.FAIL}App Inspect returned {failures} failures." + ) with open(os.environ["GITHUB_OUTPUT"], "a") as fh: print("status=fail", file=fh) - pprint(result["summary"]) - print("Failure List:") + print(f"{BCOLORS.OKBLUE}{BCOLORS.BOLD} SUMMARY") + format_result(result["summary"]) + print(f"{BCOLORS.OKBLUE}{BCOLORS.BOLD} Failure List:") for group in result["reports"][0]["groups"]: for check in group["checks"]: if check["result"] == "failure": + print(f'{BCOLORS.FAIL} {check["name"]}') for msg in check["messages"]: print(msg["message"]) sys.exit(1) diff --git a/requirements.txt b/requirements.txt index 9f03473..9b000d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pyyaml==5.4.1 -splunk-appinspect==2.14.1 \ No newline at end of file +splunk-appinspect==2.30.0 +tabulate==0.9.0 \ No newline at end of file
manual checkcheck comment
{manual_check} +{check} {comment}