diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 695b8e38d..328e2ff49 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,9 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Check environment variables + run: | + /bin/bash scripts/check_vars.sh - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -33,6 +36,9 @@ jobs: python -m pip install --upgrade pip poetry importlib-resources==1.5.0 poetry export --with dev -f requirements.txt --output requirements.txt pip install -r requirements.txt + - name: Validate Grafana dashboard + run: | + python3 scripts/validate_dashboards.py ./monitoring/grafana/dashboards/ - name: Run tests run: | /bin/bash run_tests.sh diff --git a/run_tests.sh b/run_tests.sh index b1c613dcf..c8599a749 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -14,48 +14,9 @@ echo -e "${CYAN}------------------------------${NO_COLOR}" find . -name '*.py' | xargs dirname | sort | uniq | xargs -I {} bash -c "test -f {}/__init__.py || ( echo {} directory does not have __init__.py! && false )" rc=$(($rc+$?)) -# Simple test to compare variables used in code and specified in conf/*.env files, allows to set ignored variables -not_configurable="CW_AWS_ACCESS_KEY_ID\|CW_AWS_SECRET_ACCESS_KEY\|SLACK_WEBHOOK\|VULNERABILITY_ENV"\ -"\|MINIMAL_SCHEMA\|HOSTNAME\|DB_UPGRADE_SCRIPTS_DIR\|VE_DB_USER_ADVISOR_LISTENER_PASSWORD"\ -"\|VE_DB_USER_EVALUATOR_PASSWORD\|VE_DB_USER_LISTENER_PASSWORD\|VE_DB_USER_MANAGER_PASSWORD"\ -"\|VE_DB_USER_TASKOMATIC_PASSWORD\|VE_DB_USER_VMAAS_SYNC_PASSWORD\|VE_DB_USER_NOTIFICATOR_PASSWORD" -not_in_code="API_URLS\|CYNDI_MOCK\|PGUSER" -configurable_variables=$(cat conf/*.env | grep -o "^.*=" | sed 's/.$//g' | sort -u | grep -v "$not_in_code") -code_variables=$(find . -name '*.py' -not -path './.venv/*' -not -path './scripts/*' -not -path './tests/*' -not -path './database/upgrade/*' -exec grep -oP "os\.getenv.*?\)|os\.environ\.get.*?\)" {} \; | awk -F"['\"]" '{print $2}' | sort -u | grep -v "$not_configurable") -diff <(echo "$configurable_variables") <(echo "$code_variables") -diff_rc=$? -if [ $diff_rc -gt 0 ]; then - echo "Some variables in code or conf/*.env are missing!" -else - echo "Variables in code and conf/*.env are OK" -fi -rc=$(($rc+$diff_rc)) - -echo "" - -# Are there duplicated configuration variables? -duplicates=$(cat conf/*.env | grep -o "^.*=.*" | sort | uniq -d) -duplicate_cnt=0 -for dup in $duplicates -do - echo "$dup" - duplicate_cnt=$(($duplicate_cnt+1)) -done -if [ $duplicate_cnt -gt 0 ]; then - echo "Duplicated variables were found!" - rc=$(($rc+1)) -else - echo "Variables are unique." -fi - # Run unit tests echo "Running unit tests:" pytest -vvv -s --cov-report term --cov --color=yes --durations=1 rc=$(($rc+$?)) -# Validate grafana dashboards -echo "Validating Grafana dashboards:" -./scripts/validate_dashboards.py ./monitoring/grafana/dashboards/ -rc=$(($rc+$?)) - exit $rc diff --git a/scripts/check_vars.sh b/scripts/check_vars.sh new file mode 100644 index 000000000..02fe3c35d --- /dev/null +++ b/scripts/check_vars.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +rc=0 +RED='\033[0;31m' +NC='\033[0m' # No Color + +# Are there duplicated configuration variables? +duplicates=$(cat conf/*.env | grep -o "^.*=.*" | sort | uniq -d) +duplicate_cnt=0 +for dup in $duplicates +do + echo "$dup" + duplicate_cnt=$(($duplicate_cnt+1)) +done +if [ $duplicate_cnt -gt 0 ]; then + echo -e "${RED} Error: Duplicated variables were found!${NC}" + rc=$(($rc+1)) +else + echo "Variables are unique." +fi + +# Simple test to compare variables used in code and specified in conf/*.env files, allows to set ignored variables +not_configurable="CW_AWS_ACCESS_KEY_ID\|CW_AWS_SECRET_ACCESS_KEY\|SLACK_WEBHOOK\|VULNERABILITY_ENV"\ +"\|MINIMAL_SCHEMA\|HOSTNAME\|DB_UPGRADE_SCRIPTS_DIR\|VE_DB_USER_ADVISOR_LISTENER_PASSWORD"\ +"\|VE_DB_USER_EVALUATOR_PASSWORD\|VE_DB_USER_LISTENER_PASSWORD\|VE_DB_USER_MANAGER_PASSWORD"\ +"\|VE_DB_USER_TASKOMATIC_PASSWORD\|VE_DB_USER_VMAAS_SYNC_PASSWORD\|VE_DB_USER_NOTIFICATOR_PASSWORD" +not_in_code="API_URLS\|CYNDI_MOCK\|PGUSER" +configurable_variables=$(cat conf/*.env | grep -o "^.*=" | sed 's/.$//g' | sort -u | grep -v "$not_in_code") +code_variables=$(find . -name '*.py' -not -path './.venv/*' -not -path './scripts/*' -not -path './tests/*' -not -path './database/upgrade/*' -exec grep -oP "os\.getenv.*?\)|os\.environ\.get.*?\)" {} \; | awk -F"['\"]" '{print $2}' | sort -u | grep -v "$not_configurable") +diff <(echo "$configurable_variables") <(echo "$code_variables") +diff_rc=$? +if [ $diff_rc -gt 0 ]; then + echo -e "${RED} Error: Some variables in code or conf/*.env are missing!${NC}" +else + echo "Variables in code and conf/*.env are OK" +fi +rc=$(($rc+$diff_rc)) + +exit $rc