diff --git a/commodore/cli.py b/commodore/cli.py index 4c297fd3..884eaf95 100644 --- a/commodore/cli.py +++ b/commodore/cli.py @@ -247,11 +247,32 @@ def component(config: Config, verbose): show_default=True, help="The copyright holder added to the license file.", ) +@click.option( + "--golden-tests/--no-golden-tests", + default=True, + show_default=True, + help="Add golden tests to the component.", +) +@click.option( + "--matrix-tests/--no-matrix-tests", + default=True, + show_default=True, + help="Enable test matrix for compile/golden tests.", +) @verbosity @pass_config # pylint: disable=too-many-arguments def component_new( - config: Config, slug, name, lib, pp, owner, copyright_holder, verbose + config: Config, + slug, + name, + lib, + pp, + owner, + copyright_holder, + golden_tests, + matrix_tests, + verbose, ): config.update_verbosity(verbose) f = ComponentTemplater(config, slug) @@ -260,6 +281,8 @@ def component_new( f.post_process = pp f.github_owner = owner f.copyright_holder = copyright_holder + f.golden_tests = golden_tests + f.matrix_tests = matrix_tests f.create() diff --git a/commodore/component-template/cookiecutter.json b/commodore/component-template/cookiecutter.json index 6ffc9969..6b5c08d3 100644 --- a/commodore/component-template/cookiecutter.json +++ b/commodore/component-template/cookiecutter.json @@ -5,6 +5,8 @@ "add_lib": "n", "add_pp": "n", + "add_golden": "y", + "add_matrix": "y", "copyright_holder": "VSHN AG ", "copyright_year": "1950", diff --git a/commodore/component-template/hooks/post_gen_project.py b/commodore/component-template/hooks/post_gen_project.py index f999573d..c42b04fd 100644 --- a/commodore/component-template/hooks/post_gen_project.py +++ b/commodore/component-template/hooks/post_gen_project.py @@ -1,6 +1,10 @@ import shutil create_lib = "{{ cookiecutter.add_lib }}" == "y" +add_golden = "{{ cookiecutter.add_golden }}" == "y" if not create_lib: shutil.rmtree("lib") + +if not add_golden: + shutil.rmtree("tests/golden") diff --git a/commodore/component-template/{{ cookiecutter.slug }}/.editorconfig b/commodore/component-template/{{ cookiecutter.slug }}/.editorconfig index e0284e66..06615f3f 100644 --- a/commodore/component-template/{{ cookiecutter.slug }}/.editorconfig +++ b/commodore/component-template/{{ cookiecutter.slug }}/.editorconfig @@ -24,3 +24,7 @@ insert_final_newline = false [Makefile] indent_style = tab + +# Don't check for trailing newlines in golden tests output +[tests/golden/**] +insert_final_newline = unset diff --git a/commodore/component-template/{{ cookiecutter.slug }}/.github/workflows/test.yaml b/commodore/component-template/{{ cookiecutter.slug }}/.github/workflows/test.yaml index 3f33e47d..d8fc21f2 100644 --- a/commodore/component-template/{{ cookiecutter.slug }}/.github/workflows/test.yaml +++ b/commodore/component-template/{{ cookiecutter.slug }}/.github/workflows/test.yaml @@ -4,6 +4,9 @@ on: branches: - master +env: + COMPONENT_NAME: {{ cookiecutter.slug }} + jobs: linting: runs-on: ubuntu-latest @@ -24,3 +27,47 @@ jobs: - uses: snow-actions/eclint@v1.0.1 with: args: 'check' + test: + runs-on: ubuntu-latest +{%- if cookiecutter.add_matrix == "y" %} + strategy: + matrix: + instance: + - defaults +{%- endif %} + defaults: + run: + working-directory: {% raw %}${{ env.COMPONENT_NAME }}{% endraw %} + steps: + - uses: actions/checkout@v2 + with: + path: {% raw %}${{ env.COMPONENT_NAME }}{% endraw %} + - name: Compile component +{%- if cookiecutter.add_matrix == "y" %} + run: make test -e instance={% raw %}${{ matrix.instance }}{% endraw %} +{%- else %} + run: make test +{%- endif %} +{%- if cookiecutter.add_golden == "y" %} + golden: + runs-on: ubuntu-latest +{%- if cookiecutter.add_matrix == "y" %} + strategy: + matrix: + instance: + - defaults +{%- endif %} + defaults: + run: + working-directory: {% raw %}${{ env.COMPONENT_NAME }}{% endraw %} + steps: + - uses: actions/checkout@v2 + with: + path: {% raw %}${{ env.COMPONENT_NAME }}{% endraw %} + - name: Golden diff +{%- if cookiecutter.add_matrix == "y" %} + run: make golden-diff -e instance={% raw %}${{ matrix.instance }}{% endraw %} +{%- else %} + run: make golden-diff +{%- endif %} +{%- endif %} diff --git a/commodore/component-template/{{ cookiecutter.slug }}/.gitignore b/commodore/component-template/{{ cookiecutter.slug }}/.gitignore index 8d163768..be80b429 100644 --- a/commodore/component-template/{{ cookiecutter.slug }}/.gitignore +++ b/commodore/component-template/{{ cookiecutter.slug }}/.gitignore @@ -1,7 +1,14 @@ -_archive/ -_public/ +# Commodore .cache/ helmcharts/ manifests/ vendor/ jsonnetfile.lock.json +crds/ +compiled/ + +# Antora +_archive/ +_public/ + +# Additional entries diff --git a/commodore/component-template/{{ cookiecutter.slug }}/.sync.yml b/commodore/component-template/{{ cookiecutter.slug }}/.sync.yml index 5e7810fd..bbf7616a 100644 --- a/commodore/component-template/{{ cookiecutter.slug }}/.sync.yml +++ b/commodore/component-template/{{ cookiecutter.slug }}/.sync.yml @@ -1,7 +1,20 @@ :global: componentName: {{ cookiecutter.name }} githubUrl: {{ cookiecutter.github_url }} + feature_goldenTests: true docs/antora.yml: name: {{ cookiecutter.slug }} title: {{ cookiecutter.name }} +{% if cookiecutter.add_matrix == "y" -%} + +.github/workflows/test.yaml: + test_makeTarget: test -e instance={% raw %}${{ matrix.instance }}{% endraw %} +{%- if cookiecutter.add_golden == "y" %} + goldenTest_makeTarget: golden-diff -e instance={% raw %}${{ matrix.instance }}{% endraw %} +{%- endif %} + matrix: + key: instance + entries: + - defaults +{% endif -%} diff --git a/commodore/component-template/{{ cookiecutter.slug }}/Makefile b/commodore/component-template/{{ cookiecutter.slug }}/Makefile index ed4b557c..e193c8a0 100644 --- a/commodore/component-template/{{ cookiecutter.slug }}/Makefile +++ b/commodore/component-template/{{ cookiecutter.slug }}/Makefile @@ -49,6 +49,20 @@ docs-serve: ## Preview the documentation test: commodore_args += -f tests/$(instance).yml test: .compile ## Compile the component +{%- if cookiecutter.add_golden == "y" %} +.PHONY: gen-golden +gen-golden: commodore_args += -f tests/$(instance).yml +gen-golden: .compile ## Update the reference version for target `golden-diff`. + @rm -rf tests/golden/$(instance) + @mkdir -p tests/golden/$(instance) + @cp -R compiled/. tests/golden/$(instance)/. + +.PHONY: golden-diff +golden-diff: commodore_args += -f tests/$(instance).yml +golden-diff: .compile ## Diff compile output against the reference version. Review output and run `make gen-golden golden-diff` if this target fails. + @git diff --exit-code --minimal --no-index -- tests/golden/$(instance) compiled/ +{%- endif %} + .PHONY: clean clean: ## Clean the project rm -rf compiled dependencies vendor helmcharts jsonnetfile*.json || true diff --git a/commodore/component-template/{{ cookiecutter.slug }}/tests/golden/defaults/{{ cookiecutter.slug }}/apps/{{ cookiecutter.slug }}.yaml b/commodore/component-template/{{ cookiecutter.slug }}/tests/golden/defaults/{{ cookiecutter.slug }}/apps/{{ cookiecutter.slug }}.yaml new file mode 100644 index 00000000..e69de29b diff --git a/commodore/component/template.py b/commodore/component/template.py index 8c3c93ce..c7853d6f 100644 --- a/commodore/component/template.py +++ b/commodore/component/template.py @@ -23,6 +23,8 @@ class ComponentTemplater: github_owner: str copyright_holder: str today: datetime.date + golden_tests: bool + matrix_tests: bool def __init__(self, config, slug): self.config = config @@ -59,6 +61,8 @@ def cookiecutter_args(self): return { "add_lib": "y" if self.library else "n", "add_pp": "y" if self.post_process else "n", + "add_golden": "y" if self.golden_tests else "n", + "add_matrix": "y" if self.matrix_tests else "n", "copyright_holder": self.copyright_holder, "copyright_year": self.today.strftime("%Y"), "github_owner": self.github_owner, diff --git a/docs/modules/ROOT/pages/reference/cli.adoc b/docs/modules/ROOT/pages/reference/cli.adoc index 90572a11..1adeacf3 100644 --- a/docs/modules/ROOT/pages/reference/cli.adoc +++ b/docs/modules/ROOT/pages/reference/cli.adoc @@ -144,5 +144,11 @@ This command doesn't have any command line options. *--copyright* TEXT:: The copyright holder added to the license file. Defaults to "VSHN AG ." +*--golden-tests / --no-golden-tests*:: + Enable golden tests for the component. Defaults to _yes_. + +*--matrix-tests / --no-matrix-tests*:: + Enable test matrix for the component compile and golden tests. Defaults to _yes_. + *--help*:: Show component new usage and options then exit. diff --git a/tests/test_component_template.py b/tests/test_component_template.py index e968f0d5..ac7c83db 100644 --- a/tests/test_component_template.py +++ b/tests/test_component_template.py @@ -11,7 +11,22 @@ from test_component import setup_directory -def test_run_component_new_command(tmp_path: P): +@pytest.mark.parametrize("lib", ["--no-lib", "--lib"]) +@pytest.mark.parametrize( + "pp", + ["--no-pp", "--pp"], +) +@pytest.mark.parametrize( + "golden", + ["--no-golden-tests", "--golden-tests"], +) +@pytest.mark.parametrize( + "matrix", + ["--no-matrix-tests", "--matrix-tests"], +) +def test_run_component_new_command( + tmp_path: P, lib: str, pp: str, golden: str, matrix: str +): """ Run the component new command """ @@ -20,17 +35,17 @@ def test_run_component_new_command(tmp_path: P): component_name = "test-component" exit_status = call( - f"commodore -d '{tmp_path}' -vvv component new {component_name} --lib --pp", + f"commodore -d '{tmp_path}' -vvv component new {component_name} {lib} {pp} {golden} {matrix}", shell=True, ) assert exit_status == 0 - for file in [ + + expected_files = [ P("README.md"), P("renovate.json"), P("class", f"{component_name}.yml"), P("component", "main.jsonnet"), P("component", "app.jsonnet"), - P("lib", f"{component_name}.libsonnet"), P("docs", "modules", "ROOT", "pages", "references", "parameters.adoc"), P("docs", "modules", "ROOT", "pages", "index.adoc"), P(".github", "changelog-configuration.json"), @@ -40,7 +55,23 @@ def test_run_component_new_command(tmp_path: P): P(".github", "ISSUE_TEMPLATE", "01_bug_report.md"), P(".github", "ISSUE_TEMPLATE", "02_feature_request.md"), P(".github", "ISSUE_TEMPLATE", "config.yml"), - ]: + P(".sync.yml"), + P("tests", "defaults.yml"), + ] + if lib == "--lib": + expected_files.append(P("lib", f"{component_name}.libsonnet")) + if golden == "--golden": + expected_files.append( + P( + "tests", + "golden", + "defaults", + component_name, + "apps", + f"{component_name}.yaml", + ) + ) + for file in expected_files: assert (tmp_path / "dependencies" / component_name / file).exists() # Check that there are no uncommited files in the component repo repo = Repo(tmp_path / "dependencies" / component_name) @@ -54,10 +85,42 @@ def test_run_component_new_command(tmp_path: P): assert "parameters" in class_contents params = class_contents["parameters"] assert "kapitan" in params - assert "commodore" in params - assert "postprocess" in params["commodore"] - assert "filters" in params["commodore"]["postprocess"] - assert isinstance(params["commodore"]["postprocess"]["filters"], list) + if pp == "--pp": + assert "commodore" in params + assert "postprocess" in params["commodore"] + assert "filters" in params["commodore"]["postprocess"] + assert isinstance(params["commodore"]["postprocess"]["filters"], list) + + has_golden = golden == "--golden-tests" + has_matrix = matrix == "--matrix-tests" + with open( + tmp_path + / "dependencies" + / component_name + / ".github" + / "workflows" + / "test.yaml" + ) as ght: + ghtest = yaml.safe_load(ght) + assert ghtest["env"] == {"COMPONENT_NAME": component_name} + assert ("strategy" in ghtest["jobs"]["test"]) == has_matrix + run_step = ghtest["jobs"]["test"]["steps"][1] + if has_matrix: + assert run_step["run"] == "make test -e instance=${{ matrix.instance }}" + else: + assert run_step["run"] == "make test" + + if has_golden: + assert "golden" in ghtest["jobs"] + assert ("strategy" in ghtest["jobs"]["golden"]) == has_matrix + run_step = ghtest["jobs"]["golden"]["steps"][1] + if has_matrix: + assert ( + run_step["run"] + == "make golden-diff -e instance=${{ matrix.instance }}" + ) + else: + assert run_step["run"] == "make golden-diff" def test_run_component_new_command_with_name(tmp_path: P): @@ -156,16 +219,22 @@ def test_deleting_inexistant_component(tmp_path: P): assert exit_status == 2 +@pytest.mark.parametrize("lib", ["--no-lib", "--lib"]) @pytest.mark.parametrize( - "extra_args", - [ - "", - "--lib", - "--pp", - "--lib --pp", - ], + "pp", + ["--no-pp", "--pp"], ) -def test_check_component_template(tmp_path: P, extra_args: str): +@pytest.mark.parametrize( + "golden", + ["--no-golden-tests", "--golden-tests"], +) +@pytest.mark.parametrize( + "matrix", + ["--no-matrix-tests", "--matrix-tests"], +) +def test_check_component_template( + tmp_path: P, lib: str, pp: str, golden: str, matrix: str +): """ Run integrated lints in freshly created component """ @@ -174,7 +243,7 @@ def test_check_component_template(tmp_path: P, extra_args: str): component_name = "test-component" exit_status = call( - f"commodore -d {tmp_path} -vvv component new {component_name} {extra_args}", + f"commodore -d {tmp_path} -vvv component new {component_name} {lib} {pp} {golden} {matrix}", shell=True, ) assert exit_status == 0 @@ -186,3 +255,25 @@ def test_check_component_template(tmp_path: P, extra_args: str): cwd=tmp_path / "dependencies" / component_name, ) assert exit_status == 0 + + +def test_check_golden_diff(tmp_path: P): + """ + Verify that `make golden-diff` passes for a component which has golden tests enabled + """ + setup_directory(tmp_path) + + component_name = "test-component" + exit_status = call( + f"commodore -d {tmp_path} -vvv component new {component_name}", + shell=True, + ) + assert exit_status == 0 + + # Call `make lint` in component directory + exit_status = call( + "make golden-diff", + shell=True, + cwd=tmp_path / "dependencies" / component_name, + ) + assert exit_status == 0