From 5d1f23257ba7f11508a90c883b152349bcc2d7fd Mon Sep 17 00:00:00 2001 From: Megan Wolf <97549300+meganwolf0@users.noreply.github.com> Date: Wed, 25 Sep 2024 18:30:24 -0400 Subject: [PATCH] feat(template)!: introducing variables and sensitive configuration (#672) * feat(template): sample sensitive config * feat(template): updated funcs, tests * fix: go mod tidy * fix: small refactor on template render * docs: updated config/template docs * fix: bump timeout in tui tests * fix: small refactor for extending tmpl * fix: bumped timeout * fix: updated docs, slightly modified templaterenderer struct --- docs/cli-commands/lula_tools_template.md | 13 +- docs/getting-started/README.md | 6 +- docs/getting-started/configuration.md | 139 ++++ docs/reference/configuration.md | 40 -- go.mod | 3 - go.sum | 8 - src/cmd/common/common.go | 26 + src/cmd/common/viper.go | 30 +- src/cmd/tools/template.go | 151 +++-- src/internal/template/helpers.go | 14 + src/internal/template/template.go | 352 +++++++++- src/internal/template/template_test.go | 612 +++++++++++++++++- src/internal/tui/model_test.go | 2 +- src/test/e2e/standard/lula-config.yaml | 26 +- src/test/e2e/standard/template_test.go | 103 ++- src/test/e2e/standard/testdata/empty.golden | 0 src/test/e2e/standard/testdata/help.golden | 32 + .../e2e/standard/testdata/validation.golden | 30 + .../standard/testdata/validation_all.golden | 30 + .../testdata/validation_constants.golden | 30 + .../testdata/validation_non_sensitive.golden | 30 + .../testdata/validation_with_env_vars.golden | 30 + .../testdata/validation_with_set.golden | 30 + .../validation/validation.bad.tmpl.yaml | 30 + .../common/validation/validation.tmpl.yaml | 30 + src/test/util/utils.go | 9 +- 26 files changed, 1628 insertions(+), 178 deletions(-) create mode 100644 docs/getting-started/configuration.md delete mode 100644 docs/reference/configuration.md create mode 100644 src/cmd/common/common.go create mode 100644 src/internal/template/helpers.go create mode 100644 src/test/e2e/standard/testdata/empty.golden create mode 100644 src/test/e2e/standard/testdata/help.golden create mode 100644 src/test/e2e/standard/testdata/validation.golden create mode 100644 src/test/e2e/standard/testdata/validation_all.golden create mode 100644 src/test/e2e/standard/testdata/validation_constants.golden create mode 100644 src/test/e2e/standard/testdata/validation_non_sensitive.golden create mode 100644 src/test/e2e/standard/testdata/validation_with_env_vars.golden create mode 100644 src/test/e2e/standard/testdata/validation_with_set.golden create mode 100644 src/test/unit/common/validation/validation.bad.tmpl.yaml create mode 100644 src/test/unit/common/validation/validation.tmpl.yaml diff --git a/docs/cli-commands/lula_tools_template.md b/docs/cli-commands/lula_tools_template.md index d7a68747..70e43ec4 100644 --- a/docs/cli-commands/lula_tools_template.md +++ b/docs/cli-commands/lula_tools_template.md @@ -19,13 +19,20 @@ lula tools template [flags] ``` -To template an OSCAL Model: +To template an OSCAL Model, defaults to masking sensitive variables: lula tools template -f ./oscal-component.yaml To indicate a specific output file: lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml -Data for the templating should be stored under the 'variables' configuration item in a lula-config.yaml file +To perform overrides on the template data: + lula tools template -f ./oscal-component.yaml --set .var.key1=value1 --set .const.key2=value2 + +To perform the full template operation, including sensitive data: + lula tools template -f ./oscal-component.yaml --render all + +Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file +See documentation for more detail on configuration schema ``` @@ -35,6 +42,8 @@ Data for the templating should be stored under the 'variables' configuration ite -h, --help help for template -f, --input-file string the path to the target artifact -o, --output-file string the path to the output file. If not specified, the output file will be directed to stdout + -r, --render string values to render the template with, options are: masked, constants, non-sensitive, all (default "masked") + -s, --set strings set a value in the template data ``` ### Options inherited from parent commands diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md index 009bb49a..c4658cc5 100644 --- a/docs/getting-started/README.md +++ b/docs/getting-started/README.md @@ -63,4 +63,8 @@ See the following tutorials for some introductory lessons on how to use Lula. If Lula Validation manifests are the underlying mechanisms that dictates the evaluation of a system against a control as resulting in `satisfied` or `not satisfied`. A Lula Validation is linked to a control within a component definition via the OSCAL-specific property, [links](../oscal/oscal-validation-links.md). -Developing Lula Validations can sometimes be more art than science, but generally they should aim to be clear, concise, and robust to system changes. See our guide for [developing Lula Validations](./develop-a-validation.md) and the [references](../reference/README.md) for additional information. \ No newline at end of file +Developing Lula Validations can sometimes be more art than science, but generally they should aim to be clear, concise, and robust to system changes. See our guide for [developing Lula Validations](./develop-a-validation.md) and the [references](../reference/README.md) for additional information. + +### Configuration + +Lula supports the addition of a configuration file for specifying CLI flags and templating values. See our [configuration](./configuration.md) guide for more information. \ No newline at end of file diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md new file mode 100644 index 00000000..3df15a2d --- /dev/null +++ b/docs/getting-started/configuration.md @@ -0,0 +1,139 @@ +# Configuration + +Lula allows the use and specification of a config file in the following ways: +- Checking current working directory for a `lula-config.yaml` file +- Specification with environment variable `LULA_CONFIG=` + +Environment Variables can be used to specify configuration values through use of `LULA_` -> Example: `LULA_TARGET=il5` + +## Identification + +If identified, Lula will log which configuration file is used to stdout: +```bash +Using config file /home/dev/work/lula/lula-config.yaml +``` + +## Precedence + +The precedence for configuring settings, such as `target`, follows this hierarchy: + +### **Command Line Flag > Environment Variable > Configuration File** + +1. **Command Line Flag:** + When a setting like `target` is specified using a command line flag, this value takes the highest precedence, overriding any environment variable or configuration file settings. + +2. **Environment Variable:** + If the setting is not provided via a command line flag, an environment variable (e.g., `export LULA_TARGET=il5`) will take precedence over the configuration file. + +3. **Configuration File:** + In the absence of both a command line flag and environment variable, the value specified in the configuration file will be used. This will override system defaults. + +## Support + +Modification of command variables can be set in the configuration file: + +lula-config.yaml +```yaml +log_level: debug +target: il4 +summary: true +``` + +### Templating Configuration Fields + +Templating values are set in the configuration file via the use of `constants` and `variables` fields. + +#### Constants + +A sample `constants` section of a `lula-config.yaml` file is as follows: + +```yaml +constants: + type: software + title: lula + + resources: + name: test-pod-label + namespace: validation-test + imagelist: + - nginx + - nginx2 +``` + +Constants will respect the structure of a map[string]interface{} and can be referenced as follows: + +```yaml +# validaiton.yaml +metadata: + name: sample {{ .const.type }} validation for {{ .const.title }} +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: myPod + resource-rule: + name: {{ .const.resources.name }} + version: v1 + resource: pods + namespaces: [{{ .const.resources.namespace }}] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + validate if { + input.myPod.metadata.name == {{ .const.resources.name }} + input.myPod.containers[_].name in { {{ .const.resources.imagelist | concatToRegoList }} } + } +``` + +And will be rendered as: +```yaml +metadata: + name: sample software validation for lula +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: myPod + resource-rule: + name: myPod + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + validate if { + input.myPod.metadata.name == "myPod" + input.myPod.containers[_].image in { "nginx", "nginx2" } + } +``` + +The constant's keys should be in the format `.const.` and should not contain any '-' or '.' characters, as this will not respect the go text/template format. + +#### Variables + +A sample `variables` section of a `lula-config.yaml` file is as follows: + +```yaml +variables: + - key: some_lula_secret + sensitive: true + - key: some_env_var + default: this-should-be-overridden +``` + +The `variables` section is a list of `key`, `default`, and `sensitive` fields, where `sensitive` and `default` are optional. The `key` and `default` fields are strings, and the `sensitive` field is a boolean. + +A default value can be specified in the case where an environment variable may or may not be set, however an environment variable will always take precedence over a default value. + +The environment variable should follow the pattern of `LULA_VAR_` (not case sensitive), where `` is the key specified in the `variables` section. + +When using `sensitive` variables, the default behavior is to mask the value in the output of the template. \ No newline at end of file diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md deleted file mode 100644 index ee0fb2f0..00000000 --- a/docs/reference/configuration.md +++ /dev/null @@ -1,40 +0,0 @@ -# Configuration - -Lula allows the use and specification of a config file in the following ways: -- Checking current working directory for a `lula-config.yaml` file -- Specification with environment variable `LULA_CONFIG=` - -Environment Variables can be used to specify configuration values through use of `LULA_` -> Example: `LULA_TARGET=il5` - -## Identification - -If identified, Lula will log which configuration file is used to stdout: -```bash -Using config file /home/dev/work/lula/lula-config.yaml -``` - -## Precedence - -The precedence for configuring settings, such as `target`, follows this hierarchy: - -### **Command Line Flag > Environment Variable > Configuration File** - -1. **Command Line Flag:** - When a setting like `target` is specified using a command line flag, this value takes the highest precedence, overriding any environment variable or configuration file settings. - -2. **Environment Variable:** - If the setting is not provided via a command line flag, an environment variable (e.g., `export LULA_TARGET=il5`) will take precedence over the configuration file. - -3. **Configuration File:** - In the absence of both a command line flag and environment variable, the value specified in the configuration file will be used. This will override system defaults. - -## Support - -Modification of command variables can be set in the configuration file: - -lula-config.yaml -```yaml -log_level: debug -target: il4 -summary: true -``` \ No newline at end of file diff --git a/go.mod b/go.mod index 00ae59b6..f66d82ce 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/charmbracelet/x/exp/teatest v0.0.0-20240919170804-a4978c8e603a github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/defenseunicorns/go-oscal v0.6.0 - github.com/defenseunicorns/pkg/helpers v1.1.3 github.com/hashicorp/go-version v1.7.0 github.com/kyverno/kyverno-json v0.0.3 github.com/mattn/go-runewidth v0.0.16 @@ -114,7 +113,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/otiai10/copy v1.14.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect @@ -171,7 +169,6 @@ require ( k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/metrics v0.31.1 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - oras.land/oras-go/v2 v2.5.0 // indirect sigs.k8s.io/controller-runtime v0.18.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.17.2 // indirect diff --git a/go.sum b/go.sum index cd7f98cf..c08549bd 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= github.com/defenseunicorns/go-oscal v0.6.0 h1:eflEKfk7edu4L4kWf6aNQpS94ljfGP8lgWpsPYNtE1Q= github.com/defenseunicorns/go-oscal v0.6.0/go.mod h1:UHp2yK9ty2mYJDun7oNhbstCq6SAAwP4YGbw9n7uG6o= -github.com/defenseunicorns/pkg/helpers v1.1.3 h1:EVVuniq02qfAouR//AT0eoCngLWfFORj8H6+pI8M7uo= -github.com/defenseunicorns/pkg/helpers v1.1.3/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= @@ -294,10 +292,6 @@ github.com/open-policy-agent/opa v0.68.0 h1:Jl3U2vXRjwk7JrHmS19U3HZO5qxQRinQbJ2e github.com/open-policy-agent/opa v0.68.0/go.mod h1:5E5SvaPwTpwt2WM177I9Z3eT7qUpmOGjk1ZdHs+TZ4w= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= -github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= -github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= -github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -559,8 +553,6 @@ k8s.io/metrics v0.31.1 h1:h4I4dakgh/zKflWYAOQhwf0EXaqy8LxAIyE/GBvxqRc= k8s.io/metrics v0.31.1/go.mod h1:JuH1S9tJiH9q1VCY0yzSCawi7kzNLsDzlWDJN4xR+iA= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= -oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= sigs.k8s.io/controller-runtime v0.18.2 h1:RqVW6Kpeaji67CY5nPEfRz6ZfFMk0lWQlNrLqlNpx+Q= sigs.k8s.io/controller-runtime v0.18.2/go.mod h1:tuAt1+wbVsXIT8lPtk5RURxqAnq7xkpv2Mhttslg7Hw= sigs.k8s.io/e2e-framework v0.4.0 h1:4yYmFDNNoTnazqmZJXQ6dlQF1vrnDbutmxlyvBpC5rY= diff --git a/src/cmd/common/common.go b/src/cmd/common/common.go new file mode 100644 index 00000000..3d66ceca --- /dev/null +++ b/src/cmd/common/common.go @@ -0,0 +1,26 @@ +package common + +import ( + "fmt" + "strings" + + "github.com/defenseunicorns/lula/src/internal/template" +) + +func ParseTemplateOverrides(setFlags []string) (map[string]string, error) { + overrides := make(map[string]string) + for _, flag := range setFlags { + parts := strings.SplitN(flag, "=", 2) + if len(parts) != 2 { + return overrides, fmt.Errorf("invalid --set flag format, should be .root.key=value") + } + + if !strings.HasPrefix(parts[0], "."+template.CONST+".") && !strings.HasPrefix(parts[0], "."+template.VAR+".") { + return overrides, fmt.Errorf("invalid --set flag format, path should start with .const or .var") + } + + path, value := parts[0], parts[1] + overrides[path] = value + } + return overrides, nil +} diff --git a/src/cmd/common/viper.go b/src/cmd/common/viper.go index 7ebe0034..8763620b 100644 --- a/src/cmd/common/viper.go +++ b/src/cmd/common/viper.go @@ -2,17 +2,21 @@ package common import ( "errors" + "fmt" "os" "strings" + "github.com/defenseunicorns/lula/src/internal/template" "github.com/defenseunicorns/lula/src/pkg/message" "github.com/spf13/viper" ) const ( - VLogLevel = "log_level" - VTarget = "target" - VSummary = "summary" + VLogLevel = "log_level" + VTarget = "target" + VSummary = "summary" + VConstants = "constants" + VVariables = "variables" ) var ( @@ -70,6 +74,24 @@ func GetViper() *viper.Viper { return v } +// GetTemplateConfig loads the constants and variables from the viper config +func GetTemplateConfig() (map[string]interface{}, []template.VariableConfig, error) { + var constants map[string]interface{} + var variables []template.VariableConfig + + err := v.UnmarshalKey(VConstants, &constants) + if err != nil { + return nil, nil, fmt.Errorf("unable to unmarshal constants into map: %v", err) + } + + err = v.UnmarshalKey(VVariables, &variables) + if err != nil { + return nil, nil, fmt.Errorf("unable to unmarshal variables into slice: %v", err) + } + + return constants, variables, nil +} + func isVersionCmd() bool { args := os.Args return len(args) > 1 && (args[1] == "version" || args[1] == "v") @@ -78,6 +100,8 @@ func isVersionCmd() bool { func setDefaults() { v.SetDefault(VLogLevel, "info") v.SetDefault(VSummary, false) + v.SetDefault(VConstants, make(map[string]interface{})) + v.SetDefault(VVariables, make([]interface{}, 0)) } func printViperConfigUsed() { diff --git a/src/cmd/tools/template.go b/src/cmd/tools/template.go index e9b6f5ae..4b1520f1 100644 --- a/src/cmd/tools/template.go +++ b/src/cmd/tools/template.go @@ -1,7 +1,9 @@ package tools import ( + "fmt" "os" + "strings" "github.com/defenseunicorns/go-oscal/src/pkg/files" "github.com/defenseunicorns/lula/src/cmd/common" @@ -11,79 +13,118 @@ import ( "github.com/spf13/cobra" ) -type templateFlags struct { - InputFile string // -f --input-file - OutputFile string // -o --output-file -} - -var templateOpts = &templateFlags{} - var templateHelp = ` -To template an OSCAL Model: +To template an OSCAL Model, defaults to masking sensitive variables: lula tools template -f ./oscal-component.yaml To indicate a specific output file: lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml -Data for the templating should be stored under the 'variables' configuration item in a lula-config.yaml file +To perform overrides on the template data: + lula tools template -f ./oscal-component.yaml --set .var.key1=value1 --set .const.key2=value2 + +To perform the full template operation, including sensitive data: + lula tools template -f ./oscal-component.yaml --render all + +Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file +See documentation for more detail on configuration schema ` -var templateCmd = &cobra.Command{ - Use: "template", - Short: "Template an artifact", - Long: "Resolving templated artifacts with configuration data", - Args: cobra.NoArgs, - Example: templateHelp, - Run: func(cmd *cobra.Command, args []string) { - // Read file - data, err := pkgCommon.ReadFileToBytes(templateOpts.InputFile) - if err != nil { - message.Fatal(err, err.Error()) - } - - // Get current viper pointer - v := common.GetViper() - // Get all viper settings - // This will only return config file items and resolved environment variables - // that have an associated key in the config file - viperData := v.AllSettings() - - // Handles merging viper config file data + environment variables - mergedMap := template.CollectTemplatingData(viperData) - - templatedData, err := template.ExecuteTemplate(mergedMap, string(data)) - if err != nil { - message.Fatalf(err, "error templating validation: %v", err) - } - - if templateOpts.OutputFile == "" { - _, err := os.Stdout.Write(templatedData) + +func TemplateCommand() *cobra.Command { + var ( + inputFile string + outputFile string + setOpts []string + renderTypeString string + ) + + cmd := &cobra.Command{ + Use: "template", + Short: "Template an artifact", + Long: "Resolving templated artifacts with configuration data", + Args: cobra.NoArgs, + Example: templateHelp, + RunE: func(cmd *cobra.Command, args []string) error { + // Read file + data, err := pkgCommon.ReadFileToBytes(inputFile) if err != nil { - message.Fatalf(err, "failed to write to stdout: %v", err) + return fmt.Errorf("error reading file: %v", err) } - } else { - err = files.CreateFileDirs(templateOpts.OutputFile) + + // Validate render type + renderType, err := parseRenderType(renderTypeString) if err != nil { - message.Fatalf(err, "failed to create output file path: %s\n", err) + message.Warnf("invalid render type, defaulting to masked: %v", err) } - err = os.WriteFile(templateOpts.OutputFile, templatedData, 0644) + + // Get constants and variables for templating from viper config + constants, variables, err := common.GetTemplateConfig() if err != nil { - message.Fatal(err, err.Error()) + return fmt.Errorf("error getting template config: %v", err) } - } - }, -} + // Get overrides from --set flag + overrides, err := common.ParseTemplateOverrides(setOpts) + if err != nil { + return fmt.Errorf("error parsing template overrides: %v", err) + } -func TemplateCommand() *cobra.Command { - return templateCmd + // Handles merging viper config file data + environment variables + // Throws an error if config keys are invalid for templating + templateData, err := template.CollectTemplatingData(constants, variables, overrides) + if err != nil { + return fmt.Errorf("error collecting templating data: %v", err) + } + + templateRenderer := template.NewTemplateRenderer(templateData) + output, err := templateRenderer.Render(string(data), renderType) + if err != nil { + return fmt.Errorf("error rendering template: %v", err) + } + + if outputFile == "" { + _, err := cmd.OutOrStdout().Write(output) + if err != nil { + return fmt.Errorf("failed to write to stdout: %v", err) + } + } else { + err = files.CreateFileDirs(outputFile) + if err != nil { + return fmt.Errorf("failed to create output file path: %v", err) + } + err = os.WriteFile(outputFile, output, 0644) + if err != nil { + return err + } + } + return nil + }, + } + + cmd.Flags().StringVarP(&inputFile, "input-file", "f", "", "the path to the target artifact") + cmd.MarkFlagRequired("input-file") + cmd.Flags().StringVarP(&outputFile, "output-file", "o", "", "the path to the output file. If not specified, the output file will be directed to stdout") + cmd.Flags().StringSliceVarP(&setOpts, "set", "s", []string{}, "set a value in the template data") + cmd.Flags().StringVarP(&renderTypeString, "render", "r", "masked", "values to render the template with, options are: masked, constants, non-sensitive, all") + + return cmd } func init() { common.InitViper() + toolsCmd.AddCommand(TemplateCommand()) +} - toolsCmd.AddCommand(templateCmd) - - templateCmd.Flags().StringVarP(&templateOpts.InputFile, "input-file", "f", "", "the path to the target artifact") - templateCmd.MarkFlagRequired("input-file") - templateCmd.Flags().StringVarP(&templateOpts.OutputFile, "output-file", "o", "", "the path to the output file. If not specified, the output file will be directed to stdout") +func parseRenderType(item string) (template.RenderType, error) { + switch strings.ToLower(item) { + case "masked": + return template.MASKED, nil + case "constants": + return template.CONSTANTS, nil + case "non-sensitive": + return template.NONSENSITIVE, nil + case "all": + return template.ALL, nil + } + return template.MASKED, fmt.Errorf("invalid render type: %s", item) } diff --git a/src/internal/template/helpers.go b/src/internal/template/helpers.go new file mode 100644 index 00000000..291d226f --- /dev/null +++ b/src/internal/template/helpers.go @@ -0,0 +1,14 @@ +package template + +import "fmt" + +func concatToRegoList(list []any) string { + regoList := "" + for i, item := range list { + if i > 0 { + regoList += ", " + } + regoList += `"` + fmt.Sprintf("%v", item) + `"` + } + return regoList +} diff --git a/src/internal/template/template.go b/src/internal/template/template.go index a67ccd2f..865c4e46 100644 --- a/src/internal/template/template.go +++ b/src/internal/template/template.go @@ -1,25 +1,169 @@ package template import ( + "fmt" "os" + "regexp" "strings" "text/template" - "github.com/defenseunicorns/pkg/helpers" + "github.com/defenseunicorns/lula/src/pkg/message" ) -const PREFIX = "LULA_" +const ( + PREFIX = "LULA_VAR_" + CONST = "const" + VAR = "var" +) + +type RenderType string + +const ( + MASKED RenderType = "masked" + CONSTANTS RenderType = "constants" + NONSENSITIVE RenderType = "non-sensitive" + ALL RenderType = "all" +) + +type TemplateRenderer struct { + tpl *template.Template + templateData *TemplateData +} + +func NewTemplateRenderer(templateData *TemplateData) *TemplateRenderer { + return &TemplateRenderer{ + tpl: createTemplate(), + templateData: templateData, + } +} + +func (r *TemplateRenderer) Render(templateString string, t RenderType) ([]byte, error) { + switch t { + case MASKED: + return r.ExecuteMaskedTemplate(templateString) + case CONSTANTS: + return r.ExecuteConstTemplate(templateString) + case NONSENSITIVE: + return r.ExecuteNonSensitiveTemplate(templateString) + case ALL: + return r.ExecuteFullTemplate(templateString) + default: + return []byte{}, fmt.Errorf("invalid render type: %s", t) + } +} + +type TemplateData struct { + Constants map[string]interface{} + Variables map[string]string + SensitiveVariables map[string]string +} + +func NewTemplateData() *TemplateData { + return &TemplateData{ + Constants: make(map[string]interface{}), + Variables: make(map[string]string), + SensitiveVariables: make(map[string]string), + } +} + +type VariableConfig struct { + Key string + Default string + Sensitive bool +} + +// ExecuteFullTemplate templates everything +func (r *TemplateRenderer) ExecuteFullTemplate(templateString string) ([]byte, error) { + tpl, err := r.tpl.Parse(templateString) + if err != nil { + return []byte{}, err + } + + var buffer strings.Builder + allVars := concatStringMaps(r.templateData.Variables, r.templateData.SensitiveVariables) + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: r.templateData.Constants, + VAR: allVars}) + if err != nil { + return []byte{}, err + } + + return []byte(buffer.String()), nil +} + +// ExecuteConstTemplate templates only constants +// this templates only values in the constants map +func (r *TemplateRenderer) ExecuteConstTemplate(templateString string) ([]byte, error) { + // Find anything {{ var.KEY }} and replace with {{ "{{ var.KEY }}" }} + re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) + templateString = re.ReplaceAllString(templateString, "{{ \"{{ ."+VAR+".$1 }}\" }}") + + tpl, err := r.tpl.Parse(templateString) + if err != nil { + return []byte{}, err + } + + var buffer strings.Builder + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: r.templateData.Constants}) + if err != nil { + return []byte{}, err + } + + return []byte(buffer.String()), nil +} + +// ExecuteNonSensitiveTemplate templates only constants and non-sensitive variables +// used for compose operations +func (r *TemplateRenderer) ExecuteNonSensitiveTemplate(templateString string) ([]byte, error) { + // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ "{{ var.KEY }}" }} + re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) + varMatches := re.FindAllStringSubmatch(templateString, -1) + uniqueMatches := returnUniqueMatches(varMatches, 1) + for k, matches := range uniqueMatches { + if _, ok := r.templateData.SensitiveVariables[matches[0]]; ok { + templateString = strings.ReplaceAll(templateString, k, "{{ \""+k+"\" }}") + } + } + + tpl, err := r.tpl.Parse(templateString) + if err != nil { + return []byte{}, err + } + + var buffer strings.Builder + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: r.templateData.Constants, + VAR: r.templateData.Variables}) + if err != nil { + return []byte{}, err + } + + return []byte(buffer.String()), nil +} + +// ExecuteMaskedTemplate templates all values, but masks the sensitive ones +// for display/printing only +func (r *TemplateRenderer) ExecuteMaskedTemplate(templateString string) ([]byte, error) { + // Find any sensitive keys {{ var.KEY }}, where KEY is in templateData.SensitiveVariables and replace with {{ var.KEY | mask }} + re := regexp.MustCompile(`{{\s*\.` + VAR + `\.([a-zA-Z0-9_]+)\s*}}`) + varMatches := re.FindAllStringSubmatch(templateString, -1) + uniqueMatches := returnUniqueMatches(varMatches, 1) + for k, matches := range uniqueMatches { + if _, ok := r.templateData.SensitiveVariables[matches[0]]; ok { + templateString = strings.ReplaceAll(templateString, k, "********") + } + } -// ExecuteTemplate templates the template string with the data map -func ExecuteTemplate(data map[string]interface{}, templateString string) ([]byte, error) { - tmpl, err := template.New("template").Parse(templateString) + tpl, err := r.tpl.Parse(templateString) if err != nil { return []byte{}, err } - tmpl.Option("missingkey=default") var buffer strings.Builder - err = tmpl.Execute(&buffer, data) + err = tpl.Execute(&buffer, map[string]interface{}{ + CONST: r.templateData.Constants, + VAR: r.templateData.Variables}) if err != nil { return []byte{}, err } @@ -27,23 +171,56 @@ func ExecuteTemplate(data map[string]interface{}, templateString string) ([]byte return []byte(buffer.String()), nil } -// Prepare the map of data for use in templating +// Prepare the templateData object for use in templating +func CollectTemplatingData(constants map[string]interface{}, variables []VariableConfig, overrides map[string]string) (*TemplateData, error) { + // Create the TemplateData object from the constants and variables + templateData := NewTemplateData() + + // check for invalid characters in keys + err := checkForInvalidKeys(constants, variables) + if err != nil { + return templateData, err + } -func CollectTemplatingData(data map[string]interface{}) map[string]interface{} { + templateData.Constants = constants + for _, variable := range variables { + if variable.Sensitive { + templateData.SensitiveVariables[variable.Key] = variable.Default + } else { + templateData.Variables[variable.Key] = variable.Default + } + } // Get all environment variables with a specific prefix envMap := GetEnvVars(PREFIX) - // Merge the data into a single map for use with templating - mergedMap := helpers.MergeMapRecursive(envMap, data) + // Update the templateData with the environment variables overrides + templateData.Variables = mergeStringMaps(templateData.Variables, envMap) + templateData.SensitiveVariables = mergeStringMaps(templateData.SensitiveVariables, envMap) - return mergedMap + // Apply overrides + overrideTemplateValues(templateData, overrides) + + // Validate that all env vars have values - currently debug prints missing env vars (do we want to return an error?) + var variablesMissing strings.Builder + for k, v := range templateData.Variables { + if v == "" { + variablesMissing.WriteString(fmt.Sprintf("variable %s is missing a value;\n", k)) + } + } + for k, v := range templateData.SensitiveVariables { + if v == "" { + variablesMissing.WriteString(fmt.Sprintf("sensitive variable %s is missing a value;\n", k)) + } + } + message.Debugf(variablesMissing.String()) + return templateData, nil } // get all environment variables with the established prefix -func GetEnvVars(prefix string) map[string]interface{} { - envMap := make(map[string]interface{}) +func GetEnvVars(prefix string) map[string]string { + envMap := make(map[string]string) // Get all environment variables envVars := os.Environ() @@ -69,3 +246,150 @@ func GetEnvVars(prefix string) map[string]interface{} { return envMap } + +// createTemplate creates a new template object +func createTemplate() *template.Template { + // Register custom template functions + funcMap := template.FuncMap{ + "concatToRegoList": func(a []any) string { + return concatToRegoList(a) + }, + // Add more custom functions as needed + } + + // Parse the template and apply the function map + tpl := template.New("template").Funcs(funcMap) + tpl.Option("missingkey=error") + + return tpl +} + +// mergeStringMaps merges two maps of strings into a single map of strings. +// m2 will overwrite m1 if a key exists in both maps, similar to left-join operation +func mergeStringMaps(m1, m2 map[string]string) map[string]string { + r := map[string]string{} + + for key, value := range m1 { + r[key] = value + } + + for key, value := range m2 { + // only add the key if it does exist in r + if _, ok := r[key]; ok { + r[key] = value + } + } + + return r +} + +// concatStringMaps concatenates two maps of strings into a single map of strings. +// m2 will overwrite m1 if a key exists in both maps +func concatStringMaps(m1, m2 map[string]string) map[string]string { + r := make(map[string]string) + + for key, value := range m1 { + r[key] = value + } + + for key, value := range m2 { + r[key] = value + } + return r +} + +// returnUniqueMatches returns a slice of unique matches from a slice of strings +func returnUniqueMatches(matches [][]string, captures int) map[string][]string { + uniqueMatches := make(map[string][]string) + for _, match := range matches { + fullMatch := match[0] + if _, exists := uniqueMatches[fullMatch]; !exists { + uniqueMatches[fullMatch] = match[captures:] + } + } + return uniqueMatches +} + +// checkForInvalidKeys checks for invalid characters in keys for go text/template +// cannot contain '-' or '.' +func checkForInvalidKeys(constants map[string]interface{}, variables []VariableConfig) error { + var errors strings.Builder + + containsInvalidChars := func(key string) { + if strings.Contains(key, "-") { + errors.WriteString(fmt.Sprintf("invalid key %s - cannot contain '-';", key)) + } + if strings.Contains(key, ".") { + errors.WriteString(fmt.Sprintf("invalid key %s - cannot contain '.';", key)) + } + } + + // check for invalid characters in keys, recursively through constants + var validateKeys func(m map[string]interface{}) + validateKeys = func(m map[string]interface{}) { + for key, value := range m { + containsInvalidChars(key) + if nestedMap, ok := value.(map[string]interface{}); ok { + validateKeys(nestedMap) + } + } + } + + validateKeys(constants) + + // check for invalid characters in keys in variables + for _, variable := range variables { + containsInvalidChars(variable.Key) + } + + if errors.Len() > 0 { + return fmt.Errorf(errors.String()[:len(errors.String())-1]) + } + + return nil +} + +// overrideTemplateValues overrides values in the templateData object with values from the overrides map +func overrideTemplateValues(templateData *TemplateData, overrides map[string]string) { + for path, value := range overrides { + // for each key, check if .var or .const + // if .var, set the value in the templateData.Variables or templateData.SensitiveVariables + // if .const, set the value in the templateData.Constants + if strings.HasPrefix(path, "."+VAR+".") { + key := strings.TrimPrefix(path, "."+VAR+".") + + if _, ok := templateData.SensitiveVariables[key]; ok { + templateData.SensitiveVariables[key] = value + } else { + templateData.Variables[key] = value + } + } else if strings.HasPrefix(path, "."+CONST+".") { + // Set the value in the templateData.Constants + key := strings.TrimPrefix(path, "."+CONST+".") + setNestedValue(templateData.Constants, key, value) + } + } +} + +// Helper function to set a value in a map based on a JSON-like key path +// Only supports basic map path (root.key.subkey) +func setNestedValue(m map[string]interface{}, path string, value interface{}) error { + keys := strings.Split(path, ".") + lastKey := keys[len(keys)-1] + + // Traverse the map, creating intermediate maps if necessary + for _, key := range keys[:len(keys)-1] { + if _, exists := m[key]; !exists { + m[key] = make(map[string]interface{}) + } + if nestedMap, ok := m[key].(map[string]interface{}); ok { + m = nestedMap + } else { + return fmt.Errorf("path %s contains a non-map value", key) + } + } + + // Set the final value + m[lastKey] = value + return nil +} diff --git a/src/internal/template/template_test.go b/src/internal/template/template_test.go index 379ebe9f..63fcc2a7 100644 --- a/src/internal/template/template_test.go +++ b/src/internal/template/template_test.go @@ -1,42 +1,626 @@ package template_test import ( + "fmt" "os" + "reflect" "strings" "testing" "github.com/defenseunicorns/lula/src/internal/template" ) -func TestExecuteTemplate(t *testing.T) { +func testRender(t *testing.T, templateRenderer *template.TemplateRenderer, templateString string, renderType template.RenderType, expected string) error { + t.Helper() - test := func(t *testing.T, data map[string]interface{}, preTemplate string, expected string) { + got, err := templateRenderer.Render(templateString, renderType) + if err != nil { + return fmt.Errorf("error templating data: %v\n", err.Error()) + } + + if string(got) != expected { + t.Fatalf("Expected %s - Got %s\n", expected, string(got)) + } + return nil +} + +func TestExecuteFullTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test template all with data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + secret template: my-secret + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template all with all empty data, error", func(t *testing.T) { + templateData := template.NewTemplateData() + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template all with invalid template paths, error", func(t *testing.T) { + templateData := template.NewTemplateData() + + templateString := ` + constant template: {{ .constant.testVar }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) + + t.Run("Test template all with invalid template characters, error", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "test-var": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + + templateString := ` + constant template: {{ .const.test-var }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template all with invalid template subpath, error", func(t *testing.T) { + templateData := template.NewTemplateData() + + templateString := ` + variable template: {{ .var.nokey.sub }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) + + t.Run("Test template all with invalid template, error", func(t *testing.T) { + templateData := template.NewTemplateData() + + templateString := ` + constant template: {{ constant.testVar }} + ` + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, "") + if err == nil { + t.Fatal("expected error, got nil") + } + }) + + t.Run("Test template on 'concatToRegoList' function", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "exemptions": []interface{}{"one", "two", "three"}, + }, + } + + templateString := ` + constant template: {{ .const.exemptions | concatToRegoList }} + ` + expected := ` + constant template: "one", "two", "three" + ` + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.ALL, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + +} + +func TestExecuteConstTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test template const with data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test template const with missing var data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test template const with weird spacing in template", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{.var.some_env_var}} + secret template: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template const with empty data", func(t *testing.T) { + templateData := &template.TemplateData{ + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.CONSTANTS, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) + +} + +func TestExecuteNonSensitiveTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test template nonsensitive with data and duplicate matches", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + variable template2: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + secret template2: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + variable template2: my-env-var + secret template: {{ .var.some_lula_secret }} + secret template2: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.NONSENSITIVE, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test template nonsensitive with weird spacing in template", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{.var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + secret template: {{.var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.NONSENSITIVE, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test template nonsensitive with empty var data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.NONSENSITIVE, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) + +} + +func TestExecuteMaskedTemplate(t *testing.T) { + t.Parallel() + + t.Run("Test masked template with sensitive data and duplicates", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + variable template2: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + secret template2: {{ .var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + variable template2: my-env-var + secret template: ******** + secret template2: ******** + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.MASKED, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test masked template with weird spacing in template", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{.var.some_lula_secret }} + ` + expected := ` + constant template: testing + variable template: my-env-var + secret template: ******** + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.MASKED, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + // Note - this will change depending on the tpl.Option chosen + t.Run("Test masked template with missing data", func(t *testing.T) { + templateData := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + templateString := ` + constant template: {{ .const.testVar }} + variable template: {{ .var.some_env_var }} + secret template: {{ .var.some_lula_secret }} + ` + + tr := template.NewTemplateRenderer(templateData) + err := testRender(t, tr, templateString, template.MASKED, "") + if err == nil { + t.Fatalf("Expected an error, but got nil") + } + }) +} + +func TestCollectTemplatingData(t *testing.T) { + + test := func(t *testing.T, constants map[string]interface{}, variables []template.VariableConfig, overrides map[string]string, expected *template.TemplateData) error { t.Helper() // templateData returned - got, err := template.ExecuteTemplate(data, preTemplate) + got, err := template.CollectTemplatingData(constants, variables, overrides) if err != nil { - t.Fatalf("error templating data: %s\n", err.Error()) + return err } - if string(got) != expected { - t.Fatalf("Expected %s - Got %s\n", expected, string(got)) + if !reflect.DeepEqual(got, expected) { + t.Fatalf("Expected %v - Got %v\n", expected, got) } + return nil } - t.Run("Test {{ .testVar }} with data", func(t *testing.T) { - data := map[string]interface{}{ + t.Run("Test collect templating data", func(t *testing.T) { + var overrides map[string]string + constants := map[string]interface{}{ "testVar": "testing", } - - test(t, data, "{{ .testVar }}", "testing") + variables := []template.VariableConfig{ + { + Key: "some_env_var", + Default: "my-env-var", + Sensitive: false, + }, + { + Key: "some_lula_secret", + Default: "my-secret", + Sensitive: true, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "my-secret", + }, + } + err := test(t, constants, variables, overrides, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } }) - t.Run("Test {{ .testVar }} but empty data", func(t *testing.T) { - data := map[string]interface{}{} + t.Run("Test collect templating data with env vars", func(t *testing.T) { + os.Setenv("LULA_VAR_SOME_LULA_SECRET", "env-secret") + os.Setenv("LULA_VAR_SOME_ENV_VAR", "env-var") + defer os.Unsetenv("LULA_VAR_SOME_LULA_SECRET") + defer os.Unsetenv("LULA_VAR_SOME_ENV_VAR") - test(t, data, "{{ .testVar }}", "") + var overrides map[string]string + constants := map[string]interface{}{ + "testVar": "testing", + } + variables := []template.VariableConfig{ + { + Key: "some_env_var", + Default: "my-env-var", + Sensitive: false, + }, + { + Key: "some_lula_secret", + Default: "my-secret", + Sensitive: true, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "env-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "env-secret", + }, + } + err := test(t, constants, variables, overrides, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } + }) + + t.Run("Test collect templating data with overrides", func(t *testing.T) { + overrides := map[string]string{ + ".var.some_env_var": "override-var", + ".var.some_lula_secret": "override-secret", + ".const.test.subkey": "override-subkey", + } + constants := map[string]interface{}{ + "testVar": "testing", + "test": map[string]interface{}{ + "subkey": "subkey-value", + }, + } + variables := []template.VariableConfig{ + { + Key: "some_env_var", + Default: "my-env-var", + Sensitive: false, + }, + { + Key: "some_lula_secret", + Default: "my-secret", + Sensitive: true, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + "test": map[string]interface{}{ + "subkey": "override-subkey", + }, + }, + Variables: map[string]string{ + "some_env_var": "override-var", + }, + SensitiveVariables: map[string]string{ + "some_lula_secret": "override-secret", + }, + } + err := test(t, constants, variables, overrides, expected) + if err != nil { + t.Fatalf("Expected no error, but got %v", err) + } }) + t.Run("Test collect templating data with bad keys", func(t *testing.T) { + var overrides map[string]string + constants := map[string]interface{}{ + "test-var": "testing", + "anotherkey": map[string]interface{}{ + "sub.key": "testing", + }, + } + variables := []template.VariableConfig{ + { + Key: "some-env-var", + Default: "my-env-var", + Sensitive: false, + }, + } + expected := &template.TemplateData{ + Constants: map[string]interface{}{ + "testVar": "testing", + }, + Variables: map[string]string{ + "some_env_var": "my-env-var", + }, + SensitiveVariables: map[string]string{}, + } + err := test(t, constants, variables, overrides, expected) + if err == nil { + t.Fatal("expected error, got nil") + } + numErrors := strings.Count(err.Error(), "invalid key") + if numErrors != 3 { + t.Fatalf("expected 3 invalid constant keys, got %d", numErrors) + } + }) } func TestGetEnvVars(t *testing.T) { @@ -45,6 +629,7 @@ func TestGetEnvVars(t *testing.T) { t.Helper() os.Setenv(key, value) + defer os.Unsetenv(key) envMap := template.GetEnvVars(prefix) // convert key to expected format @@ -53,7 +638,6 @@ func TestGetEnvVars(t *testing.T) { if envMap[strings.ToLower(strippedKey)] != value { t.Fatalf("Expected %s - Got %s\n", value, envMap[strings.ToLower(strippedKey)]) } - os.Unsetenv(key) } t.Run("Test LULA_RESOURCE - Pass", func(t *testing.T) { diff --git a/src/internal/tui/model_test.go b/src/internal/tui/model_test.go index eb2d72e2..89fa7e77 100644 --- a/src/internal/tui/model_test.go +++ b/src/internal/tui/model_test.go @@ -14,7 +14,7 @@ import ( "github.com/muesli/termenv" ) -const timeout = time.Second * 10 +const timeout = time.Second * 20 func init() { lipgloss.SetColorProfile(termenv.Ascii) diff --git a/src/test/e2e/standard/lula-config.yaml b/src/test/e2e/standard/lula-config.yaml index 70e8fe6f..911f39ff 100644 --- a/src/test/e2e/standard/lula-config.yaml +++ b/src/test/e2e/standard/lula-config.yaml @@ -1,8 +1,20 @@ -resources: - jsoncm: configmaps - yamlcm: configmaps - secret: secrets - pod: pods +constants: + type: software + title: lula -type: software -title: lula \ No newline at end of file + resources: + name: test-pod-label + namespace: validation-test + exemptions: + - one + - two + - three + +variables: + - key: some_lula_secret + sensitive: true + - key: some_env_var + default: this-should-be-overridden + +log_level: info +target: il5 diff --git a/src/test/e2e/standard/template_test.go b/src/test/e2e/standard/template_test.go index 478a8424..d2f84528 100644 --- a/src/test/e2e/standard/template_test.go +++ b/src/test/e2e/standard/template_test.go @@ -1,19 +1,21 @@ package test import ( - "bytes" + "flag" "os" + "path/filepath" - "strings" "testing" "github.com/defenseunicorns/lula/src/cmd" "github.com/defenseunicorns/lula/src/test/util" ) +var updateGolden = flag.Bool("update", false, "update golden files") + func TestTemplateCommand(t *testing.T) { - test := func(t *testing.T, expectError bool, args ...string) (string, error) { + test := func(t *testing.T, goldenFileName string, expectError bool, args ...string) error { t.Helper() cmdArgs := []string{"tools", "template"} @@ -22,54 +24,101 @@ func TestTemplateCommand(t *testing.T) { cmd := cmd.RootCommand() _, output, err := util.ExecuteCommand(cmd, cmdArgs...) - if err != nil && !expectError { - t.Fatal(err) + if err != nil { + if !expectError { + return err + } else { + return nil + } } - return output, err + if !expectError { + goldenFile := filepath.Join("testdata", goldenFileName+".golden") + + if *updateGolden && !expectError { + err = os.WriteFile(goldenFile, []byte(output), 0644) + if err != nil { + return err + } + } + + expected, err := os.ReadFile(goldenFile) + if err != nil { + return err + } + + if output != string(expected) { + t.Fatalf("Expected:\n%s\n - Got \n%s\n", expected, output) + } + } + + return nil } - t.Run("Template Valid File", func(t *testing.T) { + t.Run("Template Validation", func(t *testing.T) { + err := test(t, "validation", false, "-f", "../../unit/common/validation/validation.tmpl.yaml") + if err != nil { + t.Fatal(err) + } + }) - _, err := test(t, false, "-f", "../../unit/common/oscal/valid-component-template.yaml", "-o", "valid.yaml") - defer os.Remove("valid.yaml") + t.Run("Template Validation with env vars", func(t *testing.T) { + os.Setenv("LULA_VAR_SOME_ENV_VAR", "my-env-var") + defer os.Unsetenv("LULA_VAR_SOME_ENV_VAR") + err := test(t, "validation_with_env_vars", false, "-f", "../../unit/common/validation/validation.tmpl.yaml") if err != nil { t.Fatal(err) } + }) - // this comparison using golden files would make more sense - templated, err := os.ReadFile("valid.yaml") + t.Run("Template Validation with set", func(t *testing.T) { + err := test(t, "validation_with_set", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--set", ".const.resources.name=foo") if err != nil { t.Fatal(err) } + }) - valid, err := os.ReadFile("../../unit/common/oscal/valid-component.yaml") + t.Run("Template Validation for all", func(t *testing.T) { + os.Setenv("LULA_VAR_SOME_LULA_SECRET", "env-secret") + defer os.Unsetenv("LULA_VAR_SOME_LULA_SECRET") + err := test(t, "validation_all", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--render", "all") if err != nil { t.Fatal(err) } + }) - if !bytes.Equal(templated, valid) { - t.Fatalf("Expected: \n%s\n - Got \n%s\n", valid, templated) + t.Run("Template Validation for non-sensitive", func(t *testing.T) { + err := test(t, "validation_non_sensitive", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--render", "non-sensitive") + if err != nil { + t.Fatal(err) } + }) + t.Run("Template Validation for constants", func(t *testing.T) { + err := test(t, "validation_constants", false, "-f", "../../unit/common/validation/validation.tmpl.yaml", "--render", "constants") + if err != nil { + t.Fatal(err) + } }) t.Run("Test help", func(t *testing.T) { - out, _ := test(t, false, "--help") - - if !strings.Contains(out, "Resolving templated artifacts with configuration data") { - t.Fatalf("Expected help string") + err := test(t, "help", false, "--help") + if err != nil { + t.Fatal(err) } }) - // Tests that execute unhappy-paths will hit a fatal message which exits the runtime - // TODO: review RunE command execution and ensure we don't prematurely exit where errors would still be valuable - // t.Run("Test non-existent file", func(t *testing.T) { - // out, _ := test(t, true, "-f", "non-existent.yaml") - - // if !strings.Contains(out, "Path: non-existent.yaml does not exist - unable to digest document") { - // t.Fatalf("Expected error with unable to digest document error") - // } - // }) + t.Run("Template Validation - invalid file error", func(t *testing.T) { + err := test(t, "empty", true, "-f", "not-a-file.yaml") + if err != nil { + t.Fatal(err) + } + }) + t.Run("Template Validation - invalid file schema error", func(t *testing.T) { + err := test(t, "empty", true, "-f", "../../unit/common/validation/validation.bad.tmpl.yaml") + if err != nil { + t.Fatal(err) + } + }) } diff --git a/src/test/e2e/standard/testdata/empty.golden b/src/test/e2e/standard/testdata/empty.golden new file mode 100644 index 00000000..e69de29b diff --git a/src/test/e2e/standard/testdata/help.golden b/src/test/e2e/standard/testdata/help.golden new file mode 100644 index 00000000..4b858bed --- /dev/null +++ b/src/test/e2e/standard/testdata/help.golden @@ -0,0 +1,32 @@ +Resolving templated artifacts with configuration data + +Usage: + lula tools template [flags] + +Examples: + +To template an OSCAL Model, defaults to masking sensitive variables: + lula tools template -f ./oscal-component.yaml + +To indicate a specific output file: + lula tools template -f ./oscal-component.yaml -o templated-oscal-component.yaml + +To perform overrides on the template data: + lula tools template -f ./oscal-component.yaml --set .var.key1=value1 --set .const.key2=value2 + +To perform the full template operation, including sensitive data: + lula tools template -f ./oscal-component.yaml --render all + +Data for templating should be stored under 'constants' or 'variables' configuration items in a lula-config.yaml file +See documentation for more detail on configuration schema + + +Flags: + -h, --help help for template + -f, --input-file string the path to the target artifact + -o, --output-file string the path to the output file. If not specified, the output file will be directed to stdout + -r, --render string values to render the template with, options are: masked, constants, non-sensitive, all (default "masked") + -s, --set strings set a value in the template data + +Global Flags: + -l, --log-level string Log level when running Lula. Valid options are: warn, info, debug, trace (default "info") diff --git a/src/test/e2e/standard/testdata/validation.golden b/src/test/e2e/standard/testdata/validation.golden new file mode 100644 index 00000000..ff319eb2 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: test-pod-label + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "********" == "********" + } + msg = validate.msg + + value_of_my_secret := ******** \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_all.golden b/src/test/e2e/standard/testdata/validation_all.golden new file mode 100644 index 00000000..dc22f17b --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_all.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "env-secret" == "********" + } + msg = validate.msg + + value_of_my_secret := env-secret \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_constants.golden b/src/test/e2e/standard/testdata/validation_constants.golden new file mode 100644 index 00000000..91e814e7 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_constants.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "{{ .var.some_env_var }}" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_non_sensitive.golden b/src/test/e2e/standard/testdata/validation_non_sensitive.golden new file mode 100644 index 00000000..e1282c92 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_non_sensitive.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_with_env_vars.golden b/src/test/e2e/standard/testdata/validation_with_env_vars.golden new file mode 100644 index 00000000..c429cbc2 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_with_env_vars.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: test-pod-label + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "my-env-var" == "my-env-var" + "********" == "********" + } + msg = validate.msg + + value_of_my_secret := ******** \ No newline at end of file diff --git a/src/test/e2e/standard/testdata/validation_with_set.golden b/src/test/e2e/standard/testdata/validation_with_set.golden new file mode 100644 index 00000000..f7be2dc4 --- /dev/null +++ b/src/test/e2e/standard/testdata/validation_with_set.golden @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: foo + version: v1 + resource: pods + namespaces: [validation-test] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { "one", "two", "three" } + "this-should-be-overridden" == "my-env-var" + "********" == "********" + } + msg = validate.msg + + value_of_my_secret := ******** \ No newline at end of file diff --git a/src/test/unit/common/validation/validation.bad.tmpl.yaml b/src/test/unit/common/validation/validation.bad.tmpl.yaml new file mode 100644 index 00000000..3ea347df --- /dev/null +++ b/src/test/unit/common/validation/validation.bad.tmpl.yaml @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: {{ constant.resources.name }} + version: v1 + resource: pods + namespaces: [{{ .const.missing-key }}] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { {{ .const.resources.exemptions | concatToRegoList }} } + "{{ .var.some_env_var }}" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/unit/common/validation/validation.tmpl.yaml b/src/test/unit/common/validation/validation.tmpl.yaml new file mode 100644 index 00000000..f9eb5baf --- /dev/null +++ b/src/test/unit/common/validation/validation.tmpl.yaml @@ -0,0 +1,30 @@ +domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podvt + resource-rule: + name: {{ .const.resources.name }} + version: v1 + resource: pods + namespaces: [{{ .const.resources.namespace }}] +provider: + type: opa + opa-spec: + rego: | + package validate + import rego.v1 + + # Default values + default validate := false + default msg := "Not evaluated" + + # Validation result + validate if { + { "one", "two", "three" } == { {{ .const.resources.exemptions | concatToRegoList }} } + "{{ .var.some_env_var }}" == "my-env-var" + "{{ .var.some_lula_secret }}" == "********" + } + msg = validate.msg + + value_of_my_secret := {{ .var.some_lula_secret }} \ No newline at end of file diff --git a/src/test/util/utils.go b/src/test/util/utils.go index cb1663d7..978b3990 100644 --- a/src/test/util/utils.go +++ b/src/test/util/utils.go @@ -124,9 +124,12 @@ func ExecuteCommandC(cmd *cobra.Command, args ...string) (c *cobra.Command, outp cmd.SetErr(buf) cmd.SetArgs(args) - cmd.Execute() + execErr := cmd.Execute() - out, err := io.ReadAll(buf) + out, readErr := io.ReadAll(buf) + if readErr != nil { + return cmd, "", readErr + } - return cmd, string(out), err + return cmd, string(out), execErr }