diff --git a/cmd/monaco/deploy/deploy.go b/cmd/monaco/deploy/deploy.go index ec138b349..5786493e4 100644 --- a/cmd/monaco/deploy/deploy.go +++ b/cmd/monaco/deploy/deploy.go @@ -192,28 +192,29 @@ func validateProjectsWithEnvironments(projects []project.Project, envs manifest. } func collectOpenPipelineCoordinatesByKind(cfgPerType v2.ConfigsPerType, dest KindCoordinates) { - cfgPerType.ForEveryConfigDo(func(cfg config.Config) { + for cfg := range cfgPerType.AllConfigs { if cfg.Skip { - return + continue } if openPipelineType, ok := cfg.Type.(config.OpenPipelineType); ok { dest[openPipelineType.Kind] = append(dest[openPipelineType.Kind], cfg.Coordinate) } - }) + } } func collectPlatformCoordinates(cfgPerType v2.ConfigsPerType) []coordinate.Coordinate { plaformCoordinates := []coordinate.Coordinate{} - cfgPerType.ForEveryConfigDo(func(cfg config.Config) { + + for cfg := range cfgPerType.AllConfigs { if cfg.Skip { - return + continue } if configRequiresPlatform(cfg) { plaformCoordinates = append(plaformCoordinates, cfg.Coordinate) } - }) + } return plaformCoordinates } diff --git a/cmd/monaco/download/download_configs.go b/cmd/monaco/download/download_configs.go index 17651db39..3f727805c 100644 --- a/cmd/monaco/download/download_configs.go +++ b/cmd/monaco/download/download_configs.go @@ -26,7 +26,9 @@ import ( "github.com/dynatrace/dynatrace-configuration-as-code/v2/cmd/monaco/support" "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/featureflags" "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log" + "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log/field" "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/secret" + "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/template" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/api" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/client" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config" @@ -218,6 +220,20 @@ func doDownloadConfigs(fs afero.Fs, clientSet *client.ClientSet, apisToDownload return nil } + for c := range downloadedConfigs.AllConfigs { + // We would need quite a huge refactoring to support Classic- and Automation-APIS here. + // Automation already also does what we do here, but does set custom {{.variables}} that we can't easily escape here. + // To fix this, it might be better do extract the variables at a later place instead of doing it before. + if c.Type.ID() == config.ClassicApiTypeID || c.Type.ID() == config.AutomationTypeID { + continue + } + + err := escapeGoTemplating(&c) + if err != nil { + log.WithFields(field.Coordinate(c.Coordinate), field.Error(err)).Warn("Failed to escape Go templating expressions. Template needs manual adaptation: %s", err) + } + } + log.Info("Resolving dependencies between configurations") downloadedConfigs, err = dependency_resolution.ResolveDependencies(downloadedConfigs) if err != nil { @@ -234,6 +250,22 @@ func doDownloadConfigs(fs afero.Fs, clientSet *client.ClientSet, apisToDownload return writeConfigs(downloadedConfigs, opts.downloadOptionsShared, fs) } +func escapeGoTemplating(c *config.Config) error { + content, err := c.Template.Content() + if err != nil { + return err + } + + content = string(template.UseGoTemplatesForDoubleCurlyBraces([]byte(content))) + + err = c.Template.UpdateContent(content) + if err != nil { + return err + } + + return nil +} + type downloadFn struct { classicDownload func(client.ConfigClient, string, api.APIs, classic.ContentFilters) (projectv2.ConfigsPerType, error) settingsDownload func(client.SettingsClient, string, settings.Filters, ...config.SettingsType) (projectv2.ConfigsPerType, error) diff --git a/cmd/monaco/download/download_integration_test.go b/cmd/monaco/download/download_integration_test.go index e62a64f45..15444e165 100644 --- a/cmd/monaco/download/download_integration_test.go +++ b/cmd/monaco/download/download_integration_test.go @@ -1099,6 +1099,66 @@ func TestDownloadIntegrationDownloadsAPIsAndSettings(t *testing.T) { assert.Equal(t, len(configs["settings-schema"]), 3, "Expected 3 settings objects") } +func TestDownloadGoTemplateExpressionsAreEscaped(t *testing.T) { + // GIVEN apis, server responses, file system + const projectName = "integration-test-go-templating-expressions-are-escaped" + const testBasePath = "test-resources/" + projectName + + // Responses + responses := map[string]string{ + "/platform/classic/environment-api/v2/settings/schemas": "settings/__SCHEMAS.json", + "/platform/classic/environment-api/v2/settings/objects": "settings/objects.json", + } + + // Server + server := dtclient.NewIntegrationTestServer(t, testBasePath, responses) + + fs := afero.NewMemMapFs() + + opts := setupTestingDownloadOptions(t, server, projectName) + opts.onlySettings = false + opts.onlyAPIs = false + + configClient, err := dtclient.NewClassicConfigClientForTesting(server.URL, server.Client()) + require.NoError(t, err) + + settingsClient, err := dtclient.NewPlatformSettingsClientForTesting(server.URL, server.Client()) + require.NoError(t, err) + + err = doDownloadConfigs(fs, &client.ClientSet{ConfigClient: configClient, SettingsClient: settingsClient}, api.APIs{}, opts) + assert.NoError(t, err) + + // THEN we can load the project again and verify its content + projects, errs := loadDownloadedProjects(fs, api.APIs{}) + if len(errs) != 0 { + for _, err := range errs { + t.Errorf("%v", err) + } + return + } + + assert.Len(t, projects, 1) + p := projects[0] + assert.Equal(t, p.Id, projectName) + assert.Len(t, p.Configs, 1) + + configsPerType, found := p.Configs[projectName] + assert.True(t, found) + assert.Equal(t, len(configsPerType), 1, "Expected one Settings schema to be downloaded") + + settingsDownloaded, f := configsPerType["settings-schema"] + assert.True(t, f) + assert.Len(t, settingsDownloaded, 1, "Expected 1 settings object") + + obj := settingsDownloaded[0] + content, err := obj.Template.Content() + + assert.JSONEq(t, "{"+ + "\"name\": \"SettingsTest-1\","+ + "\"DQL\": \"fetch bizevents | FILTER like(event.type,\\\"platform.LoginEvent%\\\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{`{{`}}CountryIso, alias:countryIso}, {Country, alias:country{{`}}`}} | sort quantity desc\""+ + "}", content) +} + func TestDownloadIntegrationDownloadsOnlyAPIsIfConfigured(t *testing.T) { // GIVEN apis, server responses, file system diff --git a/cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/__SCHEMAS.json b/cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/__SCHEMAS.json new file mode 100644 index 000000000..dd6162244 --- /dev/null +++ b/cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/__SCHEMAS.json @@ -0,0 +1,8 @@ +{ + "totalCount": 1, + "items": [ + { + "schemaId": "settings-schema" + } + ] +} diff --git a/cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/objects.json b/cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/objects.json new file mode 100644 index 000000000..57e7b50d2 --- /dev/null +++ b/cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/objects.json @@ -0,0 +1,15 @@ +{ + "items": [ + { + "externalId": "abc", + "schemaVersion": "1", + "schemaId": "settings-schema", + "objectId": "so_1", + "scope": "test", + "value": { + "name": "SettingsTest-1", + "DQL": "fetch bizevents | FILTER like(event.type,\"platform.LoginEvent%\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{CountryIso, alias:countryIso}, {Country, alias:country}} | sort quantity desc" + } + } + ] +} diff --git a/pkg/download/automation/internal/jinja.go b/internal/template/escape.go similarity index 68% rename from pkg/download/automation/internal/jinja.go rename to internal/template/escape.go index 8ceb026d3..203867be8 100644 --- a/pkg/download/automation/internal/jinja.go +++ b/internal/template/escape.go @@ -1,6 +1,6 @@ /* * @license - * Copyright 2023 Dynatrace LLC + * Copyright 2025 Dynatrace LLC * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -14,12 +14,15 @@ * limitations under the License. */ -package internal +package template -import "bytes" +import ( + "bytes" +) -// EscapeJinjaTemplates replaces each occurrence of "{{" with "{{`{{`}}" and each occurrence of "}}" with "{{`}}`}}" -func EscapeJinjaTemplates(src []byte) []byte { +// UseGoTemplatesForDoubleCurlyBraces replaces each occurrence of "{{" with "{{`{{`}}" and each occurrence of "}}" with "{{`}}`}}". +// This ensures that when the returned string is used to render templates, e.g. during deployment, the "{{" and "}}" are not misinterpreted. +func UseGoTemplatesForDoubleCurlyBraces(src []byte) []byte { src = bytes.ReplaceAll(src, []byte("{{"), []byte("{{`{{`")) // replace is divided in 2 steps to avoid replacing of closing brackets in the next step src = bytes.ReplaceAll(src, []byte("}}"), []byte("{{`}}`}}")) src = bytes.ReplaceAll(src, []byte("{{`{{`"), []byte("{{`{{`}}")) diff --git a/pkg/download/automation/internal/jinja_test.go b/internal/template/escape_test.go similarity index 65% rename from pkg/download/automation/internal/jinja_test.go rename to internal/template/escape_test.go index ea761d0db..48860e5e5 100644 --- a/pkg/download/automation/internal/jinja_test.go +++ b/internal/template/escape_test.go @@ -2,7 +2,7 @@ /* * @license - * Copyright 2023 Dynatrace LLC + * Copyright 2025 Dynatrace LLC * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,16 +16,18 @@ * limitations under the License. */ -package internal +package template import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) -func TestEscapeJinjaTemplates(t *testing.T) { +func TestEscapeGoTemplating(t *testing.T) { tc := []struct { expected, in string + name string }{ { in: `Hello, {{planet}}!`, @@ -66,11 +68,15 @@ func TestEscapeJinjaTemplates(t *testing.T) { in: `{{ }}`, expected: "{{`{{`}} {{`}}`}}", }, + { + in: "fetch bizevents | FILTER `event.provider` == $MyVariable | FILTER like(event.type,\\\"platform.LoginEvent%\\\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{CountryIso, alias:countryIso}, {Country, alias:country}} | sort quantity desc", + expected: "fetch bizevents | FILTER `event.provider` == $MyVariable | FILTER like(event.type,\\\"platform.LoginEvent%\\\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{`{{`}}CountryIso, alias:countryIso}, {Country, alias:country{{`}}`}} | sort quantity desc", + }, } for _, tt := range tc { t.Run(tt.in, func(t *testing.T) { - out := EscapeJinjaTemplates([]byte(tt.in)) + out := UseGoTemplatesForDoubleCurlyBraces([]byte(tt.in)) assert.Equal(t, tt.expected, string(out)) }) diff --git a/pkg/download/automation/download.go b/pkg/download/automation/download.go index 783a80803..cb99b34ee 100644 --- a/pkg/download/automation/download.go +++ b/pkg/download/automation/download.go @@ -21,22 +21,24 @@ import ( "context" "encoding/json" "fmt" + "time" + + "golang.org/x/exp/maps" + automationAPI "github.com/dynatrace/dynatrace-configuration-as-code-core/api/clients/automation" "github.com/dynatrace/dynatrace-configuration-as-code-core/clients/automation" "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/automationutils" jsonutils "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/json" "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log" "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log/field" + templateEscaper "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/template" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/client" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/coordinate" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/parameter" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/parameter/value" "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/template" - "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/download/automation/internal" v2 "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/project/v2" - "golang.org/x/exp/maps" - "time" ) var automationTypesToResources = map[config.AutomationType]automationAPI.ResourceType{ @@ -127,7 +129,7 @@ func Download(cl client.AutomationClient, projectName string, automationTypes .. func escapeJinjaTemplates(src []byte) ([]byte, error) { var prettyJSON bytes.Buffer err := json.Indent(&prettyJSON, src, "", "\t") - return internal.EscapeJinjaTemplates(prettyJSON.Bytes()), err + return templateEscaper.UseGoTemplatesForDoubleCurlyBraces(prettyJSON.Bytes()), err } func createTemplateFromRawJSON(obj automationutils.Response, configType, projectName string) (t template.Template, extractedName *string) { diff --git a/pkg/project/v2/project.go b/pkg/project/v2/project.go index ef6ae8eaa..a95e3c389 100644 --- a/pkg/project/v2/project.go +++ b/pkg/project/v2/project.go @@ -102,34 +102,35 @@ func (p Project) String() string { // ForEveryConfigDo executes the given ActionOverConfig actions for each configuration defined in the project for each environment // Actions can not modify the configs inside the Project. -func (p Project) ForEveryConfigDo(actions ...ActionOverConfig) { - p.forEveryConfigDo("", actions) +func (p Project) ForEveryConfigDo(action ActionOverConfig) { + p.forEveryConfigDo("", action) } // ForEveryConfigInEnvironmentDo executes the given ActionOverConfig actions for each configuration defined in the project for a given environment. // It behaves like ForEveryConfigDo just limited to a single environment. // Actions can not modify the configs inside the Project. -func (p Project) ForEveryConfigInEnvironmentDo(environment string, actions ...ActionOverConfig) { - p.forEveryConfigDo(environment, actions) +func (p Project) ForEveryConfigInEnvironmentDo(environment string, action ActionOverConfig) { + p.forEveryConfigDo(environment, action) } // forEveryConfigDo applies the given action to every configuration, either for a single environment if requested, // or for all environments if the environemnt parameter is empty. -func (p Project) forEveryConfigDo(environment string, actions []ActionOverConfig) { +func (p Project) forEveryConfigDo(environment string, action ActionOverConfig) { for env, cpt := range p.Configs { if environment == "" || environment == env { - cpt.ForEveryConfigDo(actions...) + for c := range cpt.AllConfigs { + action(c) + } } } } -// ForEveryConfigDo executes the given ActionOverConfig actions for each configuration defined in the ConfigsPerType. -// Actions can not modify the configs inside the ConfigsPerType. -func (cpt ConfigsPerType) ForEveryConfigDo(actions ...ActionOverConfig) { +// AllConfigs is an iterator iterating over all configs +func (cpt ConfigsPerType) AllConfigs(yield func(config.Config) bool) { for _, cs := range cpt { for _, c := range cs { - for _, f := range actions { - f(c) + if !yield(c) { + return } } } diff --git a/pkg/project/v2/project_test.go b/pkg/project/v2/project_test.go index fa0cfcb4b..468fb3b6b 100644 --- a/pkg/project/v2/project_test.go +++ b/pkg/project/v2/project_test.go @@ -218,3 +218,40 @@ func TestProject_ForEveryConfigDo(t *testing.T) { assert.Contains(t, actual, "config4") }) } + +func TestConfigsPerType_AllConfigs(t *testing.T) { + + var ( + c1 = config.Config{Coordinate: coordinate.Coordinate{Project: "project1", Type: "type1", ConfigId: "config1"}} + c2 = config.Config{Coordinate: coordinate.Coordinate{Project: "project1", Type: "type1", ConfigId: "config2"}} + c3 = config.Config{Coordinate: coordinate.Coordinate{Project: "project1", Type: "type2", ConfigId: "config1"}} + ) + + cpt := project.ConfigsPerType{ + "type1": {c1, c2}, + "type2": {c3}, + } + + t.Run("All items are yielded", func(t *testing.T) { + var result []config.Config + for c := range cpt.AllConfigs { + result = append(result, c) + } + + assert.ElementsMatch(t, []config.Config{c1, c2, c3}, result) + }) + + t.Run("Returning after the second item does no longer iterate", func(t *testing.T) { + var result []config.Config + for c := range cpt.AllConfigs { + result = append(result, c) + + if len(result) == 2 { + break + } + } + + assert.Len(t, result, 2) + }) + +}