fix(download): Escape all Go templating expressions after downloading

cmd/monaco/download/download_configs.go

@@ -26,7 +26,9 @@ import (
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/cmd/monaco/support"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/featureflags"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log"
+	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log/field"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/secret"
+	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/template"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/api"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/client"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config"
@@ -218,6 +220,20 @@ func doDownloadConfigs(fs afero.Fs, clientSet *client.ClientSet, apisToDownload
 		return nil
 	}
 
+	for c := range downloadedConfigs.AllConfigs {
+		// We would need quite a huge refactoring to support Classic- and Automation-APIS here.
+		// Automation already also does what we do here, but does set custom {{.variables}} that we can't easily escape here.
+		// To fix this, it might be better do extract the variables at a later place instead of doing it before.
+		if c.Type.ID() == config.ClassicApiTypeID || c.Type.ID() == config.AutomationTypeID {
+			continue
+		}
+
+		err := escapeGoTemplating(&c)
+		if err != nil {
+			log.WithFields(field.Coordinate(c.Coordinate), field.Error(err)).Warn("Failed to escape Go templating expressions. Template needs manual adaptation: %s", err)
+		}
+	}
+
 	log.Info("Resolving dependencies between configurations")
 	downloadedConfigs, err = dependency_resolution.ResolveDependencies(downloadedConfigs)
 	if err != nil {
@@ -234,6 +250,22 @@ func doDownloadConfigs(fs afero.Fs, clientSet *client.ClientSet, apisToDownload
 	return writeConfigs(downloadedConfigs, opts.downloadOptionsShared, fs)
 }
 
+func escapeGoTemplating(c *config.Config) error {
+	content, err := c.Template.Content()
+	if err != nil {
+		return err
+	}
+
+	content = string(template.EscapeGoTemplating([]byte(content)))
+
+	err = c.Template.UpdateContent(content)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 type downloadFn struct {
 	classicDownload      func(client.ConfigClient, string, api.APIs, classic.ContentFilters) (projectv2.ConfigsPerType, error)
 	settingsDownload     func(client.SettingsClient, string, settings.Filters, ...config.SettingsType) (projectv2.ConfigsPerType, error)

cmd/monaco/download/download_integration_test.go

@@ -1099,6 +1099,66 @@ func TestDownloadIntegrationDownloadsAPIsAndSettings(t *testing.T) {
 	assert.Equal(t, len(configs["settings-schema"]), 3, "Expected 3 settings objects")
 }
 
+func TestDownloadGoTemplateExpressionsAreEscaped(t *testing.T) {
+	// GIVEN apis, server responses, file system
+	const projectName = "integration-test-go-templating-expressions-are-escaped"
+	const testBasePath = "test-resources/" + projectName
+
+	// Responses
+	responses := map[string]string{
+		"/platform/classic/environment-api/v2/settings/schemas": "settings/__SCHEMAS.json",
+		"/platform/classic/environment-api/v2/settings/objects": "settings/objects.json",
+	}
+
+	// Server
+	server := dtclient.NewIntegrationTestServer(t, testBasePath, responses)
+
+	fs := afero.NewMemMapFs()
+
+	opts := setupTestingDownloadOptions(t, server, projectName)
+	opts.onlySettings = false
+	opts.onlyAPIs = false
+
+	configClient, err := dtclient.NewClassicConfigClientForTesting(server.URL, server.Client())
+	require.NoError(t, err)
+
+	settingsClient, err := dtclient.NewPlatformSettingsClientForTesting(server.URL, server.Client())
+	require.NoError(t, err)
+
+	err = doDownloadConfigs(fs, &client.ClientSet{ConfigClient: configClient, SettingsClient: settingsClient}, api.APIs{}, opts)
+	assert.NoError(t, err)
+
+	// THEN we can load the project again and verify its content
+	projects, errs := loadDownloadedProjects(fs, api.APIs{})
+	if len(errs) != 0 {
+		for _, err := range errs {
+			t.Errorf("%v", err)
+		}
+		return
+	}
+
+	assert.Len(t, projects, 1)
+	p := projects[0]
+	assert.Equal(t, p.Id, projectName)
+	assert.Len(t, p.Configs, 1)
+
+	configsPerType, found := p.Configs[projectName]
+	assert.True(t, found)
+	assert.Equal(t, len(configsPerType), 1, "Expected one Settings schema to be downloaded")
+
+	settingsDownloaded, f := configsPerType["settings-schema"]
+	assert.True(t, f)
+	assert.Len(t, settingsDownloaded, 1, "Expected 1 settings object")
+
+	obj := settingsDownloaded[0]
+	content, err := obj.Template.Content()
+
+	assert.JSONEq(t, "{"+
+		"\"name\": \"SettingsTest-1\","+
+		"\"DQL\": \"fetch bizevents | FILTER like(event.type,\\\"platform.LoginEvent%\\\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{`{{`}}CountryIso, alias:countryIso}, {Country, alias:country{{`}}`}} | sort quantity desc\""+
+		"}", content)
+}
+
 func TestDownloadIntegrationDownloadsOnlyAPIsIfConfigured(t *testing.T) {
 
 	// GIVEN apis, server responses, file system

cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/__SCHEMAS.json

@@ -0,0 +1,8 @@
+{
+    "totalCount": 1,
+    "items": [
+        {
+            "schemaId": "settings-schema"
+        }
+    ]
+}

cmd/monaco/download/test-resources/integration-test-go-templating-expressions-are-escaped/settings/objects.json

@@ -0,0 +1,15 @@
+{
+    "items": [
+        {
+            "externalId": "abc",
+            "schemaVersion": "1",
+            "schemaId": "settings-schema",
+            "objectId": "so_1",
+            "scope": "test",
+            "value": {
+                "name": "SettingsTest-1",
+                "DQL": "fetch bizevents | FILTER like(event.type,\"platform.LoginEvent%\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{CountryIso, alias:countryIso}, {Country, alias:country}} | sort quantity desc"
+            }
+        }
+    ]
+}

pkg/download/automation/internal/jinja.go → internal/template/escape.go

@@ -1,6 +1,6 @@
 /*
  * @license
- * Copyright 2023 Dynatrace LLC
+ * Copyright 2025 Dynatrace LLC
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 
-package internal
+package template
 
 import "bytes"
 
-// EscapeJinjaTemplates replaces each occurrence of "{{" with "{{`{{`}}" and each occurrence of "}}" with "{{`}}`}}"
-func EscapeJinjaTemplates(src []byte) []byte {
+// EscapeGoTemplating replaces each occurrence of "{{" with "{{`{{`}}" and each occurrence of "}}" with "{{`}}`}}".
+// This is used for both Jinja expressions that resources like Workflows use, but also occurrences in DQL queries and other places.
+func EscapeGoTemplating(src []byte) []byte {
 	src = bytes.ReplaceAll(src, []byte("{{"), []byte("{{`{{`")) // replace is divided in 2 steps to avoid replacing of closing brackets in the next step
 	src = bytes.ReplaceAll(src, []byte("}}"), []byte("{{`}}`}}"))
 	src = bytes.ReplaceAll(src, []byte("{{`{{`"), []byte("{{`{{`}}"))

pkg/download/automation/internal/jinja_test.go → internal/template/escape_test.go

@@ -2,7 +2,7 @@
 
 /*
  * @license
- * Copyright 2023 Dynatrace LLC
+ * Copyright 2025 Dynatrace LLC
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -16,16 +16,18 @@
  * limitations under the License.
  */
 
-package internal
+package template
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
-func TestEscapeJinjaTemplates(t *testing.T) {
+func TestEscapeGoTemplating(t *testing.T) {
 	tc := []struct {
 		expected, in string
+		name         string
 	}{
 		{
 			in:       `Hello, {{planet}}!`,
@@ -66,11 +68,15 @@ func TestEscapeJinjaTemplates(t *testing.T) {
 			in:       `{{ }}`,
 			expected: "{{`{{`}} {{`}}`}}",
 		},
+		{
+			in:       "fetch bizevents | FILTER `event.provider` == $MyVariable | FILTER like(event.type,\\\"platform.LoginEvent%\\\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{CountryIso, alias:countryIso}, {Country, alias:country}} | sort quantity desc",
+			expected: "fetch bizevents | FILTER `event.provider` == $MyVariable | FILTER like(event.type,\\\"platform.LoginEvent%\\\") | FIELDS CountryIso, Country | SUMMARIZE quantity = toDouble(count()), by:{{`{{`}}CountryIso, alias:countryIso}, {Country, alias:country{{`}}`}} | sort quantity desc",
+		},
 	}
 
 	for _, tt := range tc {
 		t.Run(tt.in, func(t *testing.T) {
-			out := EscapeJinjaTemplates([]byte(tt.in))
+			out := EscapeGoTemplating([]byte(tt.in))
 
 			assert.Equal(t, tt.expected, string(out))
 		})

pkg/download/automation/download.go

@@ -21,22 +21,24 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"time"
+
+	"golang.org/x/exp/maps"
+
 	automationAPI "github.com/dynatrace/dynatrace-configuration-as-code-core/api/clients/automation"
 	"github.com/dynatrace/dynatrace-configuration-as-code-core/clients/automation"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/automationutils"
 	jsonutils "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/json"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/log/field"
+	templateEscaper "github.com/dynatrace/dynatrace-configuration-as-code/v2/internal/template"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/client"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/coordinate"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/parameter"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/parameter/value"
 	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/config/template"
-	"github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/download/automation/internal"
 	v2 "github.com/dynatrace/dynatrace-configuration-as-code/v2/pkg/project/v2"
-	"golang.org/x/exp/maps"
-	"time"
 )
 
 var automationTypesToResources = map[config.AutomationType]automationAPI.ResourceType{
@@ -127,7 +129,7 @@ func Download(cl client.AutomationClient, projectName string, automationTypes ..
 func escapeJinjaTemplates(src []byte) ([]byte, error) {
 	var prettyJSON bytes.Buffer
 	err := json.Indent(&prettyJSON, src, "", "\t")
-	return internal.EscapeJinjaTemplates(prettyJSON.Bytes()), err
+	return templateEscaper.EscapeGoTemplating(prettyJSON.Bytes()), err
 }
 
 func createTemplateFromRawJSON(obj automationutils.Response, configType, projectName string) (t template.Template, extractedName *string) {

pkg/project/v2/project.go

@@ -134,3 +134,14 @@ func (cpt ConfigsPerType) ForEveryConfigDo(actions ...ActionOverConfig) {
 		}
 	}
 }
+
+// AllConfigs is an iterator iterating over all configs
+func (cpt ConfigsPerType) AllConfigs(yield func(config.Config) bool) {
+	for _, cs := range cpt {
+		for _, c := range cs {
+			if !yield(c) {
+				return
+			}
+		}
+	}
+}

pkg/project/v2/project_test.go

@@ -218,3 +218,40 @@ func TestProject_ForEveryConfigDo(t *testing.T) {
 		assert.Contains(t, actual, "config4")
 	})
 }
+
+func TestConfigsPerType_AllConfigs(t *testing.T) {
+
+	var (
+		c1 = config.Config{Coordinate: coordinate.Coordinate{Project: "project1", Type: "type1", ConfigId: "config1"}}
+		c2 = config.Config{Coordinate: coordinate.Coordinate{Project: "project1", Type: "type1", ConfigId: "config2"}}
+		c3 = config.Config{Coordinate: coordinate.Coordinate{Project: "project1", Type: "type2", ConfigId: "config1"}}
+	)
+
+	cpt := project.ConfigsPerType{
+		"type1": {c1, c2},
+		"type2": {c3},
+	}
+
+	t.Run("All items are yielded", func(t *testing.T) {
+		var result []config.Config
+		for c := range cpt.AllConfigs {
+			result = append(result, c)
+		}
+
+		assert.ElementsMatch(t, []config.Config{c1, c2, c3}, result)
+	})
+
+	t.Run("Returning after the second item does no longer iterate", func(t *testing.T) {
+		var result []config.Config
+		for c := range cpt.AllConfigs {
+			result = append(result, c)
+
+			if len(result) == 2 {
+				break
+			}
+		}
+
+		assert.Len(t, result, 2)
+	})
+
+}