Generate config Go code from schema

Makefile

@@ -86,6 +86,7 @@ gofmt:
 gotidy:
 	@$(MAKE) for-all-target TARGET="tidy"
 
+# TODO: Make sure the CI fails if the config.go code is out of date.
 .PHONY: gogenerate
 gogenerate:
 	cd cmd/mdatagen && $(GOCMD) install .

cmd/mdatagen/configgen.go

@@ -0,0 +1,147 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/atombender/go-jsonschema/pkg/generator"
+	"github.com/atombender/go-jsonschema/pkg/schemas"
+)
+
+const (
+	//TODO: Get rid of this?
+	CONFIG_NAME = "config"
+)
+
+// GenerateConfig generates a "config.go", as well as any other Go files which "config.go" depends on.
+// The inputs are:
+// * "goPkgName" is the Go package at the top of the "config.go" file. For example, "batchprocessor".
+// * "dir" is the location where the "config.go" file will be written. For example, "./processor/batchprocessor".
+// * "conf" is the schema for "config.go". It is a "map[string]any".
+//
+// The output is a map, where:
+// * The key is the absolute path to the file which must be written.
+// * The value is the content of the file.
+func GenerateConfig(goPkgName string, dir string, conf any) (map[string]string, error) {
+	// load config
+	jsonBytes, err := json.Marshal(conf)
+	if err != nil {
+		return nil, fmt.Errorf("failed loading config %w", err)
+	}
+	var schema schemas.Schema
+	if err := json.Unmarshal(jsonBytes, &schema); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
+	}
+
+	// defaultOutputName := "config.go"
+	defaultOutputDir, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get absolute path for %s: %w", dir, err)
+	}
+	defaultOutputNameDest := filepath.Join(defaultOutputDir, "config.go")
+
+	// TODO: Make this configurable?
+	repoRootDir := "../../"
+
+	// TODO: Make this configurable. Or find a way to get rid of this mapping?
+	schemaMappings := []generator.SchemaMapping{
+		{
+			SchemaID:    "opentelemetry.io/collector/exporter/exporterhelper/queue_sender",
+			PackageName: "go.opentelemetry.io/collector/exporter/exporterhelper",
+			OutputName:  "./exporter/exporterhelper/queue_sender.go",
+		},
+		{
+			SchemaID:    "opentelemetry.io/collector/config/configretry/backoff/retry_on_failure",
+			PackageName: "go.opentelemetry.io/collector/config/configretry",
+			OutputName:  "./config/configretry/backoff.go",
+		},
+		{
+			SchemaID:    "opentelemetry.io/collector/config/configtelemetry/configtelemetry",
+			PackageName: "go.opentelemetry.io/collector/config/configtelemetry",
+			OutputName:  "./config/configtelemetry/configtelemetry.go",
+		},
+		{
+			SchemaID:    "opentelemetry.io/collector/config/confighttp/confighttp",
+			PackageName: "go.opentelemetry.io/collector/config/confighttp",
+			OutputName:  "./config/confighttp/confighttp.go",
+		},
+		{
+			SchemaID:    "opentelemetry.io/collector/exporter/exporterhelper/timeout_sender",
+			PackageName: "go.opentelemetry.io/collector/exporter/exporterhelper",
+			OutputName:  "./exporter/exporterhelper/timeout_sender.go",
+		},
+		{
+			SchemaID:    "opentelemetry.io/collector/config/configtls/configtls",
+			PackageName: "go.opentelemetry.io/collector/config/configtls",
+			OutputName:  "./config/configtls/configtls.go",
+		},
+		{
+			SchemaID:    "opentelemetry.io/collector/config/configcompression/configcompression",
+			PackageName: "go.opentelemetry.io/collector/config/configcompression",
+			OutputName:  "./config/configcompression/compressiontype.go",
+		},
+		{
+			SchemaID:    "opentelemetry.io/collector/config/configauth/configauth",
+			PackageName: "go.opentelemetry.io/collector/config/configauth",
+			OutputName:  "./config/configauth/configauth.go",
+		},
+		// {
+		// 	SchemaID:    "opentelemetry.io/collector/config/configopaque",
+		// 	PackageName: "go.opentelemetry.io/collector/config/configopaque",
+		// 	OutputName:  "./config/configopaque/opaque.go",
+		// },
+	}
+	for i := range schemaMappings {
+		if schemaMappings[i].OutputName != "" {
+			// The file paths in the schema mappings are relative to the repo root.
+			// Make the paths absolute.
+			relFilePath := filepath.Clean(filepath.Join(repoRootDir, schemaMappings[i].OutputName))
+			absFilePath, err := filepath.Abs(relFilePath)
+			absFilePath = filepath.Clean(absFilePath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get absolute path for %s: %w", schemaMappings[i].OutputName, err)
+			}
+			schemaMappings[i].OutputName = absFilePath
+		}
+	}
+
+	cfg := generator.Config{
+		Warner: func(message string) {
+			logf("Warning: %s", message)
+		},
+		DefaultPackageName:  goPkgName,
+		DefaultOutputName:   defaultOutputNameDest,
+		StructNameFromTitle: true,
+		Tags:                []string{"mapstructure"},
+		SchemaMappings:      schemaMappings,
+		YAMLExtensions:      []string{".yaml", ".yml"},
+		DisableOmitempty:    true,
+	}
+
+	generator, err := generator.New(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create generator: %w", err)
+	}
+	if err = generator.AddFile(CONFIG_NAME, &schema); err != nil {
+		return nil, fmt.Errorf("failed to add config: %w", err)
+	}
+
+	output := make(map[string]string)
+	for sourceName, source := range generator.Sources() {
+		fmt.Printf("Writing to %s\n", sourceName)
+		output[sourceName] = string(source)
+	}
+	fmt.Println("done")
+	return output, nil
+}
+
+func logf(format string, args ...interface{}) {
+	fmt.Fprint(os.Stderr, "go-jsonschema: ")
+	fmt.Fprintf(os.Stderr, format, args...)
+	fmt.Fprint(os.Stderr, "\n")
+}

cmd/mdatagen/configgen_test.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestJsonSchema(t *testing.T) {
+	inputDir := `./testdata/config_gen/input_schema`
+	outputDir := `./testdata/config_gen/expected_golang_output/`
+
+	inputFiles, err := os.ReadDir(inputDir)
+	require.NoError(t, err)
+
+	for _, inputFile := range inputFiles {
+		if inputFile.IsDir() {
+			continue
+		}
+
+		md, err := loadMetadata(filepath.Join(inputDir, inputFile.Name()))
+		require.NoError(t, err)
+
+		generatedConfigs, err := GenerateConfig("test_pkg", "test_dir", md.Config)
+		require.NoError(t, err)
+
+		expectedOutputFile := filepath.Join(outputDir, inputFile.Name())
+		var expectedOutput []byte
+		expectedOutput, err = os.ReadFile(expectedOutputFile)
+		require.NoError(t, err)
+
+		require.Equal(t, 1, len(generatedConfigs))
+
+		for _, fileContents := range generatedConfigs {
+			require.Equal(t, string(expectedOutput), fileContents)
+			//TODO: Also check the filename (the key in the map).
+		}
+	}
+}

cmd/mdatagen/go.mod

@@ -1,6 +1,8 @@
 module go.opentelemetry.io/collector/cmd/mdatagen
 
-go 1.21.0
+go 1.22
+
+toolchain go1.22.5
 
 require (
 	github.com/google/go-cmp v0.6.0
@@ -24,24 +26,38 @@ require (
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/sanity-io/litter v1.5.5 // indirect
+	github.com/sosodev/duration v1.3.1 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+)
+
+require (
+	github.com/atombender/go-jsonschema v0.16.0
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
+	github.com/goccy/go-yaml v1.12.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
 	github.com/knadh/koanf/v2 v2.1.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
@@ -55,6 +71,7 @@ require (
 	go.opentelemetry.io/otel/exporters/prometheus v0.50.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.28.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
@@ -100,3 +117,5 @@ replace go.opentelemetry.io/collector/internal/globalgates => ../../internal/glo
 replace go.opentelemetry.io/collector/consumer/consumerprofiles => ../../consumer/consumerprofiles
 
 replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest
+
+replace github.com/atombender/go-jsonschema => github.com/ptodev/go-jsonschema v0.0.0-20240919102244-759f18779eb1

cmd/mdatagen/loader.go

@@ -258,6 +258,8 @@ type metadata struct {
 	ScopeName string `mapstructure:"scope_name"`
 	// ShortFolderName is the shortened folder name of the component, removing class if present
 	ShortFolderName string `mapstructure:"-"`
+	// Config is the component configuration.
+	Config any `mapstructure:"config"`
 	// Tests is the set of tests generated with the component
 	Tests tests `mapstructure:"tests"`
 }