From b3a0e9970e35ef063b57af6d873be18fc8042224 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Thu, 19 Sep 2024 13:05:57 +0100 Subject: [PATCH 1/2] Generate config Go code from schema --- Makefile | 1 + cmd/mdatagen/configgen.go | 147 ++++ cmd/mdatagen/configgen_test.go | 41 ++ cmd/mdatagen/go.mod | 21 +- cmd/mdatagen/go.sum | 40 ++ cmd/mdatagen/loader.go | 2 + cmd/mdatagen/main.go | 25 + .../config_gen/expected_golang_output/batch | 79 ++ .../testdata/config_gen/input_schema/batch | 86 +++ config/configauth/configauth.go | 74 +- config/configauth/configauth.yaml | 14 + config/configauth/configauth_test.go | 8 +- config/configauth/configauth_util.go | 48 ++ config/configcompression/compressiontype.go | 66 +- config/configcompression/compressiontype.yaml | 13 + .../configcompression/compressiontype_test.go | 4 +- .../compressiontype_utils.go | 10 + config/configgrpc/configgrpc_test.go | 52 +- config/confighttp/compression.go | 3 + config/confighttp/confighttp.go | 680 ++++++------------ config/confighttp/confighttp.yaml | 146 ++++ config/confighttp/confighttp_test.go | 156 ++-- config/confighttp/confighttp_util.go | 338 +++++++++ config/confighttp/go.sum | 2 + config/configopaque/schema.yaml | 6 + config/configretry/backoff.go | 124 ++-- config/configretry/backoff.yaml | 27 + config/configtls/configtls.go | 543 ++++---------- config/configtls/configtls.yaml | 87 +++ config/configtls/configtls_test.go | 92 +-- config/configtls/configtls_util.go | 334 +++++++++ exporter/exporterhelper/queue_sender.go | 154 +--- exporter/exporterhelper/queue_sender.yaml | 19 + exporter/otlphttpexporter/config.go | 134 ++-- exporter/otlphttpexporter/config_test.go | 26 +- exporter/otlphttpexporter/factory_test.go | 4 +- .../generated_component_test.go | 1 - .../generated_package_test.go | 3 +- exporter/otlphttpexporter/metadata.yaml | 32 +- exporter/otlphttpexporter/otlp.go | 16 +- go.mod | 2 + processor/batchprocessor/config.go | 106 +-- processor/batchprocessor/config_helper.go | 47 ++ processor/batchprocessor/config_test.go | 1 + processor/batchprocessor/factory.go | 19 - .../generated_component_test.go | 1 - .../batchprocessor/generated_package_test.go | 3 +- processor/batchprocessor/metadata.yaml | 34 + 48 files changed, 2455 insertions(+), 1416 deletions(-) create mode 100644 cmd/mdatagen/configgen.go create mode 100644 cmd/mdatagen/configgen_test.go create mode 100644 cmd/mdatagen/testdata/config_gen/expected_golang_output/batch create mode 100644 cmd/mdatagen/testdata/config_gen/input_schema/batch create mode 100644 config/configauth/configauth.yaml create mode 100644 config/configauth/configauth_util.go create mode 100644 config/configcompression/compressiontype.yaml create mode 100644 config/configcompression/compressiontype_utils.go create mode 100644 config/confighttp/confighttp.yaml create mode 100644 config/confighttp/confighttp_util.go create mode 100644 config/configopaque/schema.yaml create mode 100644 config/configretry/backoff.yaml create mode 100644 config/configtls/configtls.yaml create mode 100644 config/configtls/configtls_util.go create mode 100644 exporter/exporterhelper/queue_sender.yaml create mode 100644 processor/batchprocessor/config_helper.go diff --git a/Makefile b/Makefile index 052cbcd5d81..1b58b12c720 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,7 @@ gofmt: gotidy: @$(MAKE) for-all-target TARGET="tidy" +# TODO: Make sure the CI fails if the config.go code is out of date. .PHONY: gogenerate gogenerate: cd cmd/mdatagen && $(GOCMD) install . diff --git a/cmd/mdatagen/configgen.go b/cmd/mdatagen/configgen.go new file mode 100644 index 00000000000..6955388e86c --- /dev/null +++ b/cmd/mdatagen/configgen.go @@ -0,0 +1,147 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/atombender/go-jsonschema/pkg/generator" + "github.com/atombender/go-jsonschema/pkg/schemas" +) + +const ( + //TODO: Get rid of this? + CONFIG_NAME = "config" +) + +// GenerateConfig generates a "config.go", as well as any other Go files which "config.go" depends on. +// The inputs are: +// * "goPkgName" is the Go package at the top of the "config.go" file. For example, "batchprocessor". +// * "dir" is the location where the "config.go" file will be written. For example, "./processor/batchprocessor". +// * "conf" is the schema for "config.go". It is a "map[string]any". +// +// The output is a map, where: +// * The key is the absolute path to the file which must be written. +// * The value is the content of the file. +func GenerateConfig(goPkgName string, dir string, conf any) (map[string]string, error) { + // load config + jsonBytes, err := json.Marshal(conf) + if err != nil { + return nil, fmt.Errorf("failed loading config %w", err) + } + var schema schemas.Schema + if err := json.Unmarshal(jsonBytes, &schema); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) + } + + // defaultOutputName := "config.go" + defaultOutputDir, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for %s: %w", dir, err) + } + defaultOutputNameDest := filepath.Join(defaultOutputDir, "config.go") + + // TODO: Make this configurable? + repoRootDir := "../../" + + // TODO: Make this configurable. Or find a way to get rid of this mapping? + schemaMappings := []generator.SchemaMapping{ + { + SchemaID: "opentelemetry.io/collector/exporter/exporterhelper/queue_sender", + PackageName: "go.opentelemetry.io/collector/exporter/exporterhelper", + OutputName: "./exporter/exporterhelper/queue_sender.go", + }, + { + SchemaID: "opentelemetry.io/collector/config/configretry/backoff/retry_on_failure", + PackageName: "go.opentelemetry.io/collector/config/configretry", + OutputName: "./config/configretry/backoff.go", + }, + { + SchemaID: "opentelemetry.io/collector/config/configtelemetry/configtelemetry", + PackageName: "go.opentelemetry.io/collector/config/configtelemetry", + OutputName: "./config/configtelemetry/configtelemetry.go", + }, + { + SchemaID: "opentelemetry.io/collector/config/confighttp/confighttp", + PackageName: "go.opentelemetry.io/collector/config/confighttp", + OutputName: "./config/confighttp/confighttp.go", + }, + { + SchemaID: "opentelemetry.io/collector/exporter/exporterhelper/timeout_sender", + PackageName: "go.opentelemetry.io/collector/exporter/exporterhelper", + OutputName: "./exporter/exporterhelper/timeout_sender.go", + }, + { + SchemaID: "opentelemetry.io/collector/config/configtls/configtls", + PackageName: "go.opentelemetry.io/collector/config/configtls", + OutputName: "./config/configtls/configtls.go", + }, + { + SchemaID: "opentelemetry.io/collector/config/configcompression/configcompression", + PackageName: "go.opentelemetry.io/collector/config/configcompression", + OutputName: "./config/configcompression/compressiontype.go", + }, + { + SchemaID: "opentelemetry.io/collector/config/configauth/configauth", + PackageName: "go.opentelemetry.io/collector/config/configauth", + OutputName: "./config/configauth/configauth.go", + }, + // { + // SchemaID: "opentelemetry.io/collector/config/configopaque", + // PackageName: "go.opentelemetry.io/collector/config/configopaque", + // OutputName: "./config/configopaque/opaque.go", + // }, + } + for i := range schemaMappings { + if schemaMappings[i].OutputName != "" { + // The file paths in the schema mappings are relative to the repo root. + // Make the paths absolute. + relFilePath := filepath.Clean(filepath.Join(repoRootDir, schemaMappings[i].OutputName)) + absFilePath, err := filepath.Abs(relFilePath) + absFilePath = filepath.Clean(absFilePath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for %s: %w", schemaMappings[i].OutputName, err) + } + schemaMappings[i].OutputName = absFilePath + } + } + + cfg := generator.Config{ + Warner: func(message string) { + logf("Warning: %s", message) + }, + DefaultPackageName: goPkgName, + DefaultOutputName: defaultOutputNameDest, + StructNameFromTitle: true, + Tags: []string{"mapstructure"}, + SchemaMappings: schemaMappings, + YAMLExtensions: []string{".yaml", ".yml"}, + DisableOmitempty: true, + } + + generator, err := generator.New(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create generator: %w", err) + } + if err = generator.AddFile(CONFIG_NAME, &schema); err != nil { + return nil, fmt.Errorf("failed to add config: %w", err) + } + + output := make(map[string]string) + for sourceName, source := range generator.Sources() { + fmt.Printf("Writing to %s\n", sourceName) + output[sourceName] = string(source) + } + fmt.Println("done") + return output, nil +} + +func logf(format string, args ...interface{}) { + fmt.Fprint(os.Stderr, "go-jsonschema: ") + fmt.Fprintf(os.Stderr, format, args...) + fmt.Fprint(os.Stderr, "\n") +} diff --git a/cmd/mdatagen/configgen_test.go b/cmd/mdatagen/configgen_test.go new file mode 100644 index 00000000000..5483a98245a --- /dev/null +++ b/cmd/mdatagen/configgen_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestJsonSchema(t *testing.T) { + inputDir := `./testdata/config_gen/input_schema` + outputDir := `./testdata/config_gen/expected_golang_output/` + + inputFiles, err := os.ReadDir(inputDir) + require.NoError(t, err) + + for _, inputFile := range inputFiles { + if inputFile.IsDir() { + continue + } + + md, err := loadMetadata(filepath.Join(inputDir, inputFile.Name())) + require.NoError(t, err) + + generatedConfigs, err := GenerateConfig("test_pkg", "test_dir", md.Config) + require.NoError(t, err) + + expectedOutputFile := filepath.Join(outputDir, inputFile.Name()) + var expectedOutput []byte + expectedOutput, err = os.ReadFile(expectedOutputFile) + require.NoError(t, err) + + require.Equal(t, 1, len(generatedConfigs)) + + for _, fileContents := range generatedConfigs { + require.Equal(t, string(expectedOutput), fileContents) + //TODO: Also check the filename (the key in the map). + } + } +} diff --git a/cmd/mdatagen/go.mod b/cmd/mdatagen/go.mod index fae0b9393a2..928cefba5d8 100644 --- a/cmd/mdatagen/go.mod +++ b/cmd/mdatagen/go.mod @@ -1,6 +1,8 @@ module go.opentelemetry.io/collector/cmd/mdatagen -go 1.21.0 +go 1.22 + +toolchain go1.22.5 require ( github.com/google/go-cmp v0.6.0 @@ -24,12 +26,22 @@ require ( ) require ( + dario.cat/mergo v1.0.0 // indirect + github.com/sanity-io/litter v1.5.5 // indirect + github.com/sosodev/duration v1.3.1 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect +) + +require ( + github.com/atombender/go-jsonschema v0.16.0 github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.0.0 // indirect + github.com/goccy/go-yaml v1.12.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect @@ -37,11 +49,15 @@ require ( github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -55,6 +71,7 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.50.0 // indirect go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.22.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect @@ -100,3 +117,5 @@ replace go.opentelemetry.io/collector/internal/globalgates => ../../internal/glo replace go.opentelemetry.io/collector/consumer/consumerprofiles => ../../consumer/consumerprofiles replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest + +replace github.com/atombender/go-jsonschema => github.com/ptodev/go-jsonschema v0.0.0-20240919102244-759f18779eb1 diff --git a/cmd/mdatagen/go.sum b/cmd/mdatagen/go.sum index 819ce91ab81..697e8e1ee5d 100644 --- a/cmd/mdatagen/go.sum +++ b/cmd/mdatagen/go.sum @@ -1,17 +1,30 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAprG2mQfMfc= github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM= +github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -35,8 +48,17 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -46,6 +68,9 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= @@ -56,9 +81,16 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/ptodev/go-jsonschema v0.0.0-20240919102244-759f18779eb1 h1:v+2BtYN3SkcLg6QbZhjtBLhoNAlO2daM4EO0ThjqAas= +github.com/ptodev/go-jsonschema v0.0.0-20240919102244-759f18779eb1/go.mod h1:Dklh87r+yTgyHDZSRdfDl03KcaOSFkb5Hh8fbYCs97g= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= +github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= +github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -85,6 +117,10 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -99,6 +135,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -113,6 +151,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= diff --git a/cmd/mdatagen/loader.go b/cmd/mdatagen/loader.go index ac216d9e681..2237bd3a52f 100644 --- a/cmd/mdatagen/loader.go +++ b/cmd/mdatagen/loader.go @@ -258,6 +258,8 @@ type metadata struct { ScopeName string `mapstructure:"scope_name"` // ShortFolderName is the shortened folder name of the component, removing class if present ShortFolderName string `mapstructure:"-"` + // Config is the component configuration. + Config any `mapstructure:"config"` // Tests is the set of tests generated with the component Tests tests `mapstructure:"tests"` } diff --git a/cmd/mdatagen/main.go b/cmd/mdatagen/main.go index cc325376826..1b01704337d 100644 --- a/cmd/mdatagen/main.go +++ b/cmd/mdatagen/main.go @@ -52,6 +52,13 @@ func run(ymlPath string) error { return fmt.Errorf("failed loading %v: %w", ymlPath, err) } + if md.Config != nil { + err = generateConfigGoCode(ymlDir, packageName, md.Config) + if err != nil { + return err + } + } + tmplDir := "templates" codeDir := filepath.Join(ymlDir, "internal", "metadata") @@ -139,6 +146,24 @@ func run(ymlPath string) error { return nil } +func generateConfigGoCode(dir string, goPkgName string, conf any) error { + output, err := GenerateConfig(goPkgName, dir, conf) + if err != nil { + return fmt.Errorf("error generating config %w", err) + } + + for fileName, fileContents := range output { + generatedCfgFile, err := os.Create(fileName) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + generatedCfgFile.Write([]byte(fileContents)) + generatedCfgFile.Close() + } + + return nil +} + func templatize(tmplFile string, md metadata) *template.Template { return template.Must( template. diff --git a/cmd/mdatagen/testdata/config_gen/expected_golang_output/batch b/cmd/mdatagen/testdata/config_gen/expected_golang_output/batch new file mode 100644 index 00000000000..cc70ffc9a3c --- /dev/null +++ b/cmd/mdatagen/testdata/config_gen/expected_golang_output/batch @@ -0,0 +1,79 @@ +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. + +package test_pkg + +import "encoding/json" +import "fmt" +import "time" + +// Configuration parameters for the batch processor. +type Config struct { + // MetadataCardinalityLimit corresponds to the JSON schema field + // "metadata_cardinality_limit". + MetadataCardinalityLimit int `mapstructure:"metadata_cardinality_limit"` + + // MetadataKeys corresponds to the JSON schema field "metadata_keys". + MetadataKeys []string `mapstructure:"metadata_keys"` + + // PaulinTest corresponds to the JSON schema field "paulin_test". + PaulinTest string `mapstructure:"paulin_test"` + + // PaulinTest2 corresponds to the JSON schema field "paulin_test2". + PaulinTest2 *float64 `mapstructure:"paulin_test2"` + + // PaulinTest3 corresponds to the JSON schema field "paulin_test3". + PaulinTest3 *bool `mapstructure:"paulin_test3"` + + // SendBatchMaxSize corresponds to the JSON schema field "send_batch_max_size". + SendBatchMaxSize int `mapstructure:"send_batch_max_size"` + + // SendBatchSize corresponds to the JSON schema field "send_batch_size". + SendBatchSize int `mapstructure:"send_batch_size"` + + // Timeout corresponds to the JSON schema field "timeout". + Timeout time.Duration `mapstructure:"timeout"` +} + +// UnmarshalJSON implements json.Unmarshaler. +func (j *Config) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err + } + type Plain Config + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err + } + if v, ok := raw["metadata_cardinality_limit"]; !ok || v == nil { + plain.MetadataCardinalityLimit = 1000.0 + } + if v, ok := raw["paulin_test"]; !ok || v == nil { + plain.PaulinTest = "test" + } + if v, ok := raw["send_batch_max_size"]; !ok || v == nil { + plain.SendBatchMaxSize = 0.0 + } + if v, ok := raw["send_batch_size"]; !ok || v == nil { + plain.SendBatchSize = 8192.0 + } + if v, ok := raw["timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") + if err != nil { + return fmt.Errorf("failed to parse the \"33.3s\" default value for field timeout:%w }", err) + } + plain.Timeout = defaultDuration + } + *j = Config(plain) + return nil +} + +// SetDefaults sets the fields of Config to their defaults. +// Fields which do not have a default value are left untouched. +func (c *Config) SetDefaults() { + c.MetadataCardinalityLimit = 1000.0 + c.PaulinTest = "test" + c.SendBatchMaxSize = 0.0 + c.SendBatchSize = 8192.0 + c.Timeout = "PT33.3S" +} diff --git a/cmd/mdatagen/testdata/config_gen/input_schema/batch b/cmd/mdatagen/testdata/config_gen/input_schema/batch new file mode 100644 index 00000000000..9d10186f3e4 --- /dev/null +++ b/cmd/mdatagen/testdata/config_gen/input_schema/batch @@ -0,0 +1,86 @@ +type: batch + +status: + class: processor + stability: + beta: [traces, metrics, logs] + distributions: [core, contrib, k8s] + +tests: + +telemetry: + level: normal + metrics: + processor_batch_batch_size_trigger_send: + enabled: true + description: Number of times the batch was sent due to a size trigger + unit: "1" + sum: + value_type: int + monotonic: true + processor_batch_timeout_trigger_send: + enabled: true + description: Number of times the batch was sent due to a timeout trigger + unit: "1" + sum: + value_type: int + monotonic: true + processor_batch_batch_send_size: + enabled: true + description: Number of units in the batch + unit: "1" + histogram: + value_type: int + bucket_boundaries: [10, 25, 50, 75, 100, 250, 500, 750, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 20000, 30000, 50000, 100000] + processor_batch_batch_send_size_bytes: + enabled: true + description: Number of bytes in batch that was sent + unit: By + histogram: + value_type: int + bucket_boundaries: [10, 25, 50, 75, 100, 250, 500, 750, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 20000, 30000, 50000, 100_000, 200_000, 300_000, 400_000, 500_000, 600_000, 700_000, 800_000, 900_000, 1000_000, 2000_000, 3000_000, 4000_000, 5000_000, 6000_000, 7000_000, 8000_000, 9000_000] + processor_batch_metadata_cardinality: + enabled: true + description: Number of distinct metadata value combinations being processed + unit: "1" + sum: + value_type: int + async: true + +config: + type: object + additionalProperties: false + description: "Configuration parameters for the batch processor." + properties: + timeout: + type: string + format: duration + default: "PT33.3S" + description: + send_batch_size: + type: integer + minimum: 0 + default: 8192 + description: + send_batch_max_size: + type: integer + minimum: 0 + default: 0 + description: + metadata_keys: + type: array + items: + type: string + description: + metadata_cardinality_limit: + type: integer + minimum: 0 + default: 1000 + description: + paulin_test: + type: string + default: "test" + paulin_test2: + type: number + paulin_test3: + type: boolean diff --git a/config/configauth/configauth.go b/config/configauth/configauth.go index a3501cd0bed..c3a852e51a1 100644 --- a/config/configauth/configauth.go +++ b/config/configauth/configauth.go @@ -1,59 +1,35 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. -// Package configauth implements the configuration settings to -// ensure authentication on incoming requests, and allows -// exporters to add authentication on outgoing requests. -package configauth // import "go.opentelemetry.io/collector/config/configauth" +package configauth -import ( - "context" - "errors" - "fmt" +import "encoding/json" +import "fmt" +import "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/extension/auth" -) - -var ( - errAuthenticatorNotFound = errors.New("authenticator not found") - errNotClient = errors.New("requested authenticator is not a client authenticator") - errNotServer = errors.New("requested authenticator is not a server authenticator") -) - -// Authentication defines the auth settings for the receiver. type Authentication struct { - // AuthenticatorID specifies the name of the extension to use in order to authenticate the incoming data point. - AuthenticatorID component.ID `mapstructure:"authenticator"` -} - -// NewDefaultAuthentication returns a default authentication configuration. -func NewDefaultAuthentication() *Authentication { - return &Authentication{} + // Authenticator corresponds to the JSON schema field "authenticator". + Authenticator component.ID `mapstructure:"authenticator"` } -// GetServerAuthenticator attempts to select the appropriate auth.Server from the list of extensions, -// based on the requested extension name. If an authenticator is not found, an error is returned. -func (a Authentication) GetServerAuthenticator(_ context.Context, extensions map[component.ID]component.Component) (auth.Server, error) { - if ext, found := extensions[a.AuthenticatorID]; found { - if server, ok := ext.(auth.Server); ok { - return server, nil - } - return nil, errNotServer +// UnmarshalJSON implements json.Unmarshaler. +func (j *Authentication) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - - return nil, fmt.Errorf("failed to resolve authenticator %q: %w", a.AuthenticatorID, errAuthenticatorNotFound) + if _, ok := raw["authenticator"]; raw != nil && !ok { + return fmt.Errorf("field authenticator in Authentication: required") + } + type Plain Authentication + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err + } + *j = Authentication(plain) + return nil } -// GetClientAuthenticator attempts to select the appropriate auth.Client from the list of extensions, -// based on the component id of the extension. If an authenticator is not found, an error is returned. -// This should be only used by HTTP clients. -func (a Authentication) GetClientAuthenticator(_ context.Context, extensions map[component.ID]component.Component) (auth.Client, error) { - if ext, found := extensions[a.AuthenticatorID]; found { - if client, ok := ext.(auth.Client); ok { - return client, nil - } - return nil, errNotClient - } - return nil, fmt.Errorf("failed to resolve authenticator %q: %w", a.AuthenticatorID, errAuthenticatorNotFound) +// SetDefaults sets the fields of Authentication to their defaults. +// Fields which do not have a default value are left untouched. +func (c *Authentication) SetDefaults() { } diff --git a/config/configauth/configauth.yaml b/config/configauth/configauth.yaml new file mode 100644 index 00000000000..573ae9fd8d6 --- /dev/null +++ b/config/configauth/configauth.yaml @@ -0,0 +1,14 @@ +"$schema": http://json-schema.org/draft-04/schema# +id: opentelemetry.io/collector/config/configauth/configauth +$defs: + authentication: + type: object + title: Authentication + required: + - authenticator + properties: + authenticator: + type: string + goType: + name: ID + package: go.opentelemetry.io/collector/component diff --git a/config/configauth/configauth_test.go b/config/configauth/configauth_test.go index a19577f9d98..631215d2c56 100644 --- a/config/configauth/configauth_test.go +++ b/config/configauth/configauth_test.go @@ -43,7 +43,7 @@ func TestGetServer(t *testing.T) { t.Run(tC.desc, func(t *testing.T) { // prepare cfg := &Authentication{ - AuthenticatorID: mockID, + Authenticator: mockID, } ext := map[component.ID]component.Component{ mockID: tC.authenticator, @@ -65,7 +65,7 @@ func TestGetServer(t *testing.T) { func TestGetServerFails(t *testing.T) { cfg := &Authentication{ - AuthenticatorID: component.MustNewID("does_not_exist"), + Authenticator: component.MustNewID("does_not_exist"), } authenticator, err := cfg.GetServerAuthenticator(context.Background(), map[component.ID]component.Component{}) @@ -94,7 +94,7 @@ func TestGetClient(t *testing.T) { t.Run(tC.desc, func(t *testing.T) { // prepare cfg := &Authentication{ - AuthenticatorID: mockID, + Authenticator: mockID, } ext := map[component.ID]component.Component{ mockID: tC.authenticator, @@ -116,7 +116,7 @@ func TestGetClient(t *testing.T) { func TestGetClientFails(t *testing.T) { cfg := &Authentication{ - AuthenticatorID: component.MustNewID("does_not_exist"), + Authenticator: component.MustNewID("does_not_exist"), } authenticator, err := cfg.GetClientAuthenticator(context.Background(), map[component.ID]component.Component{}) assert.ErrorIs(t, err, errAuthenticatorNotFound) diff --git a/config/configauth/configauth_util.go b/config/configauth/configauth_util.go new file mode 100644 index 00000000000..e4a9b064b35 --- /dev/null +++ b/config/configauth/configauth_util.go @@ -0,0 +1,48 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package configauth implements the configuration settings to +// ensure authentication on incoming requests, and allows +// exporters to add authentication on outgoing requests. +package configauth // import "go.opentelemetry.io/collector/config/configauth" + +import ( + "context" + "errors" + "fmt" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension/auth" +) + +var ( + errAuthenticatorNotFound = errors.New("authenticator not found") + errNotClient = errors.New("requested authenticator is not a client authenticator") + errNotServer = errors.New("requested authenticator is not a server authenticator") +) + +// GetServerAuthenticator attempts to select the appropriate auth.Server from the list of extensions, +// based on the requested extension name. If an authenticator is not found, an error is returned. +func (a Authentication) GetServerAuthenticator(_ context.Context, extensions map[component.ID]component.Component) (auth.Server, error) { + if ext, found := extensions[a.Authenticator]; found { + if server, ok := ext.(auth.Server); ok { + return server, nil + } + return nil, errNotServer + } + + return nil, fmt.Errorf("failed to resolve authenticator %q: %w", a.AuthenticatorID, errAuthenticatorNotFound) +} + +// GetClientAuthenticator attempts to select the appropriate auth.Client from the list of extensions, +// based on the component id of the extension. If an authenticator is not found, an error is returned. +// This should be only used by HTTP clients. +func (a Authentication) GetClientAuthenticator(_ context.Context, extensions map[component.ID]component.Component) (auth.Client, error) { + if ext, found := extensions[a.Authenticator]; found { + if client, ok := ext.(auth.Client); ok { + return client, nil + } + return nil, errNotClient + } + return nil, fmt.Errorf("failed to resolve authenticator %q: %w", a.AuthenticatorID, errAuthenticatorNotFound) +} diff --git a/config/configcompression/compressiontype.go b/config/configcompression/compressiontype.go index 004e9558665..c5ddfbfc421 100644 --- a/config/configcompression/compressiontype.go +++ b/config/configcompression/compressiontype.go @@ -1,41 +1,47 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. -package configcompression // import "go.opentelemetry.io/collector/config/configcompression" +package configcompression +import "encoding/json" import "fmt" +import "reflect" -// Type represents a compression method type Type string -const ( - TypeGzip Type = "gzip" - TypeZlib Type = "zlib" - TypeDeflate Type = "deflate" - TypeSnappy Type = "snappy" - TypeZstd Type = "zstd" - typeNone Type = "none" - typeEmpty Type = "" -) +const TypeBlank Type = "" +const TypeDeflate Type = "deflate" +const TypeGzip Type = "gzip" +const TypeNone Type = "none" +const TypeSnappy Type = "snappy" +const TypeZlib Type = "zlib" +const TypeZstd Type = "zstd" -// IsCompressed returns false if CompressionType is nil, none, or empty. -// Otherwise, returns true. -func (ct *Type) IsCompressed() bool { - return *ct != typeEmpty && *ct != typeNone +var enumValues_Type = []interface{}{ + "gzip", + "zlib", + "deflate", + "snappy", + "zstd", + "none", + "", } -func (ct *Type) UnmarshalText(in []byte) error { - typ := Type(in) - if typ == TypeGzip || - typ == TypeZlib || - typ == TypeDeflate || - typ == TypeSnappy || - typ == TypeZstd || - typ == typeNone || - typ == typeEmpty { - *ct = typ - return nil +// UnmarshalJSON implements json.Unmarshaler. +func (j *Type) UnmarshalJSON(b []byte) error { + var v string + if err := json.Unmarshal(b, &v); err != nil { + return err } - return fmt.Errorf("unsupported compression type %q", typ) - + var ok bool + for _, expected := range enumValues_Type { + if reflect.DeepEqual(v, expected) { + ok = true + break + } + } + if !ok { + return fmt.Errorf("invalid value (expected one of %#v): %#v", enumValues_Type, v) + } + *j = Type(v) + return nil } diff --git a/config/configcompression/compressiontype.yaml b/config/configcompression/compressiontype.yaml new file mode 100644 index 00000000000..ac3745e3dcc --- /dev/null +++ b/config/configcompression/compressiontype.yaml @@ -0,0 +1,13 @@ +"$schema": http://json-schema.org/draft-04/schema# +id: opentelemetry.io/collector/config/configcompression/configcompression +title: Type +type: string +enum: +- gzip +- zlib +- deflate +- snappy +- zstd +- none +- '' +default: 'none' \ No newline at end of file diff --git a/config/configcompression/compressiontype_test.go b/config/configcompression/compressiontype_test.go index cf8166d5a24..440ff5bfdcc 100644 --- a/config/configcompression/compressiontype_test.go +++ b/config/configcompression/compressiontype_test.go @@ -65,8 +65,8 @@ func TestUnmarshalText(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - temp := typeNone - err := temp.UnmarshalText(tt.compressionName) + temp := TypeNone + err := temp.Unmarshal(tt.compressionName) if tt.shouldError { assert.Error(t, err) return diff --git a/config/configcompression/compressiontype_utils.go b/config/configcompression/compressiontype_utils.go new file mode 100644 index 00000000000..aaa8134b485 --- /dev/null +++ b/config/configcompression/compressiontype_utils.go @@ -0,0 +1,10 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package configcompression // import "go.opentelemetry.io/collector/config/configcompression" + +// IsCompressed returns false if CompressionType is nil, none, or empty. +// Otherwise, returns true. +func (ct *Type) IsCompressed() bool { + return *ct != TypeBlank && *ct != TypeNone +} diff --git a/config/configgrpc/configgrpc_test.go b/config/configgrpc/configgrpc_test.go index 1769733dea6..b8d1165a96d 100644 --- a/config/configgrpc/configgrpc_test.go +++ b/config/configgrpc/configgrpc_test.go @@ -257,7 +257,7 @@ func TestAllGrpcServerSettingsExceptAuth(t *testing.T) { }, TLSSetting: &configtls.ServerConfig{ Config: configtls.Config{}, - ClientCAFile: "", + ClientCaFile: "", }, MaxRecvMsgSizeMiB: 1, MaxConcurrentStreams: 1024, @@ -319,10 +319,10 @@ func TestGRPCClientSettingsError(t *testing.T) { Compression: "", TLSSetting: configtls.ClientConfig{ Config: configtls.Config{ - CAFile: "/doesnt/exist", + CaFile: "/doesnt/exist", }, - Insecure: false, - ServerName: "", + Insecure: false, + ServerNameOverride: "", }, Keepalive: nil, }, @@ -337,8 +337,8 @@ func TestGRPCClientSettingsError(t *testing.T) { Config: configtls.Config{ CertFile: "/doesnt/exist", }, - Insecure: false, - ServerName: "", + Insecure: false, + ServerNameOverride: "", }, Keepalive: nil, }, @@ -513,7 +513,7 @@ func TestGRPCServerSettingsError(t *testing.T) { }, TLSSetting: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: "/doesnt/exist", + CaFile: "/doesnt/exist", }, }, }, @@ -540,7 +540,7 @@ func TestGRPCServerSettingsError(t *testing.T) { Transport: confignet.TransportTypeTCP, }, TLSSetting: &configtls.ServerConfig{ - ClientCAFile: "/doesnt/exist", + ClientCaFile: "/doesnt/exist", }, }, }, @@ -586,30 +586,30 @@ func TestHttpReception(t *testing.T) { name: "TLS", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, }, { name: "NoServerCertificates", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, hasError: true, }, @@ -617,36 +617,36 @@ func TestHttpReception(t *testing.T) { name: "mTLS", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, - ClientCAFile: filepath.Join("testdata", "ca.crt"), + ClientCaFile: filepath.Join("testdata", "ca.crt"), }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "client.crt"), KeyFile: filepath.Join("testdata", "client.key"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, }, { name: "NoClientCertificate", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, - ClientCAFile: filepath.Join("testdata", "ca.crt"), + ClientCaFile: filepath.Join("testdata", "ca.crt"), }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, hasError: true, }, @@ -654,19 +654,19 @@ func TestHttpReception(t *testing.T) { name: "WrongClientCA", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, - ClientCAFile: filepath.Join("testdata", "server.crt"), + ClientCaFile: filepath.Join("testdata", "server.crt"), }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "client.crt"), KeyFile: filepath.Join("testdata", "client.key"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, hasError: true, }, diff --git a/config/confighttp/compression.go b/config/confighttp/compression.go index 4498fefe864..f085a67f3e2 100644 --- a/config/confighttp/compression.go +++ b/config/confighttp/compression.go @@ -19,12 +19,15 @@ import ( "go.opentelemetry.io/collector/config/configcompression" ) +const headerContentEncoding = "Content-Encoding" + type compressRoundTripper struct { rt http.RoundTripper compressionType configcompression.Type compressor *compressor } +// TODO: Use the new Compression enum instead of "string" var availableDecoders = map[string]func(body io.ReadCloser) (io.ReadCloser, error){ "": func(io.ReadCloser) (io.ReadCloser, error) { // Not a compressed payload. Nothing to do. diff --git a/config/confighttp/confighttp.go b/config/confighttp/confighttp.go index a62c5d7b2f5..4ddb146f82d 100644 --- a/config/confighttp/confighttp.go +++ b/config/confighttp/confighttp.go @@ -1,545 +1,311 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package confighttp // import "go.opentelemetry.io/collector/config/confighttp" - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/http/cookiejar" - "net/url" - "time" - - "github.com/rs/cors" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/otel" - "golang.org/x/net/http2" - "golang.org/x/net/publicsuffix" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/configauth" - "go.opentelemetry.io/collector/config/configcompression" - "go.opentelemetry.io/collector/config/configopaque" - "go.opentelemetry.io/collector/config/configtelemetry" - "go.opentelemetry.io/collector/config/configtls" - "go.opentelemetry.io/collector/config/internal" - "go.opentelemetry.io/collector/extension/auth" -) - -const headerContentEncoding = "Content-Encoding" -const defaultMaxRequestBodySize = 20 * 1024 * 1024 // 20MiB -var defaultCompressionAlgorithms = []string{"", "gzip", "zstd", "zlib", "snappy", "deflate"} - -// ClientConfig defines settings for creating an HTTP client. +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. + +package confighttp + +import "encoding/json" +import "fmt" +import configauth "go.opentelemetry.io/collector/config/configauth" +import configcompression "go.opentelemetry.io/collector/config/configcompression" +import "go.opentelemetry.io/collector/config/configopaque" +import configtls "go.opentelemetry.io/collector/config/configtls" +import "time" + type ClientConfig struct { - // The target URL to send data to (e.g.: http://some.url:9411/v1/traces). - Endpoint string `mapstructure:"endpoint"` + // TLSSetting corresponds to the JSON schema field "TLSSetting". + TLSSetting *configtls.ClientConfig `mapstructure:"TLSSetting"` - // ProxyURL setting for the collector - ProxyURL string `mapstructure:"proxy_url"` + // Auth corresponds to the JSON schema field "auth". + Auth *configauth.Authentication `mapstructure:"auth"` - // TLSSetting struct exposes TLS client configuration. - TLSSetting configtls.ClientConfig `mapstructure:"tls"` + // Compression corresponds to the JSON schema field "compression". + Compression configcompression.Type `mapstructure:"compression"` - // ReadBufferSize for HTTP client. See http.Transport.ReadBufferSize. - ReadBufferSize int `mapstructure:"read_buffer_size"` + // Cookies corresponds to the JSON schema field "cookies". + Cookies *ClientConfigCookies `mapstructure:"cookies"` - // WriteBufferSize for HTTP client. See http.Transport.WriteBufferSize. - WriteBufferSize int `mapstructure:"write_buffer_size"` + // DisableKeepAlives corresponds to the JSON schema field "disable_keep_alives". + DisableKeepAlives bool `mapstructure:"disable_keep_alives"` - // Timeout parameter configures `http.Client.Timeout`. - Timeout time.Duration `mapstructure:"timeout"` + // Endpoint corresponds to the JSON schema field "endpoint". + Endpoint string `mapstructure:"endpoint"` - // Additional headers attached to each HTTP request sent by the client. - // Existing header values are overwritten if collision happens. - // Header values are opaque since they may be sensitive. + // Headers corresponds to the JSON schema field "headers". Headers map[string]configopaque.String `mapstructure:"headers"` - // Auth configuration for outgoing HTTP calls. - Auth *configauth.Authentication `mapstructure:"auth"` + // Http2PingTimeout corresponds to the JSON schema field "http2_ping_timeout". + Http2PingTimeout time.Duration `mapstructure:"http2_ping_timeout"` - // The compression key for supported compression types within collector. - Compression configcompression.Type `mapstructure:"compression"` + // Http2ReadIdleTimeout corresponds to the JSON schema field + // "http2_read_idle_timeout". + Http2ReadIdleTimeout time.Duration `mapstructure:"http2_read_idle_timeout"` + + // IdleConnTimeout corresponds to the JSON schema field "idle_conn_timeout". + IdleConnTimeout *time.Duration `mapstructure:"idle_conn_timeout"` + + // MaxConnsPerHost corresponds to the JSON schema field "max_conns_per_host". + MaxConnsPerHost *int `mapstructure:"max_conns_per_host"` - // MaxIdleConns is used to set a limit to the maximum idle HTTP connections the client can keep open. - // There's an already set value, and we want to override it only if an explicit value provided + // MaxIdleConns corresponds to the JSON schema field "max_idle_conns". MaxIdleConns *int `mapstructure:"max_idle_conns"` - // MaxIdleConnsPerHost is used to set a limit to the maximum idle HTTP connections the host can keep open. - // There's an already set value, and we want to override it only if an explicit value provided + // MaxIdleConnsPerHost corresponds to the JSON schema field + // "max_idle_conns_per_host". MaxIdleConnsPerHost *int `mapstructure:"max_idle_conns_per_host"` - // MaxConnsPerHost limits the total number of connections per host, including connections in the dialing, - // active, and idle states. - // There's an already set value, and we want to override it only if an explicit value provided - MaxConnsPerHost *int `mapstructure:"max_conns_per_host"` + // ProxyUrl corresponds to the JSON schema field "proxy_url". + ProxyUrl string `mapstructure:"proxy_url"` - // IdleConnTimeout is the maximum amount of time a connection will remain open before closing itself. - // There's an already set value, and we want to override it only if an explicit value provided - IdleConnTimeout *time.Duration `mapstructure:"idle_conn_timeout"` + // ReadBufferSize corresponds to the JSON schema field "read_buffer_size". + ReadBufferSize int `mapstructure:"read_buffer_size"` - // DisableKeepAlives, if true, disables HTTP keep-alives and will only use the connection to the server - // for a single HTTP request. - // - // WARNING: enabling this option can result in significant overhead establishing a new HTTP(S) - // connection for every request. Before enabling this option please consider whether changes - // to idle connection settings can achieve your goal. - DisableKeepAlives bool `mapstructure:"disable_keep_alives"` + // Timeout corresponds to the JSON schema field "timeout". + Timeout time.Duration `mapstructure:"timeout"` - // This is needed in case you run into - // https://github.com/golang/go/issues/59690 - // https://github.com/golang/go/issues/36026 - // HTTP2ReadIdleTimeout if the connection has been idle for the configured value send a ping frame for health check - // 0s means no health check will be performed. - HTTP2ReadIdleTimeout time.Duration `mapstructure:"http2_read_idle_timeout"` - // HTTP2PingTimeout if there's no response to the ping within the configured value, the connection will be closed. - // If not set or set to 0, it defaults to 15s. - HTTP2PingTimeout time.Duration `mapstructure:"http2_ping_timeout"` - // Cookies configures the cookie management of the HTTP client. - Cookies *CookiesConfig `mapstructure:"cookies"` + // WriteBufferSize corresponds to the JSON schema field "write_buffer_size". + WriteBufferSize int `mapstructure:"write_buffer_size"` } -// CookiesConfig defines the configuration of the HTTP client regarding cookies served by the server. -type CookiesConfig struct { - // Enabled if true, cookies from HTTP responses will be reused in further HTTP requests with the same server. +type ClientConfigCookies struct { + // Enabled corresponds to the JSON schema field "enabled". Enabled bool `mapstructure:"enabled"` } -// NewDefaultClientConfig returns ClientConfig type object with -// the default values of 'MaxIdleConns' and 'IdleConnTimeout'. -// Other config options are not added as they are initialized with 'zero value' by GoLang as default. -// We encourage to use this function to create an object of ClientConfig. -func NewDefaultClientConfig() ClientConfig { - // The default values are taken from the values of 'DefaultTransport' of 'http' package. - maxIdleConns := 100 - idleConnTimeout := 90 * time.Second - - return ClientConfig{ - MaxIdleConns: &maxIdleConns, - IdleConnTimeout: &idleConnTimeout, - } -} - -// ToClient creates an HTTP client. -func (hcs *ClientConfig) ToClient(ctx context.Context, host component.Host, settings component.TelemetrySettings) (*http.Client, error) { - tlsCfg, err := hcs.TLSSetting.LoadTLSConfig(ctx) - if err != nil { - return nil, err - } - transport := http.DefaultTransport.(*http.Transport).Clone() - if tlsCfg != nil { - transport.TLSClientConfig = tlsCfg +// UnmarshalJSON implements json.Unmarshaler. +func (j *ClientConfigCookies) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - if hcs.ReadBufferSize > 0 { - transport.ReadBufferSize = hcs.ReadBufferSize + type Plain ClientConfigCookies + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - if hcs.WriteBufferSize > 0 { - transport.WriteBufferSize = hcs.WriteBufferSize + if v, ok := raw["enabled"]; !ok || v == nil { + plain.Enabled = false } + *j = ClientConfigCookies(plain) + return nil +} - if hcs.MaxIdleConns != nil { - transport.MaxIdleConns = *hcs.MaxIdleConns - } +// SetDefaults sets the fields of ClientConfigCookies to their defaults. +// Fields which do not have a default value are left untouched. +func (c *ClientConfigCookies) SetDefaults() { + c.Enabled = false +} - if hcs.MaxIdleConnsPerHost != nil { - transport.MaxIdleConnsPerHost = *hcs.MaxIdleConnsPerHost +// UnmarshalJSON implements json.Unmarshaler. +func (j *ClientConfig) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - - if hcs.MaxConnsPerHost != nil { - transport.MaxConnsPerHost = *hcs.MaxConnsPerHost + type Plain ClientConfig + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - - if hcs.IdleConnTimeout != nil { - transport.IdleConnTimeout = *hcs.IdleConnTimeout + if v, ok := raw["compression"]; !ok || v == nil { + plain.Compression = "none" } - - // Setting the Proxy URL - if hcs.ProxyURL != "" { - proxyURL, parseErr := url.ParseRequestURI(hcs.ProxyURL) - if parseErr != nil { - return nil, parseErr - } - transport.Proxy = http.ProxyURL(proxyURL) + if v, ok := raw["disable_keep_alives"]; !ok || v == nil { + plain.DisableKeepAlives = false } - - transport.DisableKeepAlives = hcs.DisableKeepAlives - - if hcs.HTTP2ReadIdleTimeout > 0 { - transport2, transportErr := http2.ConfigureTransports(transport) - if transportErr != nil { - return nil, fmt.Errorf("failed to configure http2 transport: %w", transportErr) - } - transport2.ReadIdleTimeout = hcs.HTTP2ReadIdleTimeout - transport2.PingTimeout = hcs.HTTP2PingTimeout + if v, ok := raw["endpoint"]; !ok || v == nil { + plain.Endpoint = "" } - - clientTransport := (http.RoundTripper)(transport) - - // The Auth RoundTripper should always be the innermost to ensure that - // request signing-based auth mechanisms operate after compression - // and header middleware modifies the request - if hcs.Auth != nil { - ext := host.GetExtensions() - if ext == nil { - return nil, errors.New("extensions configuration not found") - } - - httpCustomAuthRoundTripper, aerr := hcs.Auth.GetClientAuthenticator(ctx, ext) - if aerr != nil { - return nil, aerr - } - - clientTransport, err = httpCustomAuthRoundTripper.RoundTripper(clientTransport) + if v, ok := raw["http2_ping_timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") if err != nil { - return nil, err - } - } - - if len(hcs.Headers) > 0 { - clientTransport = &headerRoundTripper{ - transport: clientTransport, - headers: hcs.Headers, + return fmt.Errorf("failed to parse the \"33.3s\" default value for field http2_ping_timeout:%w }", err) } + plain.Http2PingTimeout = defaultDuration } - - // Compress the body using specified compression methods if non-empty string is provided. - // Supporting gzip, zlib, deflate, snappy, and zstd; none is treated as uncompressed. - if hcs.Compression.IsCompressed() { - clientTransport, err = newCompressRoundTripper(clientTransport, hcs.Compression) + if v, ok := raw["http2_read_idle_timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") if err != nil { - return nil, err + return fmt.Errorf("failed to parse the \"33.3s\" default value for field http2_read_idle_timeout:%w }", err) } + plain.Http2ReadIdleTimeout = defaultDuration } - - otelOpts := []otelhttp.Option{ - otelhttp.WithTracerProvider(settings.TracerProvider), - otelhttp.WithPropagators(otel.GetTextMapPropagator()), - } - if settings.MetricsLevel >= configtelemetry.LevelDetailed { - otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider)) + if v, ok := raw["proxy_url"]; !ok || v == nil { + plain.ProxyUrl = "" } - - // wrapping http transport with otelhttp transport to enable otel instrumentation - if settings.TracerProvider != nil && settings.MeterProvider != nil { - clientTransport = otelhttp.NewTransport(clientTransport, otelOpts...) + if v, ok := raw["read_buffer_size"]; !ok || v == nil { + plain.ReadBufferSize = 0.0 } - - var jar http.CookieJar - if hcs.Cookies != nil && hcs.Cookies.Enabled { - jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + if v, ok := raw["timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") if err != nil { - return nil, err + return fmt.Errorf("failed to parse the \"33.3s\" default value for field timeout:%w }", err) } + plain.Timeout = defaultDuration + } + if v, ok := raw["write_buffer_size"]; !ok || v == nil { + plain.WriteBufferSize = 0.0 } + *j = ClientConfig(plain) + return nil +} - return &http.Client{ - Transport: clientTransport, - Timeout: hcs.Timeout, - Jar: jar, - }, nil +// SetDefaults sets the fields of ClientConfig to their defaults. +// Fields which do not have a default value are left untouched. +func (c *ClientConfig) SetDefaults() { + c.Compression = "none" + c.DisableKeepAlives = false + c.Endpoint = "" + c.Http2PingTimeout = "PT33.3S" + c.Http2ReadIdleTimeout = "PT33.3S" + c.ProxyUrl = "" + c.ReadBufferSize = 0.0 + c.Timeout = "PT33.3S" + c.WriteBufferSize = 0.0 } -// Custom RoundTripper that adds headers. -type headerRoundTripper struct { - transport http.RoundTripper - headers map[string]configopaque.String +type Cors struct { + // AllowedHeaders corresponds to the JSON schema field "allowed_headers". + AllowedHeaders []string `mapstructure:"allowed_headers"` + + // AllowedOrigins corresponds to the JSON schema field "allowed_origins". + AllowedOrigins []string `mapstructure:"allowed_origins"` + + // MaxAge corresponds to the JSON schema field "max_age". + MaxAge int `mapstructure:"max_age"` } -// RoundTrip is a custom RoundTripper that adds headers to the request. -func (interceptor *headerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - // Set Host header if provided - hostHeader, found := interceptor.headers["Host"] - if found && hostHeader != "" { - // `Host` field should be set to override default `Host` header value which is Endpoint - req.Host = string(hostHeader) +// UnmarshalJSON implements json.Unmarshaler. +func (j *Cors) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - for k, v := range interceptor.headers { - req.Header.Set(k, string(v)) + type Plain Cors + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } + if v, ok := raw["max_age"]; !ok || v == nil { + plain.MaxAge = 0.0 + } + *j = Cors(plain) + return nil +} - // Send the request to next transport. - return interceptor.transport.RoundTrip(req) +// SetDefaults sets the fields of Cors to their defaults. +// Fields which do not have a default value are left untouched. +func (c *Cors) SetDefaults() { + c.MaxAge = 0.0 } -// ServerConfig defines settings for creating an HTTP server. type ServerConfig struct { - // Endpoint configures the listening address for the server. - Endpoint string `mapstructure:"endpoint"` + // TLSSetting corresponds to the JSON schema field "TLSSetting". + TLSSetting *configtls.ServerConfig `mapstructure:"TLSSetting"` + + // Auth corresponds to the JSON schema field "auth". + Auth *ServerConfigAuth `mapstructure:"auth"` - // TLSSetting struct exposes TLS client configuration. - TLSSetting *configtls.ServerConfig `mapstructure:"tls"` + // CompressionAlgorithms corresponds to the JSON schema field + // "compression_algorithms". + CompressionAlgorithms configcompression.Type `mapstructure:"compression_algorithms"` - // CORS configures the server for HTTP cross-origin resource sharing (CORS). - CORS *CORSConfig `mapstructure:"cors"` + // Cors corresponds to the JSON schema field "cors". + Cors *Cors `mapstructure:"cors"` - // Auth for this receiver - Auth *AuthConfig `mapstructure:"auth"` + // Endpoint corresponds to the JSON schema field "endpoint". + Endpoint string `mapstructure:"endpoint"` - // MaxRequestBodySize sets the maximum request body size in bytes. Default: 20MiB. - MaxRequestBodySize int64 `mapstructure:"max_request_body_size"` + // IdleTimeout corresponds to the JSON schema field "idle_timeout". + IdleTimeout time.Duration `mapstructure:"idle_timeout"` - // IncludeMetadata propagates the client metadata from the incoming requests to the downstream consumers + // IncludeMetadata corresponds to the JSON schema field "include_metadata". IncludeMetadata bool `mapstructure:"include_metadata"` - // Additional headers attached to each HTTP response sent to the client. - // Header values are opaque since they may be sensitive. - ResponseHeaders map[string]configopaque.String `mapstructure:"response_headers"` - - // CompressionAlgorithms configures the list of compression algorithms the server can accept. Default: ["", "gzip", "zstd", "zlib", "snappy", "deflate"] - CompressionAlgorithms []string `mapstructure:"compression_algorithms"` - - // ReadTimeout is the maximum duration for reading the entire - // request, including the body. A zero or negative value means - // there will be no timeout. - // - // Because ReadTimeout does not let Handlers make per-request - // decisions on each request body's acceptable deadline or - // upload rate, most users will prefer to use - // ReadHeaderTimeout. It is valid to use them both. - ReadTimeout time.Duration `mapstructure:"read_timeout"` + // MaxRequestBodySize corresponds to the JSON schema field + // "max_request_body_size". + MaxRequestBodySize int `mapstructure:"max_request_body_size"` - // ReadHeaderTimeout is the amount of time allowed to read - // request headers. The connection's read deadline is reset - // after reading the headers and the Handler can decide what - // is considered too slow for the body. If ReadHeaderTimeout - // is zero, the value of ReadTimeout is used. If both are - // zero, there is no timeout. + // ReadHeaderTimeout corresponds to the JSON schema field "read_header_timeout". ReadHeaderTimeout time.Duration `mapstructure:"read_header_timeout"` - // WriteTimeout is the maximum duration before timing out - // writes of the response. It is reset whenever a new - // request's header is read. Like ReadTimeout, it does not - // let Handlers make decisions on a per-request basis. - // A zero or negative value means there will be no timeout. - WriteTimeout time.Duration `mapstructure:"write_timeout"` + // ReadTimeout corresponds to the JSON schema field "read_timeout". + ReadTimeout time.Duration `mapstructure:"read_timeout"` - // IdleTimeout is the maximum amount of time to wait for the - // next request when keep-alives are enabled. If IdleTimeout - // is zero, the value of ReadTimeout is used. If both are - // zero, there is no timeout. - IdleTimeout time.Duration `mapstructure:"idle_timeout"` -} + // ResponseHeaders corresponds to the JSON schema field "response_headers". + ResponseHeaders map[string]configopaque.String `mapstructure:"response_headers"` -// NewDefaultServerConfig returns ServerConfig type object with default values. -// We encourage to use this function to create an object of ServerConfig. -func NewDefaultServerConfig() ServerConfig { - tlsDefaultServerConfig := configtls.NewDefaultServerConfig() - return ServerConfig{ - ResponseHeaders: map[string]configopaque.String{}, - TLSSetting: &tlsDefaultServerConfig, - CORS: NewDefaultCORSConfig(), - WriteTimeout: 30 * time.Second, - ReadHeaderTimeout: 1 * time.Minute, - IdleTimeout: 1 * time.Minute, - } + // WriteTimeout corresponds to the JSON schema field "write_timeout". + WriteTimeout time.Duration `mapstructure:"write_timeout"` } -type AuthConfig struct { - // Auth for this receiver. - configauth.Authentication `mapstructure:",squash"` - - // RequestParameters is a list of parameters that should be extracted from the request and added to the context. - // When a parameter is found in both the query string and the header, the value from the query string will be used. - RequestParameters []string `mapstructure:"request_params"` +type ServerConfigAuth struct { + // RequestParams corresponds to the JSON schema field "request_params". + RequestParams []string `mapstructure:"request_params"` } -// ToListener creates a net.Listener. -func (hss *ServerConfig) ToListener(ctx context.Context) (net.Listener, error) { - listener, err := net.Listen("tcp", hss.Endpoint) - if err != nil { - return nil, err +// UnmarshalJSON implements json.Unmarshaler. +func (j *ServerConfig) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - - if hss.TLSSetting != nil { - var tlsCfg *tls.Config - tlsCfg, err = hss.TLSSetting.LoadTLSConfig(ctx) - if err != nil { - return nil, err - } - tlsCfg.NextProtos = []string{http2.NextProtoTLS, "http/1.1"} - listener = tls.NewListener(listener, tlsCfg) + type Plain ServerConfig + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - - return listener, nil -} - -// toServerOptions has options that change the behavior of the HTTP server -// returned by ServerConfig.ToServer(). -type toServerOptions struct { - errHandler func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int) - decoders map[string]func(body io.ReadCloser) (io.ReadCloser, error) -} - -// ToServerOption is an option to change the behavior of the HTTP server -// returned by ServerConfig.ToServer(). -type ToServerOption func(opts *toServerOptions) - -// WithErrorHandler overrides the HTTP error handler that gets invoked -// when there is a failure inside httpContentDecompressor. -func WithErrorHandler(e func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int)) ToServerOption { - return func(opts *toServerOptions) { - opts.errHandler = e - } -} - -// WithDecoder provides support for additional decoders to be configured -// by the caller. -func WithDecoder(key string, dec func(body io.ReadCloser) (io.ReadCloser, error)) ToServerOption { - return func(opts *toServerOptions) { - if opts.decoders == nil { - opts.decoders = map[string]func(body io.ReadCloser) (io.ReadCloser, error){} - } - opts.decoders[key] = dec + if v, ok := raw["compression_algorithms"]; !ok || v == nil { + plain.CompressionAlgorithms = "none" } -} - -// ToServer creates an http.Server from settings object. -func (hss *ServerConfig) ToServer(_ context.Context, host component.Host, settings component.TelemetrySettings, handler http.Handler, opts ...ToServerOption) (*http.Server, error) { - internal.WarnOnUnspecifiedHost(settings.Logger, hss.Endpoint) - - serverOpts := &toServerOptions{} - for _, o := range opts { - o(serverOpts) + if v, ok := raw["endpoint"]; !ok || v == nil { + plain.Endpoint = "" } - - if hss.MaxRequestBodySize <= 0 { - hss.MaxRequestBodySize = defaultMaxRequestBodySize + if v, ok := raw["idle_timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") + if err != nil { + return fmt.Errorf("failed to parse the \"33.3s\" default value for field idle_timeout:%w }", err) + } + plain.IdleTimeout = defaultDuration } - - if hss.CompressionAlgorithms == nil { - hss.CompressionAlgorithms = defaultCompressionAlgorithms + if v, ok := raw["include_metadata"]; !ok || v == nil { + plain.IncludeMetadata = false } - - handler = httpContentDecompressor(handler, hss.MaxRequestBodySize, serverOpts.errHandler, hss.CompressionAlgorithms, serverOpts.decoders) - - if hss.MaxRequestBodySize > 0 { - handler = maxRequestBodySizeInterceptor(handler, hss.MaxRequestBodySize) + if v, ok := raw["max_request_body_size"]; !ok || v == nil { + plain.MaxRequestBodySize = 0.0 } - - if hss.Auth != nil { - server, err := hss.Auth.GetServerAuthenticator(context.Background(), host.GetExtensions()) + if v, ok := raw["read_header_timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") if err != nil { - return nil, err + return fmt.Errorf("failed to parse the \"33.3s\" default value for field read_header_timeout:%w }", err) } - - handler = authInterceptor(handler, server, hss.Auth.RequestParameters) + plain.ReadHeaderTimeout = defaultDuration } - - if hss.CORS != nil && len(hss.CORS.AllowedOrigins) > 0 { - co := cors.Options{ - AllowedOrigins: hss.CORS.AllowedOrigins, - AllowCredentials: true, - AllowedHeaders: hss.CORS.AllowedHeaders, - MaxAge: hss.CORS.MaxAge, + if v, ok := raw["read_timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") + if err != nil { + return fmt.Errorf("failed to parse the \"33.3s\" default value for field read_timeout:%w }", err) } - handler = cors.New(co).Handler(handler) - } - if hss.CORS != nil && len(hss.CORS.AllowedOrigins) == 0 && len(hss.CORS.AllowedHeaders) > 0 { - settings.Logger.Warn("The CORS configuration specifies allowed headers but no allowed origins, and is therefore ignored.") - } - - if hss.ResponseHeaders != nil { - handler = responseHeadersHandler(handler, hss.ResponseHeaders) + plain.ReadTimeout = defaultDuration } - - otelOpts := []otelhttp.Option{ - otelhttp.WithTracerProvider(settings.TracerProvider), - otelhttp.WithPropagators(otel.GetTextMapPropagator()), - otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { - return r.URL.Path - }), - } - if settings.MetricsLevel >= configtelemetry.LevelDetailed { - otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider)) - } - - // Enable OpenTelemetry observability plugin. - // TODO: Consider to use component ID string as prefix for all the operations. - handler = otelhttp.NewHandler(handler, "", otelOpts...) - - // wrap the current handler in an interceptor that will add client.Info to the request's context - handler = &clientInfoHandler{ - next: handler, - includeMetadata: hss.IncludeMetadata, - } - - server := &http.Server{ - Handler: handler, - ReadTimeout: hss.ReadTimeout, - ReadHeaderTimeout: hss.ReadHeaderTimeout, - WriteTimeout: hss.WriteTimeout, - IdleTimeout: hss.IdleTimeout, - } - - return server, nil -} - -func responseHeadersHandler(handler http.Handler, headers map[string]configopaque.String) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h := w.Header() - - for k, v := range headers { - h.Set(k, string(v)) - } - - handler.ServeHTTP(w, r) - }) -} - -// CORSConfig configures a receiver for HTTP cross-origin resource sharing (CORS). -// See the underlying https://github.com/rs/cors package for details. -type CORSConfig struct { - // AllowedOrigins sets the allowed values of the Origin header for - // HTTP/JSON requests to an OTLP receiver. An origin may contain a - // wildcard (*) to replace 0 or more characters (e.g., - // "http://*.domain.com", or "*" to allow any origin). - AllowedOrigins []string `mapstructure:"allowed_origins"` - - // AllowedHeaders sets what headers will be allowed in CORS requests. - // The Accept, Accept-Language, Content-Type, and Content-Language - // headers are implicitly allowed. If no headers are listed, - // X-Requested-With will also be accepted by default. Include "*" to - // allow any request header. - AllowedHeaders []string `mapstructure:"allowed_headers"` - - // MaxAge sets the value of the Access-Control-Max-Age response header. - // Set it to the number of seconds that browsers should cache a CORS - // preflight response for. - MaxAge int `mapstructure:"max_age"` -} - -// NewDefaultCORSConfig creates a default cross-origin resource sharing (CORS) configuration. -func NewDefaultCORSConfig() *CORSConfig { - return &CORSConfig{} -} - -func authInterceptor(next http.Handler, server auth.Server, requestParams []string) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - sources := r.Header - query := r.URL.Query() - for _, param := range requestParams { - if val, ok := query[param]; ok { - sources[param] = val - } - } - ctx, err := server.Authenticate(r.Context(), sources) + if v, ok := raw["write_timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") if err != nil { - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) - return + return fmt.Errorf("failed to parse the \"33.3s\" default value for field write_timeout:%w }", err) } - - next.ServeHTTP(w, r.WithContext(ctx)) - }) + plain.WriteTimeout = defaultDuration + } + *j = ServerConfig(plain) + return nil } -func maxRequestBodySizeInterceptor(next http.Handler, maxRecvSize int64) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r.Body = http.MaxBytesReader(w, r.Body, maxRecvSize) - next.ServeHTTP(w, r) - }) +// SetDefaults sets the fields of ServerConfig to their defaults. +// Fields which do not have a default value are left untouched. +func (c *ServerConfig) SetDefaults() { + c.CompressionAlgorithms = "none" + c.Endpoint = "" + c.IdleTimeout = "PT33.3S" + c.IncludeMetadata = false + c.MaxRequestBodySize = 0.0 + c.ReadHeaderTimeout = "PT33.3S" + c.ReadTimeout = "PT33.3S" + c.WriteTimeout = "PT33.3S" } diff --git a/config/confighttp/confighttp.yaml b/config/confighttp/confighttp.yaml new file mode 100644 index 00000000000..1c99b38c393 --- /dev/null +++ b/config/confighttp/confighttp.yaml @@ -0,0 +1,146 @@ +"$schema": http://json-schema.org/draft-04/schema# +id: opentelemetry.io/collector/config/confighttp/confighttp +"$defs": + cors: + type: object + title: CorsConfig + properties: + allowed_origins: + type: array + items: + type: string + allowed_headers: + type: array + items: + type: string + max_age: + type: integer + default: 0 + client_config: + type: object + title: ClientConfig + properties: + TLSSetting: + "$ref": "../configtls/configtls.yaml#/$defs/client_config" + default: + additionalProperties: false + endpoint: + type: string + default: '' + proxy_url: + type: string + default: '' + read_buffer_size: + type: integer + default: 0 + write_buffer_size: + type: integer + default: 0 + timeout: + type: string + format: duration + default: "PT33.3S" + headers: + type: object + # TODO: This shouldn't generate a new type ClientConfigHeaders + additionalProperties: + type: string + goType: + # TODO: Make this import work + # "$ref": "../configopaque/schema.yaml#/$defs/String" + name: String + package: go.opentelemetry.io/collector/config/configopaque + auth: + "$ref": "../../config/configauth/configauth.yaml#/$defs/authentication" + compression: + "$ref": "../configcompression/compressiontype.yaml" + # TODO: Why isn't the default inside compressiontype.yaml enough? + default: 'none' + max_idle_conns: + type: integer + max_idle_conns_per_host: + type: integer + max_conns_per_host: + type: integer + idle_conn_timeout: + type: string + format: duration + disable_keep_alives: + type: boolean + default: false + http2_read_idle_timeout: + type: string + format: duration + default: "PT33.3S" + http2_ping_timeout: + type: string + format: duration + default: "PT33.3S" + cookies: + type: object + properties: + enabled: + type: boolean + default: false + server_config: + type: object + title: ServerConfig + properties: + TLSSetting: + "$ref": "../configtls/configtls.yaml#/$defs/server_config" + cors: + "$ref": "#/$defs/cors" + endpoint: + type: string + default: '' + auth: + type: object + title: AuthConfig + # TODO: This allOf doesn't seem to work properly + # allOf: + # - "$ref": "../../config/configauth/configauth.yaml#/$defs/authentication" + properties: + request_params: + type: array + items: + type: string + default: + # TODO: The above schema doesn't work properly. It generates: + # if v, ok := raw["auth"]; !ok || v == nil { + # plain.Auth = ServerConfigAuth{ + # RequestParams: []interface{}{}, + # } + # } + max_request_body_size: + type: integer + default: 0 + include_metadata: + type: boolean + default: false + response_headers: + type: object + additionalProperties: + type: string + goType: + name: String + package: go.opentelemetry.io/collector/config/configopaque + compression_algorithms: + "$ref": "../configcompression/compressiontype.yaml" + # TODO: Why isn't the default inside compressiontype.yaml enough? + default: 'none' + read_timeout: + type: string + format: duration + default: "PT33.3S" + read_header_timeout: + type: string + format: duration + default: "PT33.3S" + write_timeout: + type: string + format: duration + default: "PT33.3S" + idle_timeout: + type: string + format: duration + default: "PT33.3S" diff --git a/config/confighttp/confighttp_test.go b/config/confighttp/confighttp_test.go index 9c9a6fd1695..8d4e5b5fece 100644 --- a/config/confighttp/confighttp_test.go +++ b/config/confighttp/confighttp_test.go @@ -85,9 +85,9 @@ func TestAllHTTPClientSettings(t *testing.T) { IdleConnTimeout: &idleConnTimeout, Compression: "", DisableKeepAlives: true, - Cookies: &CookiesConfig{Enabled: true}, - HTTP2ReadIdleTimeout: idleConnTimeout, - HTTP2PingTimeout: http2PingTimeout, + Cookies: &ClientConfigCookies{Enabled: true}, + Http2ReadIdleTimeout: idleConnTimeout, + Http2PingTimeout: http2PingTimeout, }, shouldError: false, }, @@ -106,8 +106,8 @@ func TestAllHTTPClientSettings(t *testing.T) { IdleConnTimeout: &idleConnTimeout, Compression: "none", DisableKeepAlives: true, - HTTP2ReadIdleTimeout: idleConnTimeout, - HTTP2PingTimeout: http2PingTimeout, + Http2ReadIdleTimeout: idleConnTimeout, + Http2PingTimeout: http2PingTimeout, }, shouldError: false, }, @@ -126,8 +126,8 @@ func TestAllHTTPClientSettings(t *testing.T) { IdleConnTimeout: &idleConnTimeout, Compression: "gzip", DisableKeepAlives: true, - HTTP2ReadIdleTimeout: idleConnTimeout, - HTTP2PingTimeout: http2PingTimeout, + Http2ReadIdleTimeout: idleConnTimeout, + Http2PingTimeout: http2PingTimeout, }, shouldError: false, }, @@ -146,8 +146,8 @@ func TestAllHTTPClientSettings(t *testing.T) { IdleConnTimeout: &idleConnTimeout, Compression: "gzip", DisableKeepAlives: true, - HTTP2ReadIdleTimeout: idleConnTimeout, - HTTP2PingTimeout: http2PingTimeout, + Http2ReadIdleTimeout: idleConnTimeout, + Http2PingTimeout: http2PingTimeout, }, shouldError: false, }, @@ -255,7 +255,7 @@ func TestProxyURL(t *testing.T) { for _, tC := range testCases { t.Run(tC.desc, func(t *testing.T) { s := NewDefaultClientConfig() - s.ProxyURL = tC.proxyURL + s.ProxyUrl = tC.proxyURL tt := componenttest.NewNopTelemetrySettings() tt.TracerProvider = nil @@ -299,10 +299,10 @@ func TestHTTPClientSettingsError(t *testing.T) { Endpoint: "", TLSSetting: configtls.ClientConfig{ Config: configtls.Config{ - CAFile: "/doesnt/exist", + CaFile: "/doesnt/exist", }, - Insecure: false, - ServerName: "", + Insecure: false, + ServerNameOverride: "", }, }, }, @@ -314,8 +314,8 @@ func TestHTTPClientSettingsError(t *testing.T) { Config: configtls.Config{ CertFile: "/doesnt/exist", }, - Insecure: false, - ServerName: "", + Insecure: false, + ServerNameOverride: "", }, }, }, @@ -323,7 +323,7 @@ func TestHTTPClientSettingsError(t *testing.T) { err: "failed to resolve authenticator \"dummy\": authenticator not found", settings: ClientConfig{ Endpoint: "https://localhost:1234/v1/traces", - Auth: &configauth.Authentication{AuthenticatorID: dummyID}, + Auth: &configauth.Authentication{Authenticator: dummyID}, }, }, } @@ -361,7 +361,7 @@ func TestHTTPClientSettingWithAuthConfig(t *testing.T) { name: "with_auth_configuration_and_no_extension", settings: ClientConfig{ Endpoint: "localhost:1234", - Auth: &configauth.Authentication{AuthenticatorID: dummyID}, + Auth: &configauth.Authentication{Authenticator: dummyID}, }, shouldErr: true, host: &mockHost{ @@ -374,7 +374,7 @@ func TestHTTPClientSettingWithAuthConfig(t *testing.T) { name: "with_auth_configuration_and_no_extension_map", settings: ClientConfig{ Endpoint: "localhost:1234", - Auth: &configauth.Authentication{AuthenticatorID: dummyID}, + Auth: &configauth.Authentication{Authenticator: dummyID}, }, shouldErr: true, host: componenttest.NewNopHost(), @@ -383,7 +383,7 @@ func TestHTTPClientSettingWithAuthConfig(t *testing.T) { name: "with_auth_configuration_has_extension", settings: ClientConfig{ Endpoint: "localhost:1234", - Auth: &configauth.Authentication{AuthenticatorID: mockID}, + Auth: &configauth.Authentication{Authenticator: mockID}, }, shouldErr: false, host: &mockHost{ @@ -396,7 +396,7 @@ func TestHTTPClientSettingWithAuthConfig(t *testing.T) { name: "with_auth_configuration_has_extension_and_headers", settings: ClientConfig{ Endpoint: "localhost:1234", - Auth: &configauth.Authentication{AuthenticatorID: mockID}, + Auth: &configauth.Authentication{Authenticator: mockID}, Headers: map[string]configopaque.String{"foo": "bar"}, }, shouldErr: false, @@ -410,7 +410,7 @@ func TestHTTPClientSettingWithAuthConfig(t *testing.T) { name: "with_auth_configuration_has_extension_and_compression", settings: ClientConfig{ Endpoint: "localhost:1234", - Auth: &configauth.Authentication{AuthenticatorID: component.MustNewID("mock")}, + Auth: &configauth.Authentication{Authenticator: component.MustNewID("mock")}, Compression: configcompression.TypeGzip, }, shouldErr: false, @@ -424,7 +424,7 @@ func TestHTTPClientSettingWithAuthConfig(t *testing.T) { name: "with_auth_configuration_has_err_extension", settings: ClientConfig{ Endpoint: "localhost:1234", - Auth: &configauth.Authentication{AuthenticatorID: mockID}, + Auth: &configauth.Authentication{Authenticator: mockID}, }, shouldErr: true, host: &mockHost{ @@ -482,7 +482,7 @@ func TestHTTPServerSettingsError(t *testing.T) { Endpoint: "localhost:0", TLSSetting: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: "/doesnt/exist", + CaFile: "/doesnt/exist", }, }, }, @@ -503,7 +503,7 @@ func TestHTTPServerSettingsError(t *testing.T) { settings: ServerConfig{ Endpoint: "localhost:0", TLSSetting: &configtls.ServerConfig{ - ClientCAFile: "/doesnt/exist", + ClientCaFile: "/doesnt/exist", }, }, }, @@ -582,32 +582,32 @@ func TestHttpReception(t *testing.T) { name: "TLS", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, }, { name: "TLS (HTTP/1.1)", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, forceHTTP1: true, }, @@ -615,14 +615,14 @@ func TestHttpReception(t *testing.T) { name: "NoServerCertificates", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, hasError: true, }, @@ -630,36 +630,36 @@ func TestHttpReception(t *testing.T) { name: "mTLS", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, - ClientCAFile: filepath.Join("testdata", "ca.crt"), + ClientCaFile: filepath.Join("testdata", "ca.crt"), }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "client.crt"), KeyFile: filepath.Join("testdata", "client.key"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, }, { name: "NoClientCertificate", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, - ClientCAFile: filepath.Join("testdata", "ca.crt"), + ClientCaFile: filepath.Join("testdata", "ca.crt"), }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, hasError: true, }, @@ -667,19 +667,19 @@ func TestHttpReception(t *testing.T) { name: "WrongClientCA", tlsServerCreds: &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, - ClientCAFile: filepath.Join("testdata", "server.crt"), + ClientCaFile: filepath.Join("testdata", "server.crt"), }, tlsClientCreds: &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "client.crt"), KeyFile: filepath.Join("testdata", "client.key"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", }, hasError: true, }, @@ -748,7 +748,7 @@ func TestHttpCors(t *testing.T) { tests := []struct { name string - *CORSConfig + *Cors allowedWorks bool disallowedWorks bool @@ -762,14 +762,14 @@ func TestHttpCors(t *testing.T) { }, { name: "emptyCORS", - CORSConfig: NewDefaultCORSConfig(), + Cors: NewDefaultCORSConfig(), allowedWorks: false, disallowedWorks: false, extraHeaderWorks: false, }, { name: "OriginCORS", - CORSConfig: &CORSConfig{ + Cors: &Cors{ AllowedOrigins: []string{"allowed-*.com"}, }, allowedWorks: true, @@ -778,7 +778,7 @@ func TestHttpCors(t *testing.T) { }, { name: "CacheableCORS", - CORSConfig: &CORSConfig{ + Cors: &Cors{ AllowedOrigins: []string{"allowed-*.com"}, MaxAge: 360, }, @@ -788,7 +788,7 @@ func TestHttpCors(t *testing.T) { }, { name: "HeaderCORS", - CORSConfig: &CORSConfig{ + Cors: &Cors{ AllowedOrigins: []string{"allowed-*.com"}, AllowedHeaders: []string{"ExtraHeader"}, }, @@ -802,7 +802,7 @@ func TestHttpCors(t *testing.T) { t.Run(tt.name, func(t *testing.T) { hss := &ServerConfig{ Endpoint: "localhost:0", - CORS: tt.CORSConfig, + Cors: tt.Cors, } ln, err := hss.ToListener(context.Background()) @@ -824,18 +824,18 @@ func TestHttpCors(t *testing.T) { url := fmt.Sprintf("http://%s", ln.Addr().String()) expectedStatus := http.StatusNoContent - if tt.CORSConfig == nil || len(tt.AllowedOrigins) == 0 { + if tt.Cors == nil || len(tt.AllowedOrigins) == 0 { expectedStatus = http.StatusOK } - // Verify allowed domain gets responses that allow CORS. - verifyCorsResp(t, url, "allowed-origin.com", tt.CORSConfig, false, expectedStatus, tt.allowedWorks) + // Verify allowed domain gets responses that allow Cors. + verifyCorsResp(t, url, "allowed-origin.com", tt.Cors, false, expectedStatus, tt.allowedWorks) - // Verify allowed domain and extra headers gets responses that allow CORS. - verifyCorsResp(t, url, "allowed-origin.com", tt.CORSConfig, true, expectedStatus, tt.extraHeaderWorks) + // Verify allowed domain and extra headers gets responses that allow Cors. + verifyCorsResp(t, url, "allowed-origin.com", tt.Cors, true, expectedStatus, tt.extraHeaderWorks) - // Verify disallowed domain gets responses that disallow CORS. - verifyCorsResp(t, url, "disallowed-origin.com", tt.CORSConfig, false, expectedStatus, tt.disallowedWorks) + // Verify disallowed domain gets responses that disallow Cors. + verifyCorsResp(t, url, "disallowed-origin.com", tt.Cors, false, expectedStatus, tt.disallowedWorks) require.NoError(t, s.Close()) }) @@ -845,10 +845,10 @@ func TestHttpCors(t *testing.T) { func TestHttpCorsInvalidSettings(t *testing.T) { hss := &ServerConfig{ Endpoint: "localhost:0", - CORS: &CORSConfig{AllowedHeaders: []string{"some-header"}}, + Cors: &Cors{AllowedHeaders: []string{"some-header"}}, } - // This effectively does not enable CORS but should also not cause an error + // This effectively does not enable Cors but should also not cause an error s, err := hss.ToServer( context.Background(), componenttest.NewNopHost(), @@ -862,12 +862,12 @@ func TestHttpCorsInvalidSettings(t *testing.T) { func TestHttpCorsWithSettings(t *testing.T) { hss := &ServerConfig{ Endpoint: "localhost:0", - CORS: &CORSConfig{ + Cors: &Cors{ AllowedOrigins: []string{"*"}, }, - Auth: &AuthConfig{ + Auth: &ServerConfigAuth{ Authentication: configauth.Authentication{ - AuthenticatorID: mockID, + Authenticator: mockID, }, }, } @@ -943,7 +943,7 @@ func TestHttpServerHeaders(t *testing.T) { url := fmt.Sprintf("http://%s", ln.Addr().String()) - // Verify allowed domain gets responses that allow CORS. + // Verify allowed domain gets responses that allow Cors. verifyHeadersResp(t, url, tt.headers) require.NoError(t, s.Close()) @@ -951,7 +951,7 @@ func TestHttpServerHeaders(t *testing.T) { } } -func verifyCorsResp(t *testing.T, url string, origin string, set *CORSConfig, extraHeader bool, wantStatus int, wantAllowed bool) { +func verifyCorsResp(t *testing.T, url string, origin string, set *Cors, extraHeader bool, wantStatus int, wantAllowed bool) { req, err := http.NewRequest(http.MethodOptions, url, nil) require.NoError(t, err, "Error creating trace OPTIONS request: %v", err) req.Header.Set("Origin", origin) @@ -1170,9 +1170,9 @@ func TestServerAuth(t *testing.T) { authCalled := false hss := ServerConfig{ Endpoint: "localhost:0", - Auth: &AuthConfig{ + Auth: &ServerConfigAuth{ Authentication: configauth.Authentication{ - AuthenticatorID: mockID, + Authenticator: mockID, }, }, } @@ -1206,9 +1206,9 @@ func TestServerAuth(t *testing.T) { func TestInvalidServerAuth(t *testing.T) { hss := ServerConfig{ - Auth: &AuthConfig{ + Auth: &ServerConfigAuth{ Authentication: configauth.Authentication{ - AuthenticatorID: nonExistingID, + Authenticator: nonExistingID, }, }, } @@ -1222,9 +1222,9 @@ func TestFailedServerAuth(t *testing.T) { // prepare hss := ServerConfig{ Endpoint: "localhost:0", - Auth: &AuthConfig{ + Auth: &ServerConfigAuth{ Authentication: configauth.Authentication{ - AuthenticatorID: mockID, + Authenticator: mockID, }, }, } @@ -1402,10 +1402,10 @@ func TestAuthWithQueryParams(t *testing.T) { authCalled := false hss := ServerConfig{ Endpoint: "localhost:0", - Auth: &AuthConfig{ - RequestParameters: []string{"auth"}, + Auth: &ServerConfigAuth{ + RequestParams: []string{"auth"}, Authentication: configauth.Authentication{ - AuthenticatorID: mockID, + Authenticator: mockID, }, }, } @@ -1478,16 +1478,16 @@ func BenchmarkHttpRequest(b *testing.B) { tlsServerCreds := &configtls.ServerConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), CertFile: filepath.Join("testdata", "server.crt"), KeyFile: filepath.Join("testdata", "server.key"), }, } tlsClientCreds := &configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "ca.crt"), + CaFile: filepath.Join("testdata", "ca.crt"), }, - ServerName: "localhost", + ServerNameOverride: "localhost", } hss := &ServerConfig{ @@ -1555,7 +1555,7 @@ func BenchmarkHttpRequest(b *testing.B) { func TestDefaultHTTPServerSettings(t *testing.T) { httpServerSettings := NewDefaultServerConfig() assert.NotNil(t, httpServerSettings.ResponseHeaders) - assert.NotNil(t, httpServerSettings.CORS) + assert.NotNil(t, httpServerSettings.Cors) assert.NotNil(t, httpServerSettings.TLSSetting) assert.Equal(t, 1*time.Minute, httpServerSettings.IdleTimeout) assert.Equal(t, 30*time.Second, httpServerSettings.WriteTimeout) diff --git a/config/confighttp/confighttp_util.go b/config/confighttp/confighttp_util.go new file mode 100644 index 00000000000..f1effd59467 --- /dev/null +++ b/config/confighttp/confighttp_util.go @@ -0,0 +1,338 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package confighttp // import "go.opentelemetry.io/collector/config/confighttp" + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + + "github.com/rs/cors" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "golang.org/x/net/http2" + "golang.org/x/net/publicsuffix" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configtelemetry" + "go.opentelemetry.io/collector/config/internal" + "go.opentelemetry.io/collector/extension/auth" +) + +// ToClient creates an HTTP client. +func (hcs *ClientConfig) ToClient(ctx context.Context, host component.Host, settings component.TelemetrySettings) (*http.Client, error) { + tlsCfg, err := hcs.TLSSetting.LoadTLSConfig(ctx) + if err != nil { + return nil, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + if tlsCfg != nil { + transport.TLSClientConfig = tlsCfg + } + if hcs.ReadBufferSize > 0 { + transport.ReadBufferSize = hcs.ReadBufferSize + } + if hcs.WriteBufferSize > 0 { + transport.WriteBufferSize = hcs.WriteBufferSize + } + + if hcs.MaxIdleConns != nil { + transport.MaxIdleConns = *hcs.MaxIdleConns + } + + if hcs.MaxIdleConnsPerHost != nil { + transport.MaxIdleConnsPerHost = *hcs.MaxIdleConnsPerHost + } + + if hcs.MaxConnsPerHost != nil { + transport.MaxConnsPerHost = *hcs.MaxConnsPerHost + } + + if hcs.IdleConnTimeout != nil { + transport.IdleConnTimeout = *hcs.IdleConnTimeout + } + + // Setting the Proxy URL + if hcs.ProxyUrl != "" { + proxyURL, parseErr := url.ParseRequestURI(hcs.ProxyUrl) + if parseErr != nil { + return nil, parseErr + } + transport.Proxy = http.ProxyURL(proxyURL) + } + + transport.DisableKeepAlives = hcs.DisableKeepAlives + + if hcs.Http2ReadIdleTimeout > 0 { + transport2, transportErr := http2.ConfigureTransports(transport) + if transportErr != nil { + return nil, fmt.Errorf("failed to configure http2 transport: %w", transportErr) + } + transport2.ReadIdleTimeout = hcs.Http2ReadIdleTimeout + transport2.PingTimeout = hcs.Http2PingTimeout + } + + clientTransport := (http.RoundTripper)(transport) + + // The Auth RoundTripper should always be the innermost to ensure that + // request signing-based auth mechanisms operate after compression + // and header middleware modifies the request + if hcs.Auth != nil { + ext := host.GetExtensions() + if ext == nil { + return nil, errors.New("extensions configuration not found") + } + + httpCustomAuthRoundTripper, aerr := hcs.Auth.GetClientAuthenticator(ctx, ext) + if aerr != nil { + return nil, aerr + } + + clientTransport, err = httpCustomAuthRoundTripper.RoundTripper(clientTransport) + if err != nil { + return nil, err + } + } + + if len(hcs.Headers) > 0 { + clientTransport = &headerRoundTripper{ + transport: clientTransport, + headers: hcs.Headers, + } + } + + // Compress the body using specified compression methods if non-empty string is provided. + // Supporting gzip, zlib, deflate, snappy, and zstd; none is treated as uncompressed. + if hcs.Compression.IsCompressed() { + clientTransport, err = newCompressRoundTripper(clientTransport, hcs.Compression) + if err != nil { + return nil, err + } + } + + otelOpts := []otelhttp.Option{ + otelhttp.WithTracerProvider(settings.TracerProvider), + otelhttp.WithPropagators(otel.GetTextMapPropagator()), + } + if settings.MetricsLevel >= configtelemetry.LevelDetailed { + otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider)) + } + + // wrapping http transport with otelhttp transport to enable otel instrumentation + if settings.TracerProvider != nil && settings.MeterProvider != nil { + clientTransport = otelhttp.NewTransport(clientTransport, otelOpts...) + } + + var jar http.CookieJar + if hcs.Cookies != nil && hcs.Cookies.Enabled { + jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + if err != nil { + return nil, err + } + } + + return &http.Client{ + Transport: clientTransport, + Timeout: hcs.Timeout, + Jar: jar, + }, nil +} + +// Custom RoundTripper that adds headers. +type headerRoundTripper struct { + transport http.RoundTripper + headers map[string]configopaque.String +} + +// RoundTrip is a custom RoundTripper that adds headers to the request. +func (interceptor *headerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Set Host header if provided + hostHeader, found := interceptor.headers["Host"] + if found && hostHeader != "" { + // `Host` field should be set to override default `Host` header value which is Endpoint + req.Host = string(hostHeader) + } + for k, v := range interceptor.headers { + req.Header.Set(k, string(v)) + } + + // Send the request to next transport. + return interceptor.transport.RoundTrip(req) +} + +// ToListener creates a net.Listener. +func (hss *ServerConfig) ToListener(ctx context.Context) (net.Listener, error) { + listener, err := net.Listen("tcp", hss.Endpoint) + if err != nil { + return nil, err + } + + if hss.TLSSetting != nil { + var tlsCfg *tls.Config + tlsCfg, err = hss.TLSSetting.LoadTLSConfig(ctx) + if err != nil { + return nil, err + } + tlsCfg.NextProtos = []string{http2.NextProtoTLS, "http/1.1"} + listener = tls.NewListener(listener, tlsCfg) + } + + return listener, nil +} + +// toServerOptions has options that change the behavior of the HTTP server +// returned by ServerConfig.ToServer(). +type toServerOptions struct { + errHandler func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int) + decoders map[string]func(body io.ReadCloser) (io.ReadCloser, error) +} + +// ToServerOption is an option to change the behavior of the HTTP server +// returned by ServerConfig.ToServer(). +type ToServerOption func(opts *toServerOptions) + +// WithErrorHandler overrides the HTTP error handler that gets invoked +// when there is a failure inside httpContentDecompressor. +func WithErrorHandler(e func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int)) ToServerOption { + return func(opts *toServerOptions) { + opts.errHandler = e + } +} + +// WithDecoder provides support for additional decoders to be configured +// by the caller. +func WithDecoder(key string, dec func(body io.ReadCloser) (io.ReadCloser, error)) ToServerOption { + return func(opts *toServerOptions) { + if opts.decoders == nil { + opts.decoders = map[string]func(body io.ReadCloser) (io.ReadCloser, error){} + } + opts.decoders[key] = dec + } +} + +// ToServer creates an http.Server from settings object. +func (hss *ServerConfig) ToServer(_ context.Context, host component.Host, settings component.TelemetrySettings, handler http.Handler, opts ...ToServerOption) (*http.Server, error) { + internal.WarnOnUnspecifiedHost(settings.Logger, hss.Endpoint) + + serverOpts := &toServerOptions{} + for _, o := range opts { + o(serverOpts) + } + + if hss.MaxRequestBodySize <= 0 { + hss.MaxRequestBodySize = defaultMaxRequestBodySize + } + + if hss.CompressionAlgorithms == nil { + hss.CompressionAlgorithms = defaultCompressionAlgorithms + } + + handler = httpContentDecompressor(handler, hss.MaxRequestBodySize, serverOpts.errHandler, hss.CompressionAlgorithms, serverOpts.decoders) + + if hss.MaxRequestBodySize > 0 { + handler = maxRequestBodySizeInterceptor(handler, hss.MaxRequestBodySize) + } + + if hss.Auth != nil { + server, err := hss.Auth.GetServerAuthenticator(context.Background(), host.GetExtensions()) + if err != nil { + return nil, err + } + + handler = authInterceptor(handler, server, hss.Auth.RequestParameters) + } + + if hss.Cors != nil && len(hss.Cors.AllowedOrigins) > 0 { + co := cors.Options{ + AllowedOrigins: hss.Cors.AllowedOrigins, + AllowCredentials: true, + AllowedHeaders: hss.Cors.AllowedHeaders, + MaxAge: hss.Cors.MaxAge, + } + handler = cors.New(co).Handler(handler) + } + if hss.Cors != nil && len(hss.Cors.AllowedOrigins) == 0 && len(hss.Cors.AllowedHeaders) > 0 { + settings.Logger.Warn("The CORS configuration specifies allowed headers but no allowed origins, and is therefore ignored.") + } + + if hss.ResponseHeaders != nil { + handler = responseHeadersHandler(handler, hss.ResponseHeaders) + } + + otelOpts := []otelhttp.Option{ + otelhttp.WithTracerProvider(settings.TracerProvider), + otelhttp.WithPropagators(otel.GetTextMapPropagator()), + otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { + return r.URL.Path + }), + } + if settings.MetricsLevel >= configtelemetry.LevelDetailed { + otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider)) + } + + // Enable OpenTelemetry observability plugin. + // TODO: Consider to use component ID string as prefix for all the operations. + handler = otelhttp.NewHandler(handler, "", otelOpts...) + + // wrap the current handler in an interceptor that will add client.Info to the request's context + handler = &clientInfoHandler{ + next: handler, + includeMetadata: hss.IncludeMetadata, + } + + server := &http.Server{ + Handler: handler, + ReadTimeout: hss.ReadTimeout, + ReadHeaderTimeout: hss.ReadHeaderTimeout, + WriteTimeout: hss.WriteTimeout, + IdleTimeout: hss.IdleTimeout, + } + + return server, nil +} + +func responseHeadersHandler(handler http.Handler, headers map[string]configopaque.String) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + + for k, v := range headers { + h.Set(k, string(v)) + } + + handler.ServeHTTP(w, r) + }) +} + +func authInterceptor(next http.Handler, server auth.Server, requestParams []string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sources := r.Header + query := r.URL.Query() + for _, param := range requestParams { + if val, ok := query[param]; ok { + sources[param] = val + } + } + ctx, err := server.Authenticate(r.Context(), sources) + if err != nil { + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func maxRequestBodySizeInterceptor(next http.Handler, maxRecvSize int64) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRecvSize) + next.ServeHTTP(w, r) + }) +} diff --git a/config/confighttp/go.sum b/config/confighttp/go.sum index c9c7b3d192a..88acc15d015 100644 --- a/config/confighttp/go.sum +++ b/config/confighttp/go.sum @@ -65,6 +65,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= +github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/config/configopaque/schema.yaml b/config/configopaque/schema.yaml new file mode 100644 index 00000000000..e6225a84de6 --- /dev/null +++ b/config/configopaque/schema.yaml @@ -0,0 +1,6 @@ +"$schema": http://json-schema.org/draft-04/schema# +id: opentelemetry.io/collector/config/configopaque +$defs: + String: + name: String + package: go.opentelemetry.io/collector/config/configopaque diff --git a/config/configretry/backoff.go b/config/configretry/backoff.go index 1fc3f8c5852..d4c688ecc00 100644 --- a/config/configretry/backoff.go +++ b/config/configretry/backoff.go @@ -1,74 +1,84 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. -package configretry // import "go.opentelemetry.io/collector/config/configretry" +package configretry -import ( - "errors" - "time" +import "encoding/json" +import "fmt" +import "time" - "github.com/cenkalti/backoff/v4" -) - -// NewDefaultBackOffConfig returns the default settings for RetryConfig. -func NewDefaultBackOffConfig() BackOffConfig { - return BackOffConfig{ - Enabled: true, - InitialInterval: 5 * time.Second, - RandomizationFactor: backoff.DefaultRandomizationFactor, - Multiplier: backoff.DefaultMultiplier, - MaxInterval: 30 * time.Second, - MaxElapsedTime: 5 * time.Minute, - } -} - -// BackOffConfig defines configuration for retrying batches in case of export failure. -// The current supported strategy is exponential backoff. -type BackOffConfig struct { - // Enabled indicates whether to not retry sending batches in case of export failure. +type RetryOnFailure struct { + // Enabled corresponds to the JSON schema field "enabled". Enabled bool `mapstructure:"enabled"` - // InitialInterval the time to wait after the first failure before retrying. + + // InitialInterval corresponds to the JSON schema field "initial_interval". InitialInterval time.Duration `mapstructure:"initial_interval"` - // RandomizationFactor is a random factor used to calculate next backoffs - // Randomized interval = RetryInterval * (1 ± RandomizationFactor) - RandomizationFactor float64 `mapstructure:"randomization_factor"` - // Multiplier is the value multiplied by the backoff interval bounds - Multiplier float64 `mapstructure:"multiplier"` - // MaxInterval is the upper bound on backoff interval. Once this value is reached the delay between - // consecutive retries will always be `MaxInterval`. - MaxInterval time.Duration `mapstructure:"max_interval"` - // MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. - // Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + + // MaxElapsedTime corresponds to the JSON schema field "max_elapsed_time". MaxElapsedTime time.Duration `mapstructure:"max_elapsed_time"` + + // MaxInterval corresponds to the JSON schema field "max_interval". + MaxInterval time.Duration `mapstructure:"max_interval"` + + // Multiplier corresponds to the JSON schema field "multiplier". + Multiplier float64 `mapstructure:"multiplier"` + + // RandomizationFactor corresponds to the JSON schema field + // "randomization_factor". + RandomizationFactor float64 `mapstructure:"randomization_factor"` } -func (bs *BackOffConfig) Validate() error { - if !bs.Enabled { - return nil +// UnmarshalJSON implements json.Unmarshaler. +func (j *RetryOnFailure) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - if bs.InitialInterval < 0 { - return errors.New("'initial_interval' must be non-negative") + type Plain RetryOnFailure + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - if bs.RandomizationFactor < 0 || bs.RandomizationFactor > 1 { - return errors.New("'randomization_factor' must be within [0, 1]") + if v, ok := raw["enabled"]; !ok || v == nil { + plain.Enabled = true } - if bs.Multiplier < 0 { - return errors.New("'multiplier' must be non-negative") - } - if bs.MaxInterval < 0 { - return errors.New("'max_interval' must be non-negative") - } - if bs.MaxElapsedTime < 0 { - return errors.New("'max_elapsed_time' must be non-negative") + if v, ok := raw["initial_interval"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("5s") + if err != nil { + return fmt.Errorf("failed to parse the \"5s\" default value for field initial_interval:%w }", err) + } + plain.InitialInterval = defaultDuration } - if bs.MaxElapsedTime > 0 { - if bs.MaxElapsedTime < bs.InitialInterval { - return errors.New("'max_elapsed_time' must not be less than 'initial_interval'") + if v, ok := raw["max_elapsed_time"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("5m0s") + if err != nil { + return fmt.Errorf("failed to parse the \"5m0s\" default value for field max_elapsed_time:%w }", err) } - if bs.MaxElapsedTime < bs.MaxInterval { - return errors.New("'max_elapsed_time' must not be less than 'max_interval'") + plain.MaxElapsedTime = defaultDuration + } + if v, ok := raw["max_interval"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("30s") + if err != nil { + return fmt.Errorf("failed to parse the \"30s\" default value for field max_interval:%w }", err) } - + plain.MaxInterval = defaultDuration + } + if v, ok := raw["multiplier"]; !ok || v == nil { + plain.Multiplier = 1.5 } + if v, ok := raw["randomization_factor"]; !ok || v == nil { + plain.RandomizationFactor = 0.5 + } + *j = RetryOnFailure(plain) return nil } + +// SetDefaults sets the fields of RetryOnFailure to their defaults. +// Fields which do not have a default value are left untouched. +func (c *RetryOnFailure) SetDefaults() { + c.Enabled = true + c.InitialInterval = "PT5S" + c.MaxElapsedTime = "PT5M" + c.MaxInterval = "PT30S" + c.Multiplier = 1.5 + c.RandomizationFactor = 0.5 +} diff --git a/config/configretry/backoff.yaml b/config/configretry/backoff.yaml new file mode 100644 index 00000000000..a2982ba9ab7 --- /dev/null +++ b/config/configretry/backoff.yaml @@ -0,0 +1,27 @@ +"$schema": http://json-schema.org/draft-04/schema# +id: opentelemetry.io/collector/config/configretry/backoff/retry_on_failure +"$defs": + retry_on_failure: + type: object + properties: + enabled: + type: boolean + default: true + initial_interval: + type: string + format: duration + default: "PT5S" + randomization_factor: + type: number + default: 0.5 + multiplier: + type: number + default: 1.5 + max_interval: + type: string + format: duration + default: "PT30S" + max_elapsed_time: + type: string + format: duration + default: "PT5M" diff --git a/config/configtls/configtls.go b/config/configtls/configtls.go index 2ce0490b16c..a4f46b502e9 100644 --- a/config/configtls/configtls.go +++ b/config/configtls/configtls.go @@ -1,450 +1,213 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. -package configtls // import "go.opentelemetry.io/collector/config/configtls" +package configtls -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "os" - "path/filepath" - "sync" - "time" +import "encoding/json" +import "fmt" +import "go.opentelemetry.io/collector/config/configopaque" +import "time" - "go.opentelemetry.io/collector/config/configopaque" -) - -// We should avoid that users unknowingly use a vulnerable TLS version. -// The defaults should be a safe configuration -const defaultMinTLSVersion = tls.VersionTLS12 - -// Uses the default MaxVersion from "crypto/tls" which is the maximum supported version -const defaultMaxTLSVersion = 0 - -var systemCertPool = x509.SystemCertPool - -// Config exposes the common client and server TLS configurations. -// Note: Since there isn't anything specific to a server connection. Components -// with server connections should use Config. -type Config struct { - // Path to the CA cert. For a client this verifies the server certificate. - // For a server this verifies client certificates. If empty uses system root CA. - // (optional) - CAFile string `mapstructure:"ca_file"` - - // In memory PEM encoded cert. (optional) - CAPem configopaque.String `mapstructure:"ca_pem"` - - // If true, load system CA certificates pool in addition to the certificates - // configured in this struct. - IncludeSystemCACertsPool bool `mapstructure:"include_system_ca_certs_pool"` - - // Path to the TLS cert to use for TLS required connections. (optional) - CertFile string `mapstructure:"cert_file"` - - // In memory PEM encoded TLS cert to use for TLS required connections. (optional) - CertPem configopaque.String `mapstructure:"cert_pem"` - - // Path to the TLS key to use for TLS required connections. (optional) - KeyFile string `mapstructure:"key_file"` - - // In memory PEM encoded TLS key to use for TLS required connections. (optional) - KeyPem configopaque.String `mapstructure:"key_pem"` - - // MinVersion sets the minimum TLS version that is acceptable. - // If not set, TLS 1.2 will be used. (optional) - MinVersion string `mapstructure:"min_version"` - - // MaxVersion sets the maximum TLS version that is acceptable. - // If not set, refer to crypto/tls for defaults. (optional) - MaxVersion string `mapstructure:"max_version"` - - // CipherSuites is a list of TLS cipher suites that the TLS transport can use. - // If left blank, a safe default list is used. - // See https://go.dev/src/crypto/tls/cipher_suites.go for a list of supported cipher suites. - CipherSuites []string `mapstructure:"cipher_suites"` - - // ReloadInterval specifies the duration after which the certificate will be reloaded - // If not set, it will never be reloaded (optional) - ReloadInterval time.Duration `mapstructure:"reload_interval"` -} - -// NewDefaultConfig creates a new TLSSetting with any default values set. -func NewDefaultConfig() Config { - return Config{} -} - -// ClientConfig contains TLS configurations that are specific to client -// connections in addition to the common configurations. This should be used by -// components configuring TLS client connections. type ClientConfig struct { - // squash ensures fields are correctly decoded in embedded struct. - Config `mapstructure:",squash"` - - // These are config options specific to client connections. - - // In gRPC and HTTP when set to true, this is used to disable the client transport security. - // See https://godoc.org/google.golang.org/grpc#WithInsecure for gRPC. - // Please refer to https://godoc.org/crypto/tls#Config for more information. - // (optional, default false) + // Insecure corresponds to the JSON schema field "insecure". Insecure bool `mapstructure:"insecure"` - // InsecureSkipVerify will enable TLS but not verify the certificate. + + // InsecureSkipVerify corresponds to the JSON schema field "insecure_skip_verify". InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"` - // ServerName requested by client for virtual hosting. - // This sets the ServerName in the TLSConfig. Please refer to - // https://godoc.org/crypto/tls#Config for more information. (optional) - ServerName string `mapstructure:"server_name_override"` -} -// NewDefaultClientConfig creates a new TLSClientSetting with any default values set. -func NewDefaultClientConfig() ClientConfig { - return ClientConfig{ - Config: NewDefaultConfig(), - } -} + // ServerNameOverride corresponds to the JSON schema field "server_name_override". + ServerNameOverride string `mapstructure:"server_name_override"` -// ServerConfig contains TLS configurations that are specific to server -// connections in addition to the common configurations. This should be used by -// components configuring TLS server connections. -type ServerConfig struct { - // squash ensures fields are correctly decoded in embedded struct. Config `mapstructure:",squash"` - - // These are config options specific to server connections. - - // Path to the TLS cert to use by the server to verify a client certificate. (optional) - // This sets the ClientCAs and ClientAuth to RequireAndVerifyClientCert in the TLSConfig. Please refer to - // https://godoc.org/crypto/tls#Config for more information. (optional) - ClientCAFile string `mapstructure:"client_ca_file"` - - // Reload the ClientCAs file when it is modified - // (optional, default false) - ReloadClientCAFile bool `mapstructure:"client_ca_file_reload"` } -// NewDefaultServerConfig creates a new TLSServerSetting with any default values set. -func NewDefaultServerConfig() ServerConfig { - return ServerConfig{ - Config: NewDefaultConfig(), +// UnmarshalJSON implements json.Unmarshaler. +func (j *ClientConfig) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } -} - -// certReloader is a wrapper object for certificate reloading -// Its GetCertificate method will either return the current certificate or reload from disk -// if the last reload happened more than ReloadInterval ago -type certReloader struct { - nextReload time.Time - cert *tls.Certificate - lock sync.RWMutex - tls Config -} - -func (c Config) newCertReloader() (*certReloader, error) { - cert, err := c.loadCertificate() - if err != nil { - return nil, err + type Plain ClientConfig + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - return &certReloader{ - tls: c, - nextReload: time.Now().Add(c.ReloadInterval), - cert: &cert, - }, nil -} - -func (r *certReloader) GetCertificate() (*tls.Certificate, error) { - now := time.Now() - // Read locking here before we do the time comparison - // If a reload is in progress this will block and we will skip reloading in the current - // call once we can continue - r.lock.RLock() - if r.tls.ReloadInterval != 0 && r.nextReload.Before(now) && (r.tls.hasCertFile() || r.tls.hasKeyFile()) { - // Need to release the read lock, otherwise we deadlock - r.lock.RUnlock() - r.lock.Lock() - defer r.lock.Unlock() - cert, err := r.tls.loadCertificate() - if err != nil { - return nil, fmt.Errorf("failed to load TLS cert and key: %w", err) - } - r.cert = &cert - r.nextReload = now.Add(r.tls.ReloadInterval) - return r.cert, nil + if v, ok := raw["insecure"]; !ok || v == nil { + plain.Insecure = true + } + if v, ok := raw["insecure_skip_verify"]; !ok || v == nil { + plain.InsecureSkipVerify = true } - defer r.lock.RUnlock() - return r.cert, nil + if v, ok := raw["server_name_override"]; !ok || v == nil { + plain.ServerNameOverride = "" + } + *j = ClientConfig(plain) + return nil } -func (c Config) Validate() error { - if c.hasCAFile() && c.hasCAPem() { - return fmt.Errorf("provide either a CA file or the PEM-encoded string, but not both") - } +// SetDefaults sets the fields of ClientConfig to their defaults. +// Fields which do not have a default value are left untouched. +func (c *ClientConfig) SetDefaults() { + c.Insecure = true + c.InsecureSkipVerify = true + c.ServerNameOverride = "" +} - minTLS, err := convertVersion(c.MinVersion, defaultMinTLSVersion) - if err != nil { - return fmt.Errorf("invalid TLS min_version: %w", err) - } +type Config struct { + // CaFile corresponds to the JSON schema field "ca_file". + CaFile string `mapstructure:"ca_file"` - maxTLS, err := convertVersion(c.MaxVersion, defaultMaxTLSVersion) - if err != nil { - return fmt.Errorf("invalid TLS max_version: %w", err) - } + // CaPem corresponds to the JSON schema field "ca_pem". + CaPem configopaque.String `mapstructure:"ca_pem"` - if maxTLS < minTLS && maxTLS != defaultMaxTLSVersion { - return errors.New("invalid TLS configuration: min_version cannot be greater than max_version") - } + // CertFile corresponds to the JSON schema field "cert_file". + CertFile string `mapstructure:"cert_file"` - return nil -} + // CertPem corresponds to the JSON schema field "cert_pem". + CertPem configopaque.String `mapstructure:"cert_pem"` -// loadTLSConfig loads TLS certificates and returns a tls.Config. -// This will set the RootCAs and Certificates of a tls.Config. -func (c Config) loadTLSConfig() (*tls.Config, error) { - certPool, err := c.loadCACertPool() - if err != nil { - return nil, err - } + // CipherSuites corresponds to the JSON schema field "cipher_suites". + CipherSuites []string `mapstructure:"cipher_suites"` - var getCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error) - var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error) - if c.hasCert() || c.hasKey() { - var certReloader *certReloader - certReloader, err = c.newCertReloader() - if err != nil { - return nil, fmt.Errorf("failed to load TLS cert and key: %w", err) - } - getCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return certReloader.GetCertificate() } - getClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return certReloader.GetCertificate() } - } + // IncludeSystemCaCertsPool corresponds to the JSON schema field + // "include_system_ca_certs_pool". + IncludeSystemCaCertsPool bool `mapstructure:"include_system_ca_certs_pool"` - minTLS, err := convertVersion(c.MinVersion, defaultMinTLSVersion) - if err != nil { - return nil, fmt.Errorf("invalid TLS min_version: %w", err) - } - maxTLS, err := convertVersion(c.MaxVersion, defaultMaxTLSVersion) - if err != nil { - return nil, fmt.Errorf("invalid TLS max_version: %w", err) - } - cipherSuites, err := convertCipherSuites(c.CipherSuites) - if err != nil { - return nil, err - } + // Insecure corresponds to the JSON schema field "insecure". + Insecure bool `mapstructure:"insecure"` - return &tls.Config{ - RootCAs: certPool, - GetCertificate: getCertificate, - GetClientCertificate: getClientCertificate, - MinVersion: minTLS, - MaxVersion: maxTLS, - CipherSuites: cipherSuites, - }, nil -} + // InsecureSkipVerify corresponds to the JSON schema field "insecure_skip_verify". + InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"` -func convertCipherSuites(cipherSuites []string) ([]uint16, error) { - var result []uint16 - var errs []error - for _, suite := range cipherSuites { - found := false - for _, supported := range tls.CipherSuites() { - if suite == supported.Name { - result = append(result, supported.ID) - found = true - break - } - } - if !found { - errs = append(errs, fmt.Errorf("invalid TLS cipher suite: %q", suite)) - } - } - return result, errors.Join(errs...) -} + // KeyFile corresponds to the JSON schema field "key_file". + KeyFile string `mapstructure:"key_file"` -func (c Config) loadCACertPool() (*x509.CertPool, error) { - // There is no need to load the System Certs for RootCAs because - // if the value is nil, it will default to checking against th System Certs. - var err error - var certPool *x509.CertPool + // KeyPem corresponds to the JSON schema field "key_pem". + KeyPem configopaque.String `mapstructure:"key_pem"` - switch { - case c.hasCAFile() && c.hasCAPem(): - return nil, fmt.Errorf("failed to load CA CertPool: provide either a CA file or the PEM-encoded string, but not both") - case c.hasCAFile(): - // Set up user specified truststore from file - certPool, err = c.loadCertFile(c.CAFile) - if err != nil { - return nil, fmt.Errorf("failed to load CA CertPool File: %w", err) - } - case c.hasCAPem(): - // Set up user specified truststore from PEM - certPool, err = c.loadCertPem([]byte(c.CAPem)) - if err != nil { - return nil, fmt.Errorf("failed to load CA CertPool PEM: %w", err) - } - } + // MaxVersion corresponds to the JSON schema field "max_version". + MaxVersion string `mapstructure:"max_version"` - return certPool, nil -} + // MinVersion corresponds to the JSON schema field "min_version". + MinVersion string `mapstructure:"min_version"` -func (c Config) loadCertFile(certPath string) (*x509.CertPool, error) { - certPem, err := os.ReadFile(filepath.Clean(certPath)) - if err != nil { - return nil, fmt.Errorf("failed to load cert %s: %w", certPath, err) - } + // ReloadInterval corresponds to the JSON schema field "reload_interval". + ReloadInterval time.Duration `mapstructure:"reload_interval"` - return c.loadCertPem(certPem) + // ServerNameOverride corresponds to the JSON schema field "server_name_override". + ServerNameOverride string `mapstructure:"server_name_override"` } -func (c Config) loadCertPem(certPem []byte) (*x509.CertPool, error) { - certPool := x509.NewCertPool() - if c.IncludeSystemCACertsPool { - scp, err := systemCertPool() - if err != nil { - return nil, err - } - if scp != nil { - certPool = scp - } +// UnmarshalJSON implements json.Unmarshaler. +func (j *Config) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - if !certPool.AppendCertsFromPEM(certPem) { - return nil, fmt.Errorf("failed to parse cert") + type Plain Config + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - return certPool, nil -} - -func (c Config) loadCertificate() (tls.Certificate, error) { - switch { - case c.hasCert() != c.hasKey(): - return tls.Certificate{}, fmt.Errorf("for auth via TLS, provide both certificate and key, or neither") - case !c.hasCert() && !c.hasKey(): - return tls.Certificate{}, nil - case c.hasCertFile() && c.hasCertPem(): - return tls.Certificate{}, fmt.Errorf("for auth via TLS, provide either a certificate or the PEM-encoded string, but not both") - case c.hasKeyFile() && c.hasKeyPem(): - return tls.Certificate{}, fmt.Errorf("for auth via TLS, provide either a key or the PEM-encoded string, but not both") + if v, ok := raw["ca_file"]; !ok || v == nil { + plain.CaFile = "" } - - var certPem, keyPem []byte - var err error - if c.hasCertFile() { - certPem, err = os.ReadFile(c.CertFile) - if err != nil { - return tls.Certificate{}, err - } - } else { - certPem = []byte(c.CertPem) + if v, ok := raw["ca_pem"]; !ok || v == nil { + plain.CaPem = "" } - - if c.hasKeyFile() { - keyPem, err = os.ReadFile(c.KeyFile) - if err != nil { - return tls.Certificate{}, err - } - } else { - keyPem = []byte(c.KeyPem) + if v, ok := raw["cert_file"]; !ok || v == nil { + plain.CertFile = "" } - - certificate, err := tls.X509KeyPair(certPem, keyPem) - if err != nil { - return tls.Certificate{}, fmt.Errorf("failed to load TLS cert and key PEMs: %w", err) + if v, ok := raw["cert_pem"]; !ok || v == nil { + plain.CertPem = "" } - - return certificate, err -} - -func (c Config) loadCert(caPath string) (*x509.CertPool, error) { - caPEM, err := os.ReadFile(filepath.Clean(caPath)) - if err != nil { - return nil, fmt.Errorf("failed to load CA %s: %w", caPath, err) + if v, ok := raw["include_system_ca_certs_pool"]; !ok || v == nil { + plain.IncludeSystemCaCertsPool = true } - - var certPool *x509.CertPool - if c.IncludeSystemCACertsPool { - if certPool, err = systemCertPool(); err != nil { - return nil, err - } + if v, ok := raw["insecure"]; !ok || v == nil { + plain.Insecure = false } - if certPool == nil { - certPool = x509.NewCertPool() + if v, ok := raw["insecure_skip_verify"]; !ok || v == nil { + plain.InsecureSkipVerify = false } - if !certPool.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("failed to parse CA %s", caPath) + if v, ok := raw["key_file"]; !ok || v == nil { + plain.KeyFile = "" } - return certPool, nil -} - -// LoadTLSConfig loads the TLS configuration. -func (c ClientConfig) LoadTLSConfig(_ context.Context) (*tls.Config, error) { - if c.Insecure && !c.hasCA() { - return nil, nil + if v, ok := raw["key_pem"]; !ok || v == nil { + plain.KeyPem = "" } - - tlsCfg, err := c.loadTLSConfig() - if err != nil { - return nil, fmt.Errorf("failed to load TLS config: %w", err) + if v, ok := raw["max_version"]; !ok || v == nil { + plain.MaxVersion = "" } - tlsCfg.ServerName = c.ServerName - tlsCfg.InsecureSkipVerify = c.InsecureSkipVerify - return tlsCfg, nil -} - -// LoadTLSConfig loads the TLS configuration. -func (c ServerConfig) LoadTLSConfig(_ context.Context) (*tls.Config, error) { - tlsCfg, err := c.loadTLSConfig() - if err != nil { - return nil, fmt.Errorf("failed to load TLS config: %w", err) + if v, ok := raw["min_version"]; !ok || v == nil { + plain.MinVersion = "" } - if c.ClientCAFile != "" { - reloader, err := newClientCAsReloader(c.ClientCAFile, &c) + if v, ok := raw["reload_interval"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("33.3s") if err != nil { - return nil, err + return fmt.Errorf("failed to parse the \"33.3s\" default value for field reload_interval:%w }", err) } - if c.ReloadClientCAFile { - err = reloader.startWatching() - if err != nil { - return nil, err - } - tlsCfg.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { return reloader.getClientConfig(tlsCfg) } - } - tlsCfg.ClientCAs = reloader.certPool - tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert + plain.ReloadInterval = defaultDuration + } + if v, ok := raw["server_name_override"]; !ok || v == nil { + plain.ServerNameOverride = "" } - return tlsCfg, nil + *j = Config(plain) + return nil } -func (c ServerConfig) loadClientCAFile() (*x509.CertPool, error) { - return c.loadCert(c.ClientCAFile) +// SetDefaults sets the fields of Config to their defaults. +// Fields which do not have a default value are left untouched. +func (c *Config) SetDefaults() { + c.CaFile = "" + c.CaPem = "" + c.CertFile = "" + c.CertPem = "" + c.IncludeSystemCaCertsPool = true + c.Insecure = false + c.InsecureSkipVerify = false + c.KeyFile = "" + c.KeyPem = "" + c.MaxVersion = "" + c.MinVersion = "" + c.ReloadInterval = "PT33.3S" + c.ServerNameOverride = "" } -func (c Config) hasCA() bool { return c.hasCAFile() || c.hasCAPem() } -func (c Config) hasCert() bool { return c.hasCertFile() || c.hasCertPem() } -func (c Config) hasKey() bool { return c.hasKeyFile() || c.hasKeyPem() } - -func (c Config) hasCAFile() bool { return c.CAFile != "" } -func (c Config) hasCAPem() bool { return len(c.CAPem) != 0 } +type ServerConfig struct { + // ClientCaFile corresponds to the JSON schema field "client_ca_file". + ClientCaFile string `mapstructure:"client_ca_file"` -func (c Config) hasCertFile() bool { return c.CertFile != "" } -func (c Config) hasCertPem() bool { return len(c.CertPem) != 0 } + // ClientCaFileReload corresponds to the JSON schema field + // "client_ca_file_reload". + ClientCaFileReload bool `mapstructure:"client_ca_file_reload"` -func (c Config) hasKeyFile() bool { return c.KeyFile != "" } -func (c Config) hasKeyPem() bool { return len(c.KeyPem) != 0 } + Config `mapstructure:",squash"` +} -func convertVersion(v string, defaultVersion uint16) (uint16, error) { - // Use a default that is explicitly defined - if v == "" { - return defaultVersion, nil +// UnmarshalJSON implements json.Unmarshaler. +func (j *ServerConfig) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - val, ok := tlsVersions[v] - if !ok { - return 0, fmt.Errorf("unsupported TLS version: %q", v) + type Plain ServerConfig + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - return val, nil + if v, ok := raw["client_ca_file"]; !ok || v == nil { + plain.ClientCaFile = "" + } + if v, ok := raw["client_ca_file_reload"]; !ok || v == nil { + plain.ClientCaFileReload = true + } + *j = ServerConfig(plain) + return nil } -var tlsVersions = map[string]uint16{ - "1.0": tls.VersionTLS10, - "1.1": tls.VersionTLS11, - "1.2": tls.VersionTLS12, - "1.3": tls.VersionTLS13, +// SetDefaults sets the fields of ServerConfig to their defaults. +// Fields which do not have a default value are left untouched. +func (c *ServerConfig) SetDefaults() { + c.ClientCaFile = "" + c.ClientCaFileReload = true } diff --git a/config/configtls/configtls.yaml b/config/configtls/configtls.yaml new file mode 100644 index 00000000000..dd1053398a9 --- /dev/null +++ b/config/configtls/configtls.yaml @@ -0,0 +1,87 @@ +"$schema": http://json-schema.org/draft-04/schema# +id: opentelemetry.io/collector/config/configtls/configtls +$defs: + config: + type: object + title: Config + properties: + ca_file: + type: string + default: "" + ca_pem: + type: string + goType: + name: String + package: go.opentelemetry.io/collector/config/configopaque + default: "" + include_system_ca_certs_pool: + type: boolean + default: true + cert_file: + type: string + default: "" + cert_pem: + type: string + default: "" + goType: + name: String + package: go.opentelemetry.io/collector/config/configopaque + key_file: + type: string + default: "" + key_pem: + type: string + default: "" + goType: + name: String + package: go.opentelemetry.io/collector/config/configopaque + min_version: + type: string + default: "" + max_version: + type: string + default: "" + cipher_suites: + type: array + items: + type: string + reload_interval: + type: string + format: duration + default: "PT33.3S" + insecure: + type: boolean + default: false + insecure_skip_verify: + type: boolean + default: false + server_name_override: + type: string + default: "" + client_config: + type: object + title: ClientConfig + allOf: + - "$ref": "#/$defs/config" + properties: + insecure: + type: boolean + default: true + insecure_skip_verify: + type: boolean + default: true + server_name_override: + type: string + default: "" + server_config: + type: object + title: ServerConfig + allOf: + - "$ref": "#/$defs/config" + properties: + client_ca_file_reload: + type: boolean + default: true + client_ca_file: + type: string + default: "" diff --git a/config/configtls/configtls_test.go b/config/configtls/configtls_test.go index 91c0e871055..16833e6fa3d 100644 --- a/config/configtls/configtls_test.go +++ b/config/configtls/configtls_test.go @@ -51,30 +51,30 @@ func TestOptionsToConfig(t *testing.T) { }{ { name: "should load system CA", - options: Config{CAFile: ""}, + options: Config{CaFile: ""}, }, { name: "should load custom CA", - options: Config{CAFile: filepath.Join("testdata", "ca-1.crt")}, + options: Config{CaFile: filepath.Join("testdata", "ca-1.crt")}, }, { name: "should load system CA and custom CA", - options: Config{IncludeSystemCACertsPool: true, CAFile: filepath.Join("testdata", "ca-1.crt")}, + options: Config{IncludeSystemCaCertsPool: true, CaFile: filepath.Join("testdata", "ca-1.crt")}, }, { name: "should fail with invalid CA file path", - options: Config{CAFile: filepath.Join("testdata", "not/valid")}, + options: Config{CaFile: filepath.Join("testdata", "not/valid")}, expectError: "failed to load CA", }, { name: "should fail with invalid CA file content", - options: Config{CAFile: filepath.Join("testdata", "testCA-bad.txt")}, + options: Config{CaFile: filepath.Join("testdata", "testCA-bad.txt")}, expectError: "failed to parse cert", }, { name: "should load valid TLS settings", options: Config{ - CAFile: filepath.Join("testdata", "ca-1.crt"), + CaFile: filepath.Join("testdata", "ca-1.crt"), CertFile: filepath.Join("testdata", "server-1.crt"), KeyFile: filepath.Join("testdata", "server-1.key"), }, @@ -82,7 +82,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail with missing TLS KeyFile", options: Config{ - CAFile: filepath.Join("testdata", "ca-1.crt"), + CaFile: filepath.Join("testdata", "ca-1.crt"), CertFile: filepath.Join("testdata", "server-1.crt"), }, expectError: "provide both certificate and key, or neither", @@ -90,7 +90,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail with invalid TLS KeyFile", options: Config{ - CAFile: filepath.Join("testdata", "ca-1.crt"), + CaFile: filepath.Join("testdata", "ca-1.crt"), CertFile: filepath.Join("testdata", "server-1.crt"), KeyFile: filepath.Join("testdata", "not/valid"), }, @@ -99,7 +99,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail with missing TLS Cert", options: Config{ - CAFile: filepath.Join("testdata", "ca-1.crt"), + CaFile: filepath.Join("testdata", "ca-1.crt"), KeyFile: filepath.Join("testdata", "server-1.key"), }, expectError: "provide both certificate and key, or neither", @@ -107,7 +107,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail with invalid TLS Cert", options: Config{ - CAFile: filepath.Join("testdata", "ca-1.crt"), + CaFile: filepath.Join("testdata", "ca-1.crt"), CertFile: filepath.Join("testdata", "not/valid"), KeyFile: filepath.Join("testdata", "server-1.key"), }, @@ -116,21 +116,21 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail with invalid TLS CA", options: Config{ - CAFile: filepath.Join("testdata", "not/valid"), + CaFile: filepath.Join("testdata", "not/valid"), }, expectError: "failed to load CA", }, { name: "should fail with invalid CA pool", options: Config{ - CAFile: filepath.Join("testdata", "testCA-bad.txt"), + CaFile: filepath.Join("testdata", "testCA-bad.txt"), }, expectError: "failed to parse cert", }, { name: "should pass with valid CA pool", options: Config{ - CAFile: filepath.Join("testdata", "ca-1.crt"), + CaFile: filepath.Join("testdata", "ca-1.crt"), }, }, { @@ -156,12 +156,12 @@ func TestOptionsToConfig(t *testing.T) { }, { name: "should load custom CA PEM", - options: Config{CAPem: readFilePanics("testdata/ca-1.crt")}, + options: Config{CaPem: readFilePanics("testdata/ca-1.crt")}, }, { name: "should load valid TLS settings with PEMs", options: Config{ - CAPem: readFilePanics("testdata/ca-1.crt"), + CaPem: readFilePanics("testdata/ca-1.crt"), CertPem: readFilePanics("testdata/server-1.crt"), KeyPem: readFilePanics("testdata/server-1.key"), }, @@ -182,14 +182,14 @@ func TestOptionsToConfig(t *testing.T) { }, { name: "should fail with invalid CA PEM", - options: Config{CAPem: readFilePanics("testdata/testCA-bad.txt")}, + options: Config{CaPem: readFilePanics("testdata/testCA-bad.txt")}, expectError: "failed to parse cert", }, { name: "should fail CA file and PEM both provided", options: Config{ - CAFile: "testdata/ca-1.crt", - CAPem: readFilePanics("testdata/ca-1.crt"), + CaFile: "testdata/ca-1.crt", + CaPem: readFilePanics("testdata/ca-1.crt"), }, expectError: "provide either a CA file or the PEM-encoded string, but not both", }, @@ -214,7 +214,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail to load valid TLS settings with bad Cert PEM", options: Config{ - CAPem: readFilePanics("testdata/ca-1.crt"), + CaPem: readFilePanics("testdata/ca-1.crt"), CertPem: readFilePanics("testdata/testCA-bad.txt"), KeyPem: readFilePanics("testdata/server-1.key"), }, @@ -223,7 +223,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail to load valid TLS settings with bad Key PEM", options: Config{ - CAPem: readFilePanics("testdata/ca-1.crt"), + CaPem: readFilePanics("testdata/ca-1.crt"), CertPem: readFilePanics("testdata/server-1.crt"), KeyPem: readFilePanics("testdata/testCA-bad.txt"), }, @@ -232,7 +232,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail with missing TLS KeyPem", options: Config{ - CAPem: readFilePanics("testdata/ca-1.crt"), + CaPem: readFilePanics("testdata/ca-1.crt"), CertPem: readFilePanics("testdata/server-1.crt"), }, expectError: "provide both certificate and key, or neither", @@ -240,7 +240,7 @@ func TestOptionsToConfig(t *testing.T) { { name: "should fail with missing TLS Cert PEM", options: Config{ - CAPem: readFilePanics("testdata/ca-1.crt"), + CaPem: readFilePanics("testdata/ca-1.crt"), KeyPem: readFilePanics("testdata/server-1.key"), }, expectError: "provide both certificate and key, or neither", @@ -314,7 +314,7 @@ func TestLoadTLSServerConfigError(t *testing.T) { assert.Error(t, err) tlsSetting = ServerConfig{ - ClientCAFile: "doesnt/exist", + ClientCaFile: "doesnt/exist", } _, err = tlsSetting.LoadTLSConfig(context.Background()) assert.Error(t, err) @@ -334,8 +334,8 @@ func TestLoadTLSServerConfigReload(t *testing.T) { overwriteClientCA(t, tmpCaPath, "ca-1.crt") tlsSetting := ServerConfig{ - ClientCAFile: tmpCaPath, - ReloadClientCAFile: true, + ClientCaFile: tmpCaPath, + ClientCaFileReload: true, } tlsCfg, err := tlsSetting.LoadTLSConfig(context.Background()) @@ -365,8 +365,8 @@ func TestLoadTLSServerConfigFailingReload(t *testing.T) { overwriteClientCA(t, tmpCaPath, "ca-1.crt") tlsSetting := ServerConfig{ - ClientCAFile: tmpCaPath, - ReloadClientCAFile: true, + ClientCaFile: tmpCaPath, + ClientCaFileReload: true, } tlsCfg, err := tlsSetting.LoadTLSConfig(context.Background()) @@ -396,8 +396,8 @@ func TestLoadTLSServerConfigFailingInitialLoad(t *testing.T) { overwriteClientCA(t, tmpCaPath, "testCA-bad.txt") tlsSetting := ServerConfig{ - ClientCAFile: tmpCaPath, - ReloadClientCAFile: true, + ClientCaFile: tmpCaPath, + ClientCaFileReload: true, } tlsCfg, err := tlsSetting.LoadTLSConfig(context.Background()) @@ -410,8 +410,8 @@ func TestLoadTLSServerConfigWrongPath(t *testing.T) { tmpCaPath := createTempClientCaFile(t) tlsSetting := ServerConfig{ - ClientCAFile: tmpCaPath + "wrong-path", - ReloadClientCAFile: true, + ClientCaFile: tmpCaPath + "wrong-path", + ClientCaFileReload: true, } tlsCfg, err := tlsSetting.LoadTLSConfig(context.Background()) @@ -426,8 +426,8 @@ func TestLoadTLSServerConfigFailing(t *testing.T) { overwriteClientCA(t, tmpCaPath, "ca-1.crt") tlsSetting := ServerConfig{ - ClientCAFile: tmpCaPath, - ReloadClientCAFile: true, + ClientCaFile: tmpCaPath, + ClientCaFileReload: true, } tlsCfg, err := tlsSetting.LoadTLSConfig(context.Background()) @@ -671,7 +671,7 @@ func TestConfigValidate(t *testing.T) { {name: `TLS Config ["", "asd"] to give [Error]`, tlsConfig: Config{MinVersion: "", MaxVersion: "asd"}, errorTxt: `invalid TLS max_version: unsupported TLS version: "asd"`}, {name: `TLS Config ["0.4", ""] to give [Error]`, tlsConfig: Config{MinVersion: "0.4", MaxVersion: ""}, errorTxt: `invalid TLS min_version: unsupported TLS version: "0.4"`}, {name: `TLS Config ["1.2", "1.1"] to give [Error]`, tlsConfig: Config{MinVersion: "1.2", MaxVersion: "1.1"}, errorTxt: `invalid TLS configuration: min_version cannot be greater than max_version`}, - {name: `TLS Config with both CA File and PEM`, tlsConfig: Config{CAFile: "test", CAPem: "test"}, errorTxt: `provide either a CA file or the PEM-encoded string, but not both`}, + {name: `TLS Config with both CA File and PEM`, tlsConfig: Config{CaFile: "test", CaPem: "test"}, errorTxt: `provide either a CA file or the PEM-encoded string, but not both`}, } for _, test := range tests { @@ -747,8 +747,8 @@ func TestSystemCertPool(t *testing.T) { { name: "not using system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: false, - CAFile: filepath.Join("testdata", "ca-1.crt"), + IncludeSystemCaCertsPool: false, + CaFile: filepath.Join("testdata", "ca-1.crt"), }, wantErr: nil, systemCertFn: x509.SystemCertPool, @@ -756,8 +756,8 @@ func TestSystemCertPool(t *testing.T) { { name: "using system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: true, - CAFile: filepath.Join("testdata", "ca-1.crt"), + IncludeSystemCaCertsPool: true, + CaFile: filepath.Join("testdata", "ca-1.crt"), }, wantErr: nil, systemCertFn: x509.SystemCertPool, @@ -765,8 +765,8 @@ func TestSystemCertPool(t *testing.T) { { name: "error loading system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: true, - CAFile: filepath.Join("testdata", "ca-1.crt"), + IncludeSystemCaCertsPool: true, + CaFile: filepath.Join("testdata", "ca-1.crt"), }, wantErr: anError, systemCertFn: func() (*x509.CertPool, error) { @@ -776,8 +776,8 @@ func TestSystemCertPool(t *testing.T) { { name: "nil system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: true, - CAFile: filepath.Join("testdata", "ca-1.crt"), + IncludeSystemCaCertsPool: true, + CaFile: filepath.Join("testdata", "ca-1.crt"), }, wantErr: nil, systemCertFn: func() (*x509.CertPool, error) { @@ -827,7 +827,7 @@ func TestSystemCertPool_loadCert(t *testing.T) { { name: "not using system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: false, + IncludeSystemCaCertsPool: false, }, wantErr: nil, systemCertFn: x509.SystemCertPool, @@ -835,7 +835,7 @@ func TestSystemCertPool_loadCert(t *testing.T) { { name: "using system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: true, + IncludeSystemCaCertsPool: true, }, wantErr: nil, systemCertFn: x509.SystemCertPool, @@ -843,7 +843,7 @@ func TestSystemCertPool_loadCert(t *testing.T) { { name: "error loading system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: true, + IncludeSystemCaCertsPool: true, }, wantErr: anError, systemCertFn: func() (*x509.CertPool, error) { @@ -853,7 +853,7 @@ func TestSystemCertPool_loadCert(t *testing.T) { { name: "nil system cert pool", tlsConfig: Config{ - IncludeSystemCACertsPool: true, + IncludeSystemCaCertsPool: true, }, wantErr: nil, systemCertFn: func() (*x509.CertPool, error) { diff --git a/config/configtls/configtls_util.go b/config/configtls/configtls_util.go new file mode 100644 index 00000000000..cc417301cb7 --- /dev/null +++ b/config/configtls/configtls_util.go @@ -0,0 +1,334 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +package configtls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" +) + +var systemCertPool = x509.SystemCertPool + +// certReloader is a wrapper object for certificate reloading +// Its GetCertificate method will either return the current certificate or reload from disk +// if the last reload happened more than ReloadInterval ago +type certReloader struct { + nextReload time.Time + cert *tls.Certificate + lock sync.RWMutex + tls Config +} + +func (c Config) newCertReloader() (*certReloader, error) { + cert, err := c.loadCertificate() + if err != nil { + return nil, err + } + return &certReloader{ + tls: c, + nextReload: time.Now().Add(c.ReloadInterval), + cert: &cert, + }, nil +} + +func (r *certReloader) GetCertificate() (*tls.Certificate, error) { + now := time.Now() + // Read locking here before we do the time comparison + // If a reload is in progress this will block and we will skip reloading in the current + // call once we can continue + r.lock.RLock() + if r.tls.ReloadInterval != 0 && r.nextReload.Before(now) && (r.tls.hasCertFile() || r.tls.hasKeyFile()) { + // Need to release the read lock, otherwise we deadlock + r.lock.RUnlock() + r.lock.Lock() + defer r.lock.Unlock() + cert, err := r.tls.loadCertificate() + if err != nil { + return nil, fmt.Errorf("failed to load TLS cert and key: %w", err) + } + r.cert = &cert + r.nextReload = now.Add(r.tls.ReloadInterval) + return r.cert, nil + } + defer r.lock.RUnlock() + return r.cert, nil +} + +func (c Config) Validate() error { + if c.hasCAFile() && c.hasCAPem() { + return fmt.Errorf("provide either a CA file or the PEM-encoded string, but not both") + } + + minTLS, err := convertVersion(c.MinVersion, defaultMinTLSVersion) + if err != nil { + return fmt.Errorf("invalid TLS min_version: %w", err) + } + + maxTLS, err := convertVersion(c.MaxVersion, defaultMaxTLSVersion) + if err != nil { + return fmt.Errorf("invalid TLS max_version: %w", err) + } + + if maxTLS < minTLS && maxTLS != defaultMaxTLSVersion { + return errors.New("invalid TLS configuration: min_version cannot be greater than max_version") + } + + return nil +} + +// loadTLSConfig loads TLS certificates and returns a tls.Config. +// This will set the RootCAs and Certificates of a tls.Config. +func (c Config) loadTLSConfig() (*tls.Config, error) { + certPool, err := c.loadCACertPool() + if err != nil { + return nil, err + } + + var getCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error) + var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + if c.hasCert() || c.hasKey() { + var certReloader *certReloader + certReloader, err = c.newCertReloader() + if err != nil { + return nil, fmt.Errorf("failed to load TLS cert and key: %w", err) + } + getCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return certReloader.GetCertificate() } + getClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return certReloader.GetCertificate() } + } + + minTLS, err := convertVersion(c.MinVersion, defaultMinTLSVersion) + if err != nil { + return nil, fmt.Errorf("invalid TLS min_version: %w", err) + } + maxTLS, err := convertVersion(c.MaxVersion, defaultMaxTLSVersion) + if err != nil { + return nil, fmt.Errorf("invalid TLS max_version: %w", err) + } + cipherSuites, err := convertCipherSuites(c.CipherSuites) + if err != nil { + return nil, err + } + + return &tls.Config{ + RootCAs: certPool, + GetCertificate: getCertificate, + GetClientCertificate: getClientCertificate, + MinVersion: minTLS, + MaxVersion: maxTLS, + CipherSuites: cipherSuites, + }, nil +} + +func convertCipherSuites(cipherSuites []string) ([]uint16, error) { + var result []uint16 + var errs []error + for _, suite := range cipherSuites { + found := false + for _, supported := range tls.CipherSuites() { + if suite == supported.Name { + result = append(result, supported.ID) + found = true + break + } + } + if !found { + errs = append(errs, fmt.Errorf("invalid TLS cipher suite: %q", suite)) + } + } + return result, errors.Join(errs...) +} + +func (c Config) loadCACertPool() (*x509.CertPool, error) { + // There is no need to load the System Certs for RootCAs because + // if the value is nil, it will default to checking against th System Certs. + var err error + var certPool *x509.CertPool + + switch { + case c.hasCAFile() && c.hasCAPem(): + return nil, fmt.Errorf("failed to load CA CertPool: provide either a CA file or the PEM-encoded string, but not both") + case c.hasCAFile(): + // Set up user specified truststore from file + certPool, err = c.loadCertFile(c.CaFile) + if err != nil { + return nil, fmt.Errorf("failed to load CA CertPool File: %w", err) + } + case c.hasCAPem(): + // Set up user specified truststore from PEM + certPool, err = c.loadCertPem([]byte(c.CaPem)) + if err != nil { + return nil, fmt.Errorf("failed to load CA CertPool PEM: %w", err) + } + } + + return certPool, nil +} + +func (c Config) loadCertFile(certPath string) (*x509.CertPool, error) { + certPem, err := os.ReadFile(filepath.Clean(certPath)) + if err != nil { + return nil, fmt.Errorf("failed to load cert %s: %w", certPath, err) + } + + return c.loadCertPem(certPem) +} + +func (c Config) loadCertPem(certPem []byte) (*x509.CertPool, error) { + certPool := x509.NewCertPool() + if c.IncludeSystemCaCertsPool { + scp, err := systemCertPool() + if err != nil { + return nil, err + } + if scp != nil { + certPool = scp + } + } + if !certPool.AppendCertsFromPEM(certPem) { + return nil, fmt.Errorf("failed to parse cert") + } + return certPool, nil +} + +func (c Config) loadCertificate() (tls.Certificate, error) { + switch { + case c.hasCert() != c.hasKey(): + return tls.Certificate{}, fmt.Errorf("for auth via TLS, provide both certificate and key, or neither") + case !c.hasCert() && !c.hasKey(): + return tls.Certificate{}, nil + case c.hasCertFile() && c.hasCertPem(): + return tls.Certificate{}, fmt.Errorf("for auth via TLS, provide either a certificate or the PEM-encoded string, but not both") + case c.hasKeyFile() && c.hasKeyPem(): + return tls.Certificate{}, fmt.Errorf("for auth via TLS, provide either a key or the PEM-encoded string, but not both") + } + + var certPem, keyPem []byte + var err error + if c.hasCertFile() { + certPem, err = os.ReadFile(c.CertFile) + if err != nil { + return tls.Certificate{}, err + } + } else { + certPem = []byte(c.CertPem) + } + + if c.hasKeyFile() { + keyPem, err = os.ReadFile(c.KeyFile) + if err != nil { + return tls.Certificate{}, err + } + } else { + keyPem = []byte(c.KeyPem) + } + + certificate, err := tls.X509KeyPair(certPem, keyPem) + if err != nil { + return tls.Certificate{}, fmt.Errorf("failed to load TLS cert and key PEMs: %w", err) + } + + return certificate, err +} + +func (c Config) loadCert(caPath string) (*x509.CertPool, error) { + caPEM, err := os.ReadFile(filepath.Clean(caPath)) + if err != nil { + return nil, fmt.Errorf("failed to load CA %s: %w", caPath, err) + } + + var certPool *x509.CertPool + if c.IncludeSystemCaCertsPool { + if certPool, err = systemCertPool(); err != nil { + return nil, err + } + } + if certPool == nil { + certPool = x509.NewCertPool() + } + if !certPool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("failed to parse CA %s", caPath) + } + return certPool, nil +} + +// LoadTLSConfig loads the TLS configuration. +func (c ClientConfig) LoadTLSConfig(_ context.Context) (*tls.Config, error) { + if c.Insecure && !c.hasCA() { + return nil, nil + } + + tlsCfg, err := c.loadTLSConfig() + if err != nil { + return nil, fmt.Errorf("failed to load TLS config: %w", err) + } + tlsCfg.ServerName = c.ServerNameOverride + tlsCfg.InsecureSkipVerify = c.InsecureSkipVerify + return tlsCfg, nil +} + +// LoadTLSConfig loads the TLS configuration. +func (c ServerConfig) LoadTLSConfig(_ context.Context) (*tls.Config, error) { + tlsCfg, err := c.loadTLSConfig() + if err != nil { + return nil, fmt.Errorf("failed to load TLS config: %w", err) + } + if c.ClientCaFile != "" { + reloader, err := newClientCAsReloader(c.ClientCaFile, &c) + if err != nil { + return nil, err + } + if c.ClientCaFileReload { + err = reloader.startWatching() + if err != nil { + return nil, err + } + tlsCfg.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { return reloader.getClientConfig(tlsCfg) } + } + tlsCfg.ClientCAs = reloader.certPool + tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert + } + return tlsCfg, nil +} + +func (c ServerConfig) loadClientCAFile() (*x509.CertPool, error) { + return c.loadCert(c.ClientCaFile) +} + +func (c Config) hasCA() bool { return c.hasCAFile() || c.hasCAPem() } +func (c Config) hasCert() bool { return c.hasCertFile() || c.hasCertPem() } +func (c Config) hasKey() bool { return c.hasKeyFile() || c.hasKeyPem() } + +func (c Config) hasCAFile() bool { return c.CaFile != "" } +func (c Config) hasCAPem() bool { return len(c.CaPem) != 0 } + +func (c Config) hasCertFile() bool { return c.CertFile != "" } +func (c Config) hasCertPem() bool { return len(c.CertPem) != 0 } + +func (c Config) hasKeyFile() bool { return c.KeyFile != "" } +func (c Config) hasKeyPem() bool { return len(c.KeyPem) != 0 } + +func convertVersion(v string, defaultVersion uint16) (uint16, error) { + // Use a default that is explicitly defined + if v == "" { + return defaultVersion, nil + } + val, ok := tlsVersions[v] + if !ok { + return 0, fmt.Errorf("unsupported TLS version: %q", v) + } + return val, nil +} + +var tlsVersions = map[string]uint16{ + "1.0": tls.VersionTLS10, + "1.1": tls.VersionTLS11, + "1.2": tls.VersionTLS12, + "1.3": tls.VersionTLS13, +} diff --git a/exporter/exporterhelper/queue_sender.go b/exporter/exporterhelper/queue_sender.go index 060edab813a..1b29e400bea 100644 --- a/exporter/exporterhelper/queue_sender.go +++ b/exporter/exporterhelper/queue_sender.go @@ -1,137 +1,51 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. -package exporterhelper // import "go.opentelemetry.io/collector/exporter/exporterhelper" +package exporterhelper -import ( - "context" - "errors" +import "encoding/json" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/trace" - "go.uber.org/multierr" - "go.uber.org/zap" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/exporter" - "go.opentelemetry.io/collector/exporter/exporterqueue" - "go.opentelemetry.io/collector/exporter/internal/queue" - "go.opentelemetry.io/collector/internal/obsreportconfig/obsmetrics" -) +type SendingQueue struct { + // Enabled corresponds to the JSON schema field "enabled". + Enabled bool `mapstructure:"enabled"` -const defaultQueueSize = 1000 + // NumWorkers corresponds to the JSON schema field "num_workers". + NumWorkers *int `mapstructure:"num_workers"` -// QueueSettings defines configuration for queueing batches before sending to the consumerSender. -type QueueSettings struct { - // Enabled indicates whether to not enqueue batches before sending to the consumerSender. - Enabled bool `mapstructure:"enabled"` - // NumConsumers is the number of consumers from the queue. Defaults to 10. - // If batching is enabled, a combined batch cannot contain more requests than the number of consumers. - // So it's recommended to set higher number of consumers if batching is enabled. - NumConsumers int `mapstructure:"num_consumers"` - // QueueSize is the maximum number of batches allowed in queue at a given time. + // QueueSize corresponds to the JSON schema field "queue_size". QueueSize int `mapstructure:"queue_size"` - // StorageID if not empty, enables the persistent storage and uses the component specified - // as a storage extension for the persistent queue - StorageID *component.ID `mapstructure:"storage"` -} -// NewDefaultQueueSettings returns the default settings for QueueSettings. -func NewDefaultQueueSettings() QueueSettings { - return QueueSettings{ - Enabled: true, - NumConsumers: 10, - // By default, batches are 8192 spans, for a total of up to 8 million spans in the queue - // This can be estimated at 1-4 GB worth of maximum memory usage - // This default is probably still too high, and may be adjusted further down in a future release - QueueSize: defaultQueueSize, - } + // Storage corresponds to the JSON schema field "storage". + Storage string `mapstructure:"storage"` } -// Validate checks if the QueueSettings configuration is valid -func (qCfg *QueueSettings) Validate() error { - if !qCfg.Enabled { - return nil - } - - if qCfg.QueueSize <= 0 { - return errors.New("queue size must be positive") +// UnmarshalJSON implements json.Unmarshaler. +func (j *SendingQueue) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - - if qCfg.NumConsumers <= 0 { - return errors.New("number of queue consumers must be positive") + type Plain SendingQueue + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } - - return nil -} - -type queueSender struct { - baseRequestSender - queue exporterqueue.Queue[Request] - numConsumers int - traceAttribute attribute.KeyValue - consumers *queue.Consumers[Request] - - obsrep *obsReport - exporterID component.ID -} - -func newQueueSender(q exporterqueue.Queue[Request], set exporter.Settings, numConsumers int, - exportFailureMessage string, obsrep *obsReport) *queueSender { - qs := &queueSender{ - queue: q, - numConsumers: numConsumers, - traceAttribute: attribute.String(obsmetrics.ExporterKey, set.ID.String()), - obsrep: obsrep, - exporterID: set.ID, + if v, ok := raw["enabled"]; !ok || v == nil { + plain.Enabled = true } - consumeFunc := func(ctx context.Context, req Request) error { - err := qs.nextSender.send(ctx, req) - if err != nil { - set.Logger.Error("Exporting failed. Dropping data."+exportFailureMessage, - zap.Error(err), zap.Int("dropped_items", req.ItemsCount())) - } - return err + if v, ok := raw["queue_size"]; !ok || v == nil { + plain.QueueSize = 1000.0 } - qs.consumers = queue.NewQueueConsumers[Request](q, numConsumers, consumeFunc) - return qs -} - -// Start is invoked during service startup. -func (qs *queueSender) Start(ctx context.Context, host component.Host) error { - if err := qs.consumers.Start(ctx, host); err != nil { - return err + if v, ok := raw["storage"]; !ok || v == nil { + plain.Storage = "" } - - dataTypeAttr := attribute.String(obsmetrics.DataTypeKey, qs.obsrep.dataType.String()) - return multierr.Append( - qs.obsrep.telemetryBuilder.InitExporterQueueSize(func() int64 { return int64(qs.queue.Size()) }, - metric.WithAttributeSet(attribute.NewSet(qs.traceAttribute, dataTypeAttr))), - qs.obsrep.telemetryBuilder.InitExporterQueueCapacity(func() int64 { return int64(qs.queue.Capacity()) }, - metric.WithAttributeSet(attribute.NewSet(qs.traceAttribute))), - ) -} - -// Shutdown is invoked during service shutdown. -func (qs *queueSender) Shutdown(ctx context.Context) error { - // Stop the queue and consumers, this will drain the queue and will call the retry (which is stopped) that will only - // try once every request. - return qs.consumers.Shutdown(ctx) + *j = SendingQueue(plain) + return nil } -// send implements the requestSender interface. It puts the request in the queue. -func (qs *queueSender) send(ctx context.Context, req Request) error { - // Prevent cancellation and deadline to propagate to the context stored in the queue. - // The grpc/http based receivers will cancel the request context after this function returns. - c := context.WithoutCancel(ctx) - - span := trace.SpanFromContext(c) - if err := qs.queue.Offer(c, req); err != nil { - span.AddEvent("Failed to enqueue item.", trace.WithAttributes(qs.traceAttribute)) - return err - } - - span.AddEvent("Enqueued item.", trace.WithAttributes(qs.traceAttribute)) - return nil +// SetDefaults sets the fields of SendingQueue to their defaults. +// Fields which do not have a default value are left untouched. +func (c *SendingQueue) SetDefaults() { + c.Enabled = true + c.QueueSize = 1000.0 + c.Storage = "" } diff --git a/exporter/exporterhelper/queue_sender.yaml b/exporter/exporterhelper/queue_sender.yaml new file mode 100644 index 00000000000..cf3da497517 --- /dev/null +++ b/exporter/exporterhelper/queue_sender.yaml @@ -0,0 +1,19 @@ +$schema: http://json-schema.org/draft-04/schema# +id: opentelemetry.io/collector/exporter/exporterhelper/queue_sender +type: object +$defs: + sending_queue: + type: object + properties: + enabled: + type: boolean + default: true + queue_size: + type: integer + default: 1000 + num_workers: + type: integer + default": 10 + storage: + type: string + default: "" \ No newline at end of file diff --git a/exporter/otlphttpexporter/config.go b/exporter/otlphttpexporter/config.go index ef59fc324a0..bae0f2b26fb 100644 --- a/exporter/otlphttpexporter/config.go +++ b/exporter/otlphttpexporter/config.go @@ -1,73 +1,99 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. -package otlphttpexporter // import "go.opentelemetry.io/collector/exporter/otlphttpexporter" +package otlphttpexporter -import ( - "encoding" - "errors" - "fmt" +import "encoding/json" +import "fmt" +import confighttp "go.opentelemetry.io/collector/config/confighttp" +import configretry "go.opentelemetry.io/collector/config/configretry" +import exporterhelper "go.opentelemetry.io/collector/exporter/exporterhelper" +import "reflect" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/confighttp" - "go.opentelemetry.io/collector/config/configretry" - "go.opentelemetry.io/collector/exporter/exporterhelper" -) +// Configuration parameters for the OTLP HTTP exporter. +type Config struct { + // Encoding corresponds to the JSON schema field "encoding". + Encoding ConfigEncoding `mapstructure:"encoding"` -// EncodingType defines the type for content encoding -type EncodingType string + // LogsEndpoint corresponds to the JSON schema field "logs_endpoint". + LogsEndpoint string `mapstructure:"logs_endpoint"` -const ( - EncodingProto EncodingType = "proto" - EncodingJSON EncodingType = "json" -) + // MetricsEndpoint corresponds to the JSON schema field "metrics_endpoint". + MetricsEndpoint string `mapstructure:"metrics_endpoint"` -var _ encoding.TextUnmarshaler = (*EncodingType)(nil) + // RetryOnFailure corresponds to the JSON schema field "retry_on_failure". + RetryOnFailure *configretry.RetryOnFailure `mapstructure:"retry_on_failure"` -// UnmarshalText unmarshalls text to an EncodingType. -func (e *EncodingType) UnmarshalText(text []byte) error { - if e == nil { - return errors.New("cannot unmarshal to a nil *EncodingType") - } + // SendingQueue corresponds to the JSON schema field "sending_queue". + SendingQueue *exporterhelper.SendingQueue `mapstructure:"sending_queue"` - str := string(text) - switch str { - case string(EncodingProto): - *e = EncodingProto - case string(EncodingJSON): - *e = EncodingJSON - default: - return fmt.Errorf("invalid encoding type: %s", str) - } + // TracesEndpoint corresponds to the JSON schema field "traces_endpoint". + TracesEndpoint string `mapstructure:"traces_endpoint"` - return nil + confighttp.ClientConfig `mapstructure:",squash"` } -// Config defines configuration for OTLP/HTTP exporter. -type Config struct { - confighttp.ClientConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. - QueueConfig exporterhelper.QueueSettings `mapstructure:"sending_queue"` - RetryConfig configretry.BackOffConfig `mapstructure:"retry_on_failure"` - - // The URL to send traces to. If omitted the Endpoint + "/v1/traces" will be used. - TracesEndpoint string `mapstructure:"traces_endpoint"` +type ConfigEncoding string - // The URL to send metrics to. If omitted the Endpoint + "/v1/metrics" will be used. - MetricsEndpoint string `mapstructure:"metrics_endpoint"` - - // The URL to send logs to. If omitted the Endpoint + "/v1/logs" will be used. - LogsEndpoint string `mapstructure:"logs_endpoint"` +const ConfigEncodingJson ConfigEncoding = "json" +const ConfigEncodingProto ConfigEncoding = "proto" - // The encoding to export telemetry (default: "proto") - Encoding EncodingType `mapstructure:"encoding"` +var enumValues_ConfigEncoding = []interface{}{ + "proto", + "json", } -var _ component.Config = (*Config)(nil) +// UnmarshalJSON implements json.Unmarshaler. +func (j *ConfigEncoding) UnmarshalJSON(b []byte) error { + var v string + if err := json.Unmarshal(b, &v); err != nil { + return err + } + var ok bool + for _, expected := range enumValues_ConfigEncoding { + if reflect.DeepEqual(v, expected) { + ok = true + break + } + } + if !ok { + return fmt.Errorf("invalid value (expected one of %#v): %#v", enumValues_ConfigEncoding, v) + } + *j = ConfigEncoding(v) + return nil +} -// Validate checks if the exporter configuration is valid -func (cfg *Config) Validate() error { - if cfg.Endpoint == "" && cfg.TracesEndpoint == "" && cfg.MetricsEndpoint == "" && cfg.LogsEndpoint == "" { - return errors.New("at least one endpoint must be specified") +// UnmarshalJSON implements json.Unmarshaler. +func (j *Config) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err + } + type Plain Config + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err } + if v, ok := raw["encoding"]; !ok || v == nil { + plain.Encoding = "proto" + } + if v, ok := raw["logs_endpoint"]; !ok || v == nil { + plain.LogsEndpoint = "" + } + if v, ok := raw["metrics_endpoint"]; !ok || v == nil { + plain.MetricsEndpoint = "" + } + if v, ok := raw["traces_endpoint"]; !ok || v == nil { + plain.TracesEndpoint = "" + } + *j = Config(plain) return nil } + +// SetDefaults sets the fields of Config to their defaults. +// Fields which do not have a default value are left untouched. +func (c *Config) SetDefaults() { + c.Encoding = "proto" + c.LogsEndpoint = "" + c.MetricsEndpoint = "" + c.TracesEndpoint = "" +} diff --git a/exporter/otlphttpexporter/config_test.go b/exporter/otlphttpexporter/config_test.go index 9a9e261308f..b65821c9d77 100644 --- a/exporter/otlphttpexporter/config_test.go +++ b/exporter/otlphttpexporter/config_test.go @@ -38,7 +38,7 @@ func TestUnmarshalConfig(t *testing.T) { assert.NoError(t, cm.Unmarshal(&cfg)) assert.Equal(t, &Config{ - RetryConfig: configretry.BackOffConfig{ + RetryOnFailure: configretry.RetryOnFailure{ Enabled: true, InitialInterval: 10 * time.Second, RandomizationFactor: 0.7, @@ -46,12 +46,12 @@ func TestUnmarshalConfig(t *testing.T) { MaxInterval: 1 * time.Minute, MaxElapsedTime: 10 * time.Minute, }, - QueueConfig: exporterhelper.QueueSettings{ + SendingQueue: exporterhelper.SendingQueue{ Enabled: true, NumConsumers: 2, QueueSize: 10, }, - Encoding: EncodingProto, + Encoding: ConfigEncodingProto, ClientConfig: confighttp.ClientConfig{ Headers: map[string]configopaque.String{ "can you have a . here?": "F0000000-0000-0000-0000-000000000000", @@ -59,12 +59,10 @@ func TestUnmarshalConfig(t *testing.T) { "another": "somevalue", }, Endpoint: "https://1.2.3.4:1234", - TLSSetting: configtls.ClientConfig{ - Config: configtls.Config{ - CAFile: "/var/lib/mycert.pem", - CertFile: "certfile", - KeyFile: "keyfile", - }, + TLSSetting: configtls.Config{ + CaFile: "/var/lib/mycert.pem", + CertFile: "certfile", + KeyFile: "keyfile", Insecure: true, }, ReadBufferSize: 123, @@ -87,19 +85,19 @@ func TestUnmarshalEncoding(t *testing.T) { tests := []struct { name string encodingBytes []byte - expected EncodingType + expected ConfigEncoding shouldError bool }{ { name: "UnmarshalEncodingProto", encodingBytes: []byte("proto"), - expected: EncodingProto, + expected: ConfigEncodingProto, shouldError: false, }, { name: "UnmarshalEncodingJson", encodingBytes: []byte("json"), - expected: EncodingJSON, + expected: ConfigEncodingJson, shouldError: false, }, { @@ -116,8 +114,8 @@ func TestUnmarshalEncoding(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var encoding EncodingType - err := encoding.UnmarshalText(tt.encodingBytes) + var encoding ConfigEncoding + err := encoding.UnmarshalJSON(tt.encodingBytes) if tt.shouldError { assert.Error(t, err) diff --git a/exporter/otlphttpexporter/factory_test.go b/exporter/otlphttpexporter/factory_test.go index ef00dfbe20d..efc466f8154 100644 --- a/exporter/otlphttpexporter/factory_test.go +++ b/exporter/otlphttpexporter/factory_test.go @@ -98,7 +98,7 @@ func TestCreateTracesExporter(t *testing.T) { Endpoint: endpoint, TLSSetting: configtls.ClientConfig{ Config: configtls.Config{ - CAFile: filepath.Join("testdata", "test_cert.pem"), + CaFile: filepath.Join("testdata", "test_cert.pem"), }, }, }, @@ -111,7 +111,7 @@ func TestCreateTracesExporter(t *testing.T) { Endpoint: endpoint, TLSSetting: configtls.ClientConfig{ Config: configtls.Config{ - CAFile: "nosuchfile", + CaFile: "nosuchfile", }, }, }, diff --git a/exporter/otlphttpexporter/generated_component_test.go b/exporter/otlphttpexporter/generated_component_test.go index 8c995ca1b34..319147ba4a4 100644 --- a/exporter/otlphttpexporter/generated_component_test.go +++ b/exporter/otlphttpexporter/generated_component_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/confmap/confmaptest" diff --git a/exporter/otlphttpexporter/generated_package_test.go b/exporter/otlphttpexporter/generated_package_test.go index 625e0cd9304..1cdc5776465 100644 --- a/exporter/otlphttpexporter/generated_package_test.go +++ b/exporter/otlphttpexporter/generated_package_test.go @@ -3,9 +3,8 @@ package otlphttpexporter import ( - "testing" - "go.uber.org/goleak" + "testing" ) func TestMain(m *testing.M) { diff --git a/exporter/otlphttpexporter/metadata.yaml b/exporter/otlphttpexporter/metadata.yaml index c5dddcf25e8..4365d271f46 100644 --- a/exporter/otlphttpexporter/metadata.yaml +++ b/exporter/otlphttpexporter/metadata.yaml @@ -10,4 +10,34 @@ status: tests: config: endpoint: "https://1.2.3.4:1234" - + +# TODO: Also schematise feature gates? +# TODO: Link to the json schema so that we get autocompletion +config: + type: object + # additionalProperties: false + description: "Configuration parameters for the OTLP HTTP exporter." + allOf: + - "$ref": "../../config/confighttp/confighttp.yaml#/$defs/client_config" + properties: + traces_endpoint: + type: string + default: "" + description: + metrics_endpoint: + type: string + default: "" + description: + logs_endpoint: + type: string + default: "" + description: + encoding: + type: string + enum: ["proto", "json"] + default: "proto" + description: + sending_queue: + "$ref": "../exporterhelper/queue_sender.yaml#/$defs/sending_queue" + retry_on_failure: + "$ref": "../../config/configretry/backoff.yaml#/$defs/retry_on_failure" \ No newline at end of file diff --git a/exporter/otlphttpexporter/otlp.go b/exporter/otlphttpexporter/otlp.go index 5c88b53c83f..f8d74f33b93 100644 --- a/exporter/otlphttpexporter/otlp.go +++ b/exporter/otlphttpexporter/otlp.go @@ -93,9 +93,9 @@ func (e *baseExporter) pushTraces(ctx context.Context, td ptrace.Traces) error { var err error var request []byte switch e.config.Encoding { - case EncodingJSON: + case ConfigEncodingJson: request, err = tr.MarshalJSON() - case EncodingProto: + case ConfigEncodingProto: request, err = tr.MarshalProto() default: err = fmt.Errorf("invalid encoding: %s", e.config.Encoding) @@ -114,9 +114,9 @@ func (e *baseExporter) pushMetrics(ctx context.Context, md pmetric.Metrics) erro var err error var request []byte switch e.config.Encoding { - case EncodingJSON: + case ConfigEncodingJson: request, err = tr.MarshalJSON() - case EncodingProto: + case ConfigEncodingProto: request, err = tr.MarshalProto() default: err = fmt.Errorf("invalid encoding: %s", e.config.Encoding) @@ -134,9 +134,9 @@ func (e *baseExporter) pushLogs(ctx context.Context, ld plog.Logs) error { var err error var request []byte switch e.config.Encoding { - case EncodingJSON: + case ConfigEncodingJson: request, err = tr.MarshalJSON() - case EncodingProto: + case ConfigEncodingProto: request, err = tr.MarshalProto() default: err = fmt.Errorf("invalid encoding: %s", e.config.Encoding) @@ -157,9 +157,9 @@ func (e *baseExporter) export(ctx context.Context, url string, request []byte, p } switch e.config.Encoding { - case EncodingJSON: + case ConfigEncodingJson: req.Header.Set("Content-Type", jsonContentType) - case EncodingProto: + case ConfigEncodingProto: req.Header.Set("Content-Type", protobufContentType) default: return fmt.Errorf("invalid encoding: %s", e.config.Encoding) diff --git a/go.mod b/go.mod index 4ead8aa092d..ff623d58f0d 100644 --- a/go.mod +++ b/go.mod @@ -118,3 +118,5 @@ replace go.opentelemetry.io/collector/pdata/pprofile => ./pdata/pprofile replace go.opentelemetry.io/collector/internal/globalgates => ./internal/globalgates replace go.opentelemetry.io/collector/consumer/consumerprofiles => ./consumer/consumerprofiles + +replace github.com/atombender/go-jsonschema => github.com/ptodev/go-jsonschema v0.0.0-20240813163654-5518ba93ee84 diff --git a/processor/batchprocessor/config.go b/processor/batchprocessor/config.go index 4d87900a17d..ad66a724ea2 100644 --- a/processor/batchprocessor/config.go +++ b/processor/batchprocessor/config.go @@ -1,68 +1,70 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. -package batchprocessor // import "go.opentelemetry.io/collector/processor/batchprocessor" +package batchprocessor -import ( - "errors" - "fmt" - "strings" - "time" +import "encoding/json" +import "fmt" +import "time" - "go.opentelemetry.io/collector/component" -) - -// Config defines configuration for batch processor. +// Configuration parameters for the batch processor. type Config struct { - // Timeout sets the time after which a batch will be sent regardless of size. - // When this is set to zero, batched data will be sent immediately. - Timeout time.Duration `mapstructure:"timeout"` + // MetadataCardinalityLimit corresponds to the JSON schema field + // "metadata_cardinality_limit". + MetadataCardinalityLimit uint32 `mapstructure:"metadata_cardinality_limit"` - // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. - // When this is set to zero, the batch size is ignored and data will be sent immediately - // subject to only send_batch_max_size. - SendBatchSize uint32 `mapstructure:"send_batch_size"` + // MetadataKeys corresponds to the JSON schema field "metadata_keys". + MetadataKeys []string `mapstructure:"metadata_keys"` - // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. - // Larger batches are split into smaller units. - // Default value is 0, that means no maximum size. + // SendBatchMaxSize corresponds to the JSON schema field "send_batch_max_size". SendBatchMaxSize uint32 `mapstructure:"send_batch_max_size"` - // MetadataKeys is a list of client.Metadata keys that will be - // used to form distinct batchers. If this setting is empty, - // a single batcher instance will be used. When this setting - // is not empty, one batcher will be used per distinct - // combination of values for the listed metadata keys. - // - // Empty value and unset metadata are treated as distinct cases. - // - // Entries are case-insensitive. Duplicated entries will - // trigger a validation error. - MetadataKeys []string `mapstructure:"metadata_keys"` + // SendBatchSize corresponds to the JSON schema field "send_batch_size". + SendBatchSize uint32 `mapstructure:"send_batch_size"` - // MetadataCardinalityLimit indicates the maximum number of - // batcher instances that will be created through a distinct - // combination of MetadataKeys. - MetadataCardinalityLimit uint32 `mapstructure:"metadata_cardinality_limit"` + // Timeout corresponds to the JSON schema field "timeout". + Timeout time.Duration `mapstructure:"timeout"` } -var _ component.Config = (*Config)(nil) - -// Validate checks if the processor configuration is valid -func (cfg *Config) Validate() error { - if cfg.SendBatchMaxSize > 0 && cfg.SendBatchMaxSize < cfg.SendBatchSize { - return errors.New("send_batch_max_size must be greater or equal to send_batch_size") +// UnmarshalJSON implements json.Unmarshaler. +func (j *Config) UnmarshalJSON(value []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(value, &raw); err != nil { + return err } - uniq := map[string]bool{} - for _, k := range cfg.MetadataKeys { - l := strings.ToLower(k) - if _, has := uniq[l]; has { - return fmt.Errorf("duplicate entry in metadata_keys: %q (case-insensitive)", l) - } - uniq[l] = true + type Plain Config + var plain Plain + if err := json.Unmarshal(value, &plain); err != nil { + return err + } + if v, ok := raw["metadata_cardinality_limit"]; !ok || v == nil { + plain.MetadataCardinalityLimit = 1000.0 + } + if v, ok := raw["metadata_keys"]; !ok || v == nil { + plain.MetadataKeys = []string{} + } + if v, ok := raw["send_batch_max_size"]; !ok || v == nil { + plain.SendBatchMaxSize = 0.0 } - if cfg.Timeout < 0 { - return errors.New("timeout must be greater or equal to 0") + if v, ok := raw["send_batch_size"]; !ok || v == nil { + plain.SendBatchSize = 8192.0 } + if v, ok := raw["timeout"]; !ok || v == nil { + defaultDuration, err := time.ParseDuration("200ms") + if err != nil { + return fmt.Errorf("failed to parse the \"200ms\" default value for field timeout:%w }", err) + } + plain.Timeout = defaultDuration + } + *j = Config(plain) return nil } + +// SetDefaults sets the fields of Config to their defaults. +// Fields which do not have a default value are left untouched. +func (c *Config) SetDefaults() { + c.MetadataCardinalityLimit = 1000.0 + c.MetadataKeys = []string{} + c.SendBatchMaxSize = 0.0 + c.SendBatchSize = 8192.0 + c.Timeout = "PT0.2S" +} diff --git a/processor/batchprocessor/config_helper.go b/processor/batchprocessor/config_helper.go new file mode 100644 index 00000000000..c260a07611b --- /dev/null +++ b/processor/batchprocessor/config_helper.go @@ -0,0 +1,47 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +package batchprocessor + +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + +func (c *Config) Validate() error { + if c.SendBatchMaxSize > 0 && c.SendBatchMaxSize < c.SendBatchSize { + return errors.New("send_batch_max_size must be greater or equal to send_batch_size") + } + + uniq := map[string]bool{} + for _, k := range c.MetadataKeys { + l := strings.ToLower(k) + if _, has := uniq[l]; has { + return fmt.Errorf("duplicate entry in metadata_keys: %q (case-insensitive)", l) + } + uniq[l] = true + } + if c.Timeout < 0 { + return errors.New("timeout must be greater or equal to 0") + } + + // Also do the regular json schema checks + c.validateSchema() + return nil +} + +// TODO: Autogenerate such a function for every config/ +func (c *Config) validateSchema() error { + buf, err := json.Marshal(c) + if err != nil { + return err + } + + var newConfig Config + err = newConfig.UnmarshalJSON(buf) + if err != nil { + return err + } + return nil +} diff --git a/processor/batchprocessor/config_test.go b/processor/batchprocessor/config_test.go index c5d3f693f92..1c0ceeb6d7f 100644 --- a/processor/batchprocessor/config_test.go +++ b/processor/batchprocessor/config_test.go @@ -34,6 +34,7 @@ func TestUnmarshalConfig(t *testing.T) { SendBatchMaxSize: uint32(11000), Timeout: time.Second * 10, MetadataCardinalityLimit: 1000, + MetadataKeys: []string{}, }, cfg) } diff --git a/processor/batchprocessor/factory.go b/processor/batchprocessor/factory.go index 12fcbb9e6ab..920ef230b34 100644 --- a/processor/batchprocessor/factory.go +++ b/processor/batchprocessor/factory.go @@ -7,7 +7,6 @@ package batchprocessor // import "go.opentelemetry.io/collector/processor/batchp import ( "context" - "time" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" @@ -15,16 +14,6 @@ import ( "go.opentelemetry.io/collector/processor/batchprocessor/internal/metadata" ) -const ( - defaultSendBatchSize = uint32(8192) - defaultTimeout = 200 * time.Millisecond - - // defaultMetadataCardinalityLimit should be set to the number - // of metadata configurations the user expects to submit to - // the collector. - defaultMetadataCardinalityLimit = 1000 -) - // NewFactory returns a new factory for the Batch processor. func NewFactory() processor.Factory { return processor.NewFactory( @@ -35,14 +24,6 @@ func NewFactory() processor.Factory { processor.WithLogs(createLogs, metadata.LogsStability)) } -func createDefaultConfig() component.Config { - return &Config{ - SendBatchSize: defaultSendBatchSize, - Timeout: defaultTimeout, - MetadataCardinalityLimit: defaultMetadataCardinalityLimit, - } -} - func createTraces( _ context.Context, set processor.Settings, diff --git a/processor/batchprocessor/generated_component_test.go b/processor/batchprocessor/generated_component_test.go index af68a5bf9a8..f4e1c220330 100644 --- a/processor/batchprocessor/generated_component_test.go +++ b/processor/batchprocessor/generated_component_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/confmap/confmaptest" diff --git a/processor/batchprocessor/generated_package_test.go b/processor/batchprocessor/generated_package_test.go index d8cd79854bc..88b07e13f0a 100644 --- a/processor/batchprocessor/generated_package_test.go +++ b/processor/batchprocessor/generated_package_test.go @@ -3,9 +3,8 @@ package batchprocessor import ( - "testing" - "go.uber.org/goleak" + "testing" ) func TestMain(m *testing.M) { diff --git a/processor/batchprocessor/metadata.yaml b/processor/batchprocessor/metadata.yaml index b0d2458c9b4..0c75a6d8cd0 100644 --- a/processor/batchprocessor/metadata.yaml +++ b/processor/batchprocessor/metadata.yaml @@ -46,3 +46,37 @@ telemetry: sum: value_type: int async: true + +config: + type: object + additionalProperties: false + description: "Configuration parameters for the batch processor." + properties: + timeout: + type: string + format: duration + default: "PT0.2S" + send_batch_size: + type: integer + minimum: 0 + default: 8192 + goType: + name: uint32 + send_batch_max_size: + type: integer + default: 0 + minimum: 0 + goType: + name: uint32 + metadata_keys: + type: array + items: + type: string + default: [] + description: + metadata_cardinality_limit: + type: integer + default: 1000 + minimum: 0 + goType: + name: uint32 From 450b909fa8aa42960c3db698766eebd9dd5dc622 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Thu, 19 Sep 2024 19:02:14 +0100 Subject: [PATCH 2/2] Switch from goType to goJSONSchema. goJSONSchema is a feature which already exists upstream. goType doesn't exist upstream. I added it, because I didn't know goJSONSchema exists. --- config/configauth/configauth.yaml | 7 ++++--- config/confighttp/confighttp.yaml | 14 ++++++++------ config/configtls/configtls.yaml | 21 ++++++++++++--------- processor/batchprocessor/metadata.yaml | 12 ++++++------ 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/config/configauth/configauth.yaml b/config/configauth/configauth.yaml index 573ae9fd8d6..4782b2291c6 100644 --- a/config/configauth/configauth.yaml +++ b/config/configauth/configauth.yaml @@ -9,6 +9,7 @@ $defs: properties: authenticator: type: string - goType: - name: ID - package: go.opentelemetry.io/collector/component + goJSONSchema: + type: component.ID + imports: + - go.opentelemetry.io/collector/component diff --git a/config/confighttp/confighttp.yaml b/config/confighttp/confighttp.yaml index 1c99b38c393..fe9ee0374cf 100644 --- a/config/confighttp/confighttp.yaml +++ b/config/confighttp/confighttp.yaml @@ -45,11 +45,12 @@ id: opentelemetry.io/collector/config/confighttp/confighttp # TODO: This shouldn't generate a new type ClientConfigHeaders additionalProperties: type: string - goType: + goJSONSchema: # TODO: Make this import work # "$ref": "../configopaque/schema.yaml#/$defs/String" - name: String - package: go.opentelemetry.io/collector/config/configopaque + type: configopaque.String + imports: + - go.opentelemetry.io/collector/config/configopaque auth: "$ref": "../../config/configauth/configauth.yaml#/$defs/authentication" compression: @@ -121,9 +122,10 @@ id: opentelemetry.io/collector/config/confighttp/confighttp type: object additionalProperties: type: string - goType: - name: String - package: go.opentelemetry.io/collector/config/configopaque + goJSONSchema: + type: configopaque.String + imports: + - go.opentelemetry.io/collector/config/configopaque compression_algorithms: "$ref": "../configcompression/compressiontype.yaml" # TODO: Why isn't the default inside compressiontype.yaml enough? diff --git a/config/configtls/configtls.yaml b/config/configtls/configtls.yaml index dd1053398a9..105ca82ac05 100644 --- a/config/configtls/configtls.yaml +++ b/config/configtls/configtls.yaml @@ -10,9 +10,10 @@ $defs: default: "" ca_pem: type: string - goType: - name: String - package: go.opentelemetry.io/collector/config/configopaque + goJSONSchema: + type: configopaque.String + imports: + - go.opentelemetry.io/collector/config/configopaque default: "" include_system_ca_certs_pool: type: boolean @@ -23,18 +24,20 @@ $defs: cert_pem: type: string default: "" - goType: - name: String - package: go.opentelemetry.io/collector/config/configopaque + goJSONSchema: + type: configopaque.String + imports: + - go.opentelemetry.io/collector/config/configopaque key_file: type: string default: "" key_pem: type: string default: "" - goType: - name: String - package: go.opentelemetry.io/collector/config/configopaque + goJSONSchema: + type: configopaque.String + imports: + - go.opentelemetry.io/collector/config/configopaque min_version: type: string default: "" diff --git a/processor/batchprocessor/metadata.yaml b/processor/batchprocessor/metadata.yaml index 0c75a6d8cd0..34f7fbf183b 100644 --- a/processor/batchprocessor/metadata.yaml +++ b/processor/batchprocessor/metadata.yaml @@ -60,14 +60,14 @@ config: type: integer minimum: 0 default: 8192 - goType: - name: uint32 + goJSONSchema: + type: uint32 send_batch_max_size: type: integer default: 0 minimum: 0 - goType: - name: uint32 + goJSONSchema: + type: uint32 metadata_keys: type: array items: @@ -78,5 +78,5 @@ config: type: integer default: 1000 minimum: 0 - goType: - name: uint32 + goJSONSchema: + type: uint32