Add OTLP HTTP to gateway

Signed-off-by: Pavol Loffay <[email protected]>
grafana · Jun 3, 2024 · ff112b9 · ff112b9
1 parent 26b93e5
commit ff112b9
Show file tree

Hide file tree

Showing 18 changed files with 112 additions and 100 deletions.
diff --git a/.chloggen/gateway-otlp-http.yaml b/.chloggen/gateway-otlp-http.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Enable OTLP HTTP on Gateway by default.
+
+# One or more tracking issues related to the change
+issues: [948]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults.go b/apis/tempo/v1alpha1/tempomonolithic_defaults.go
@@ -54,7 +54,7 @@ func (r *TempoMonolithic) Default(ctrlConfig configv1alpha1.ProjectConfig) {
 		}
 	}
 	// the gateway only supports OTLP/gRPC
-	if r.Spec.Ingestion.OTLP.HTTP == nil && !r.Spec.Multitenancy.IsGatewayEnabled() {
+	if r.Spec.Ingestion.OTLP.HTTP == nil {
 		r.Spec.Ingestion.OTLP.HTTP = &MonolithicIngestionOTLPProtocolsHTTPSpec{
 			Enabled: true,
 		}

diff --git a/internal/manifests/config/build_test.go b/internal/manifests/config/build_test.go
@@ -1678,6 +1678,13 @@ distributor:
             client_ca_file: /var/run/ca/service-ca.crt
             key_file: /var/run/tls/server/tls.key
             min_version: 1.2
+        http:
+          endpoint: "0.0.0.0:4318"
+          tls:
+            cert_file: /var/run/tls/server/tls.crt
+            client_ca_file: /var/run/ca/service-ca.crt
+            key_file: /var/run/tls/server/tls.key
+            min_version: 1.2
 
   ring:
     kvstore:

diff --git a/internal/manifests/config/tempo-config.yaml b/internal/manifests/config/tempo-config.yaml
@@ -69,9 +69,15 @@ distributor:
             key_file: {{ .ReceiverTLS.Paths.Key }}
             min_version: {{ .ReceiverTLS.MinTLSVersion }}
 {{- end }}
-{{- if not .Gateway }}
         http:
           endpoint: 0.0.0.0:4318
+{{- if and .Gates.GRPCEncryption .Gateway }}
+          tls:
+            client_ca_file:  {{ .TLS.Paths.CA }}
+            cert_file: {{ .TLS.Paths.Certificate }}
+            key_file: {{ .TLS.Paths.Key }}
+            min_version: {{ .TLS.Profile.MinTLSVersionShort }}
+{{- end }}
 {{- if .ReceiverTLS.Enabled }}
           tls:
 {{- if .ReceiverTLS.ClientCAEnabled }}
@@ -80,7 +86,6 @@ distributor:
             cert_file: {{ .ReceiverTLS.Paths.Certificate }}
             key_file: {{ .ReceiverTLS.Paths.Key }}
             min_version: {{ .ReceiverTLS.MinTLSVersion }}
-{{- end }}
 {{- end }}
   ring:
     kvstore:

diff --git a/internal/manifests/distributor/distributor.go b/internal/manifests/distributor/distributor.go
@@ -139,6 +139,11 @@ func deployment(params manifestutils.Params) *v1.Deployment {
 	}
 
 	containerPorts := []corev1.ContainerPort{
+		{
+			Name:          manifestutils.PortOtlpHttpName,
+			ContainerPort: manifestutils.PortOtlpHttp,
+			Protocol:      corev1.ProtocolTCP,
+		},
 		{
 			Name:          manifestutils.OtlpGrpcPortName,
 			ContainerPort: manifestutils.PortOtlpGrpcServer,
@@ -158,11 +163,6 @@ func deployment(params manifestutils.Params) *v1.Deployment {
 
 	if !tempo.Spec.Template.Gateway.Enabled {
 		containerPorts = append(containerPorts, []corev1.ContainerPort{
-			{
-				Name:          manifestutils.PortOtlpHttpName,
-				ContainerPort: manifestutils.PortOtlpHttp,
-				Protocol:      corev1.ProtocolTCP,
-			},
 			{
 				Name:          manifestutils.PortJaegerThriftHTTPName,
 				ContainerPort: manifestutils.PortJaegerThriftHTTP,
@@ -271,6 +271,12 @@ func service(tempo v1alpha1.TempoStack) *corev1.Service {
 	labels := manifestutils.ComponentLabels(manifestutils.DistributorComponentName, tempo.Name)
 
 	servicePorts := []corev1.ServicePort{
+		{
+			Name:       manifestutils.PortOtlpHttpName,
+			Protocol:   corev1.ProtocolTCP,
+			Port:       manifestutils.PortOtlpHttp,
+			TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName),
+		},
 		{
 			Name:       manifestutils.OtlpGrpcPortName,
 			Protocol:   corev1.ProtocolTCP,
@@ -287,12 +293,6 @@ func service(tempo v1alpha1.TempoStack) *corev1.Service {
 
 	if !tempo.Spec.Template.Gateway.Enabled {
 		servicePorts = append(servicePorts, []corev1.ServicePort{
-			{
-				Name:       manifestutils.PortOtlpHttpName,
-				Port:       manifestutils.PortOtlpHttp,
-				TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName),
-				Protocol:   corev1.ProtocolTCP,
-			},
 			{
 				Name:       manifestutils.PortJaegerThriftHTTPName,
 				Port:       manifestutils.PortJaegerThriftHTTP,

diff --git a/internal/manifests/distributor/distributor_test.go b/internal/manifests/distributor/distributor_test.go
@@ -34,6 +34,11 @@ func TestBuildDistributor(t *testing.T) {
 			name:          "Gateway disabled",
 			enableGateway: false,
 			expectedContainerPorts: []corev1.ContainerPort{
+				{
+					Name:          manifestutils.PortOtlpHttpName,
+					ContainerPort: manifestutils.PortOtlpHttp,
+					Protocol:      corev1.ProtocolTCP,
+				},
 				{
 					Name:          manifestutils.OtlpGrpcPortName,
 					ContainerPort: manifestutils.PortOtlpGrpcServer,
@@ -49,11 +54,6 @@ func TestBuildDistributor(t *testing.T) {
 					ContainerPort: manifestutils.PortMemberlist,
 					Protocol:      corev1.ProtocolTCP,
 				},
-				{
-					Name:          manifestutils.PortOtlpHttpName,
-					ContainerPort: manifestutils.PortOtlpHttp,
-					Protocol:      corev1.ProtocolTCP,
-				},
 				{
 					Name:          manifestutils.PortJaegerThriftHTTPName,
 					ContainerPort: manifestutils.PortJaegerThriftHTTP,
@@ -81,6 +81,12 @@ func TestBuildDistributor(t *testing.T) {
 				},
 			},
 			expectedServicePorts: []corev1.ServicePort{
+				{
+					Name:       manifestutils.PortOtlpHttpName,
+					Port:       manifestutils.PortOtlpHttp,
+					TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName),
+					Protocol:   corev1.ProtocolTCP,
+				},
 				{
 					Name:       manifestutils.OtlpGrpcPortName,
 					Protocol:   corev1.ProtocolTCP,
@@ -93,12 +99,6 @@ func TestBuildDistributor(t *testing.T) {
 					Port:       manifestutils.PortHTTPServer,
 					TargetPort: intstr.FromString(manifestutils.HttpPortName),
 				},
-				{
-					Name:       manifestutils.PortOtlpHttpName,
-					Port:       manifestutils.PortOtlpHttp,
-					TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName),
-					Protocol:   corev1.ProtocolTCP,
-				},
 				{
 					Name:       manifestutils.PortJaegerThriftHTTPName,
 					Port:       manifestutils.PortJaegerThriftHTTP,
@@ -175,6 +175,11 @@ func TestBuildDistributor(t *testing.T) {
 			enableGateway:     false,
 			enableReceiverTLS: true,
 			expectedContainerPorts: []corev1.ContainerPort{
+				{
+					Name:          manifestutils.PortOtlpHttpName,
+					ContainerPort: manifestutils.PortOtlpHttp,
+					Protocol:      corev1.ProtocolTCP,
+				},
 				{
 					Name:          manifestutils.OtlpGrpcPortName,
 					ContainerPort: manifestutils.PortOtlpGrpcServer,
@@ -190,11 +195,6 @@ func TestBuildDistributor(t *testing.T) {
 					ContainerPort: manifestutils.PortMemberlist,
 					Protocol:      corev1.ProtocolTCP,
 				},
-				{
-					Name:          manifestutils.PortOtlpHttpName,
-					ContainerPort: manifestutils.PortOtlpHttp,
-					Protocol:      corev1.ProtocolTCP,
-				},
 				{
 					Name:          manifestutils.PortJaegerThriftHTTPName,
 					ContainerPort: manifestutils.PortJaegerThriftHTTP,
@@ -222,6 +222,12 @@ func TestBuildDistributor(t *testing.T) {
 				},
 			},
 			expectedServicePorts: []corev1.ServicePort{
+				{
+					Name:       manifestutils.PortOtlpHttpName,
+					Port:       manifestutils.PortOtlpHttp,
+					TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName),
+					Protocol:   corev1.ProtocolTCP,
+				},
 				{
 					Name:       manifestutils.OtlpGrpcPortName,
 					Protocol:   corev1.ProtocolTCP,
@@ -234,12 +240,6 @@ func TestBuildDistributor(t *testing.T) {
 					Port:       manifestutils.PortHTTPServer,
 					TargetPort: intstr.FromString(manifestutils.HttpPortName),
 				},
-				{
-					Name:       manifestutils.PortOtlpHttpName,
-					Port:       manifestutils.PortOtlpHttp,
-					TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName),
-					Protocol:   corev1.ProtocolTCP,
-				},
 				{
 					Name:       manifestutils.PortJaegerThriftHTTPName,
 					Port:       manifestutils.PortJaegerThriftHTTP,
@@ -343,6 +343,11 @@ func TestBuildDistributor(t *testing.T) {
 			name:          "Gateway enable",
 			enableGateway: true,
 			expectedContainerPorts: []corev1.ContainerPort{
+				{
+					Name:          manifestutils.PortOtlpHttpName,
+					ContainerPort: manifestutils.PortOtlpHttp,
+					Protocol:      corev1.ProtocolTCP,
+				},
 				{
 					Name:          manifestutils.OtlpGrpcPortName,
 					ContainerPort: manifestutils.PortOtlpGrpcServer,
@@ -360,6 +365,12 @@ func TestBuildDistributor(t *testing.T) {
 				},
 			},
 			expectedServicePorts: []corev1.ServicePort{
+				{
+					Name:       manifestutils.PortOtlpHttpName,
+					Port:       manifestutils.PortOtlpHttp,
+					TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName),
+					Protocol:   corev1.ProtocolTCP,
+				},
 				{
 					Name:       manifestutils.OtlpGrpcPortName,
 					Protocol:   corev1.ProtocolTCP,

diff --git a/internal/manifests/gateway/gateway.go b/internal/manifests/gateway/gateway.go
@@ -242,9 +242,10 @@ func deployment(params manifestutils.Params, rbacCfgHash string, tenantsCfgHash
 							Env:   proxy.ReadProxyVarsFromEnv(),
 							Args: append([]string{
 								fmt.Sprintf("--traces.tenant-header=%s", manifestutils.TenantHeader),
-								fmt.Sprintf("--web.listen=0.0.0.0:%d", manifestutils.GatewayPortHTTPServer),                                                                                                      // proxies Tempo API and optionally Jaeger UI
-								fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer),                                                                                     // serves health checks
-								fmt.Sprintf("--traces.write.otlpgrpc.endpoint=%s:%d", naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpGrpcServer), // Tempo Distributor gRPC upstream
+								fmt.Sprintf("--web.listen=0.0.0.0:%d", manifestutils.GatewayPortHTTPServer),                                                                                                                                                         // proxies Tempo API and optionally Jaeger UI
+								fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer),                                                                                                                                        // serves health checks
+								fmt.Sprintf("--traces.write.otlpgrpc.endpoint=%s:%d", naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpGrpcServer),                                                    // Tempo Distributor gRPC upstream
+								fmt.Sprintf("--traces.write.otlphttp.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption), naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpHttp), // Tempo Distributor HTTP upstream
 								fmt.Sprintf("--traces.tempo.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption),
 									naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.QueryFrontendComponentName), manifestutils.PortHTTPServer), // Tempo API upstream
 								fmt.Sprintf("--grpc.listen=0.0.0.0:%d", manifestutils.GatewayPortGRPCServer), // proxies Tempo Distributor gRPC

diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go
@@ -254,6 +254,11 @@ func buildTempoConfig(opts Options) ([]byte, error) {
 				config.Distributor.Receivers.OTLP.Protocols.HTTP = &tempoReceiverConfig{
 					TLS: receiverTLS,
 				}
+
+				if tempo.Spec.Multitenancy.IsGatewayEnabled() {
+					// all connections to tempo must go via gateway
+					config.Distributor.Receivers.OTLP.Protocols.HTTP.Endpoint = fmt.Sprintf("localhost:%d", manifestutils.PortOtlpHttp)
+				}
 			}
 		}
 	}

diff --git a/internal/manifests/monolithic/statefulset.go b/internal/manifests/monolithic/statefulset.go
@@ -395,8 +395,9 @@ func configureGateway(opts Options, sts *appsv1.StatefulSet) error {
 	}
 
 	if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil && tempo.Spec.Ingestion.OTLP.GRPC != nil && tempo.Spec.Ingestion.OTLP.GRPC.Enabled {
-		args = append(args, fmt.Sprintf("--grpc.listen=0.0.0.0:%d", manifestutils.GatewayPortGRPCServer))                   // proxies Tempo Distributor gRPC
-		args = append(args, fmt.Sprintf("--traces.write.otlpgrpc.endpoint=localhost:%d", manifestutils.PortOtlpGrpcServer)) // Tempo Distributor gRPC upstream
+		args = append(args, fmt.Sprintf("--grpc.listen=0.0.0.0:%d", manifestutils.GatewayPortGRPCServer))                    // proxies Tempo Distributor gRPC
+		args = append(args, fmt.Sprintf("--traces.write.otlpgrpc.endpoint=localhost:%d", manifestutils.PortOtlpGrpcServer))  // Tempo Distributor gRPC upstream
+		args = append(args, fmt.Sprintf("--traces.write.otlphttp.endpoint=http://localhost:%d", manifestutils.PortOtlpHttp)) // Tempo Distributor HTTP upstream
 		ports = append(ports, corev1.ContainerPort{
 			Name:          manifestutils.GatewayGrpcPortName,
 			ContainerPort: manifestutils.GatewayPortGRPCServer,

diff --git a/internal/manifests/monolithic/statefulset_test.go b/internal/manifests/monolithic/statefulset_test.go
@@ -793,6 +793,7 @@ func TestStatefulsetGateway(t *testing.T) {
 			"--log.level=info",
 			"--grpc.listen=0.0.0.0:8090",
 			"--traces.write.otlpgrpc.endpoint=localhost:4317",
+			"--traces.write.otlphttp.endpoint=http://localhost:4318",
 			"--traces.read.endpoint=http://localhost:16686",
 			"--tls.server.cert-file=/etc/tempo-gateway/serving-cert/tls.crt",
 			"--tls.server.key-file=/etc/tempo-gateway/serving-cert/tls.key",

diff --git a/internal/webhooks/tempomonolithic_webhook.go b/internal/webhooks/tempomonolithic_webhook.go
@@ -146,14 +146,6 @@ func (v *monolithicValidator) validateMultitenancy(tempo tempov1alpha1.TempoMono
 	}
 
 	multitenancyBase := field.NewPath("spec", "multitenancy")
-	if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil &&
-		tempo.Spec.Ingestion.OTLP.HTTP != nil && tempo.Spec.Ingestion.OTLP.HTTP.Enabled {
-		return field.ErrorList{field.Invalid(
-			multitenancyBase.Child("enabled"),
-			tempo.Spec.Multitenancy.Enabled,
-			"OTLP/HTTP ingestion must be disabled to enable multi-tenancy",
-		)}
-	}
 
 	err := ValidateTenantConfigs(&tempo.Spec.Multitenancy.TenantsSpec, tempo.Spec.Multitenancy.IsGatewayEnabled())
 	if err != nil {

diff --git a/internal/webhooks/tempomonolithic_webhook_test.go b/internal/webhooks/tempomonolithic_webhook_test.go
@@ -125,34 +125,6 @@ func TestMonolithicValidate(t *testing.T) {
 		},
 
 		// multitenancy
-		{
-			name: "OTLP/HTTP enabled and multi-tenancy enabled",
-			tempo: v1alpha1.TempoMonolithic{
-				Spec: v1alpha1.TempoMonolithicSpec{
-					Ingestion: &v1alpha1.MonolithicIngestionSpec{
-						OTLP: &v1alpha1.MonolithicIngestionOTLPSpec{
-							HTTP: &v1alpha1.MonolithicIngestionOTLPProtocolsHTTPSpec{
-								Enabled: true,
-							},
-						},
-					},
-					Multitenancy: &v1alpha1.MonolithicMultitenancySpec{
-						Enabled: true,
-						TenantsSpec: v1alpha1.TenantsSpec{
-							Authentication: []v1alpha1.AuthenticationSpec{{
-								TenantName: "abc",
-							}},
-						},
-					},
-				},
-			},
-			warnings: admission.Warnings{},
-			errors: field.ErrorList{field.Invalid(
-				field.NewPath("spec", "multitenancy", "enabled"),
-				true,
-				"OTLP/HTTP ingestion must be disabled to enable multi-tenancy",
-			)},
-		},
 		{
 			name: "multi-tenancy enabled, OpenShift mode, authorization set",
 			tempo: v1alpha1.TempoMonolithic{

diff --git a/tests/e2e-openshift-ossm/ossm-tempostack-otel/install-tempo-assert.yaml b/tests/e2e-openshift-ossm/ossm-tempostack-otel/install-tempo-assert.yaml
@@ -166,6 +166,10 @@ metadata:
   namespace: tracing-system
 spec:
   ports:
+    - name: otlp-http
+      port: 4318
+      protocol: TCP
+      targetPort: otlp-http
     - name: otlp-grpc
       port: 4317
       protocol: TCP
@@ -174,10 +178,6 @@ spec:
       port: 3200
       protocol: TCP
       targetPort: http
-    - name: otlp-http
-      port: 4318
-      protocol: TCP
-      targetPort: otlp-http
     - name: thrift-http
       port: 14268
       protocol: TCP
@@ -390,4 +390,4 @@ spec:
   to:
     kind: Service
     name: tempo-simplest-query-frontend
-    weight: 100
+    weight: 100
diff --git a/tests/e2e-openshift-ossm/ossm-tempostack/install-tempo-assert.yaml b/tests/e2e-openshift-ossm/ossm-tempostack/install-tempo-assert.yaml
@@ -166,6 +166,10 @@ metadata:
   namespace: tracing-system
 spec:
   ports:
+    - name: otlp-http
+      port: 4318
+      protocol: TCP
+      targetPort: otlp-http
     - name: otlp-grpc
       port: 4317
       protocol: TCP
@@ -174,10 +178,6 @@ spec:
       port: 3200
       protocol: TCP
       targetPort: http
-    - name: otlp-http
-      port: 4318
-      protocol: TCP
-      targetPort: otlp-http
     - name: thrift-http
       port: 14268
       protocol: TCP
@@ -385,4 +385,4 @@ spec:
   to:
     kind: Service
     name: tempo-simplest-query-frontend
-    weight: 100
+    weight: 100