observability: add dev trace backend for testing

observability/Makefile

@@ -0,0 +1,9 @@
+all: deploy patch-frontend patch-clusterservice
+deploy:
+	kubectl apply -k deploy/
+
+patch-frontend:
+	kubectl patch -n aro-hcp deployment aro-hcp-frontend --type=json --patch-file=patches/otel-sdk.yaml
+
+patch-clusterservice:
+	kubectl patch -n cluster-service deployment cluster-service --type=json --patch-file=patches/otel-sdk.yaml

observability/README.md

@@ -0,0 +1,62 @@
+# Observability for developer environments
+
+This page explains how you can enable tracing for ARO HCP in your [development setup](../dev-infrastructure/docs/development-setup.md).
+
+## Tracing
+
+ARO frontend, cluster service and other components are instrumented with the OpenTelemetry SDK.
+In the current development environment, there is no possibility to inspect traces.
+
+### Deploy Jaeger all-in-one testing backend
+
+We will deploy Jaeger with in-memory storage to store and visualize traces received from the ARO-HCP components.
+
+#### Install
+```
+make deploy
+```
+
+After installation, the `jaeger` service becomes available in the `observability` namespace. We can access the UI using `kubectl port-forward`:
+
+```
+kubectl port-forward -n observability svc/jaeger 16686:16686
+```
+
+Open http://localhost:16686 in your browser to access the Jaeger UI.
+The `observability` namespace contains a second service named `ingest` which accepts otlp via gRPC and HTTP.
+
+#### Configure instances
+
+The export of the trace information is configured via environment variables. Existing deployments can be patched as follows:
+
+```diff
++        env:
++        - name: OTEL_EXPORTER_OTLP_ENDPOINT
++          value: https://ingest.observability:4318
++        - name: OTEL_TRACES_EXPORTER
++          value: otlp
+```
+
+You can use:
+
+```
+make patch-frontend
+make patch-clusteservice
+```
+
+
+### Correlate with ARM requests
+
+#### Generate Traces
+
+Traces are automatically generated for incoming http requests. A simple way to generate incoming requests is to run the `aro-hcp demo` scripts, e.g. to create a cluster.
+
+#### Common Span Attributes
+
+The following span attributes are set in the root span and propagated to the next service via baggage: 
+
+```
+aro.correlation.id
+aro.request.id
+aro.client.request.id
+```

observability/deploy/jaeger.yaml

@@ -0,0 +1,48 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jaeger
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: jaeger
+  template:
+    metadata:
+      labels:
+        app: jaeger
+    spec:
+      containers:
+        - name: jaeger
+          image: jaegertracing/all-in-one:1.63.0
+          env:
+            - name: SPAN_STORAGE_TYPE
+              value: memory
+            - name: JAEGER_DISABLED
+              value: "false"
+            - name: COLLECTOR_OTLP_ENABLED
+              value: "true"
+            - name: COLLECTOR_OTLP_GRPC_HOST_PORT
+              value: 0.0.0.0:4317
+            - name: COLLECTOR_OTLP_HTTP_HOST_PORT
+              value: 0.0.0.0:4318
+          ports:
+            - containerPort: 4317
+              name: grpc-otlp
+            - containerPort: 4318
+              name: http-otlp
+            - containerPort: 16686
+              name: jaeger-ui
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /
+              port: 14269
+            initialDelaySeconds: 5
+            periodSeconds: 15
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 14269
+            initialDelaySeconds: 1

observability/deploy/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+resources:
+  - namespace.yaml
+  - jaeger.yaml

observability/deploy/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: placeholder

observability/deploy/service.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jaeger
+spec:
+  selector:
+    app: jaeger
+  ports:
+    - name: jaeger-ui
+      port: 16686
+      targetPort: 16686
+      protocol: TCP
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingest
+spec:
+  selector:
+    app: jaeger
+  ports:
+    - name: grpc-otlp
+      port: 4317
+      targetPort: 4317
+      protocol: TCP
+    - name: http-otlp
+      port: 4318
+      targetPort: 4318
+      protocol: TCP
+  type: ClusterIP

observability/patches/otel-sdk.yaml

@@ -0,0 +1,14 @@
+---
+- op: add
+  path: "/spec/template/spec/containers/0/env"
+  value: []
+- op: add
+  path: "/spec/template/spec/containers/0/env/-"
+  value:
+    name: OTEL_EXPORTER_OTLP_ENDPOINT
+    value: "http://ingest.observability:4318"
+- op: add
+  path: "/spec/template/spec/containers/0/env/-"
+  value:
+    name: OTEL_TRACES_EXPORTER
+    value: "otlp"