From a5ebf8d26766b65c6d967a2e70d07c6c2cf47a08 Mon Sep 17 00:00:00 2001 From: Benedikt Bongartz Date: Thu, 13 Feb 2025 13:58:24 +0100 Subject: [PATCH 1/6] observability: provide jaeger testing deployment Signed-off-by: Benedikt Bongartz --- observability/deploy/jaeger.yaml | 48 +++++++++++++++++++++++++ observability/deploy/kustomization.yaml | 7 ++++ observability/deploy/namespace.yaml | 4 +++ observability/deploy/service.yaml | 32 +++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 observability/deploy/jaeger.yaml create mode 100644 observability/deploy/kustomization.yaml create mode 100644 observability/deploy/namespace.yaml create mode 100644 observability/deploy/service.yaml diff --git a/observability/deploy/jaeger.yaml b/observability/deploy/jaeger.yaml new file mode 100644 index 000000000..79f4688ee --- /dev/null +++ b/observability/deploy/jaeger.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: jaeger + template: + metadata: + labels: + app: jaeger + spec: + containers: + - name: jaeger + image: jaegertracing/all-in-one:1.63.0 + env: + - name: SPAN_STORAGE_TYPE + value: memory + - name: JAEGER_DISABLED + value: "false" + - name: COLLECTOR_OTLP_ENABLED + value: "true" + - name: COLLECTOR_OTLP_GRPC_HOST_PORT + value: 0.0.0.0:4317 + - name: COLLECTOR_OTLP_HTTP_HOST_PORT + value: 0.0.0.0:4318 + ports: + - containerPort: 4317 + name: grpc-otlp + - containerPort: 4318 + name: http-otlp + - containerPort: 16686 + name: jaeger-ui + livenessProbe: + failureThreshold: 5 + httpGet: + path: / + port: 14269 + initialDelaySeconds: 5 + periodSeconds: 15 + readinessProbe: + httpGet: + path: / + port: 14269 + initialDelaySeconds: 1 diff --git a/observability/deploy/kustomization.yaml b/observability/deploy/kustomization.yaml new file mode 100644 index 000000000..d7a591979 --- /dev/null +++ b/observability/deploy/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability +resources: + - namespace.yaml + - jaeger.yaml diff --git a/observability/deploy/namespace.yaml b/observability/deploy/namespace.yaml new file mode 100644 index 000000000..0a074bd87 --- /dev/null +++ b/observability/deploy/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: placeholder diff --git a/observability/deploy/service.yaml b/observability/deploy/service.yaml new file mode 100644 index 000000000..75384cb7e --- /dev/null +++ b/observability/deploy/service.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: jaeger +spec: + selector: + app: jaeger + ports: + - name: jaeger-ui + port: 16686 + targetPort: 16686 + protocol: TCP + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + name: ingest +spec: + selector: + app: jaeger + ports: + - name: grpc-otlp + port: 4317 + targetPort: 4317 + protocol: TCP + - name: http-otlp + port: 4318 + targetPort: 4318 + protocol: TCP + type: ClusterIP From c8a9d368936ce15f2ca95ee3bf504f2ad5f6b147 Mon Sep 17 00:00:00 2001 From: Benedikt Bongartz Date: Thu, 13 Feb 2025 14:37:47 +0100 Subject: [PATCH 2/6] observability: add frontend and cluster-service patches --- observability/Makefile | 9 +++++++++ observability/patches/otel-sdk.yaml | 14 ++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 observability/Makefile create mode 100644 observability/patches/otel-sdk.yaml diff --git a/observability/Makefile b/observability/Makefile new file mode 100644 index 000000000..ae6aca6c8 --- /dev/null +++ b/observability/Makefile @@ -0,0 +1,9 @@ + +deploy: + kustomize apply deploy + +patch-frontend: + kubectl patch -n aro-hcp deployment aro-hcp-frontend --type=json --patch-file=patches/otel-sdk.yaml + +patch-clusterservice: + kubectl patch -n cluster-service deployment cluster-service --type=json --patch-file=patches/otel-sdk.yaml diff --git a/observability/patches/otel-sdk.yaml b/observability/patches/otel-sdk.yaml new file mode 100644 index 000000000..de671a91b --- /dev/null +++ b/observability/patches/otel-sdk.yaml @@ -0,0 +1,14 @@ +--- +- op: add + path: "/spec/template/spec/containers/0/env" + value: [] +- op: add + path: "/spec/template/spec/containers/0/env/-" + value: + name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://ingest.observability:4318" +- op: add + path: "/spec/template/spec/containers/0/env/-" + value: + name: OTEL_TRACES_EXPORTER + value: "otlp" From 7b324f1c7f6cca0432be7bc8f46433b4471e313f Mon Sep 17 00:00:00 2001 From: Benedikt Bongartz Date: Thu, 13 Feb 2025 14:38:39 +0100 Subject: [PATCH 3/6] observability: add README.md Signed-off-by: Benedikt Bongartz --- observability/README.md | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 observability/README.md diff --git a/observability/README.md b/observability/README.md new file mode 100644 index 000000000..27cbef8c6 --- /dev/null +++ b/observability/README.md @@ -0,0 +1,51 @@ +# ARO dev observability + +## Tracing + +ARO frontend, cluster service and other components are instrumented with the OpenTelemetry SDK. +In the current development environment, there is no possibility to inspect traces. + +### Deploy Jaeger all-in-one testing backend + +#### Install +``` +make deploy +``` + +After installation, the `jaeger` service becomes available in the `observability` namespace. We can access the UI using `kubectl port-forward`: + +``` +kubectl port-forward -n observability svc/jaeger 16686:16686 +``` + +The `observability` namespace contains a second service named `ingest` which accepts otlp via gRPC and HTTP. + +#### Configure instances + +The export of the trace information is configured via environment variables. Existing deployments can be patched as follows: + +```diff ++ env: ++ - name: OTEL_EXPORTER_OTLP_ENDPOINT ++ value: https://.:4318 ++ - name: OTEL_TRACES_EXPORTER ++ value: otlp +``` + +You can use: + +``` +make patch-frontend +make patch-clusteservice +``` + + +### Correlate with ARM requests + +The following span attributes are set in the root span and propagated to the next service via baggage: + +``` +aro.correlation.id +aro.request.id +aro.client.request.id +``` From f53342c380b80879d4dff76377f389b0700ced80 Mon Sep 17 00:00:00 2001 From: "Ben B." Date: Fri, 14 Feb 2025 15:56:16 +0100 Subject: [PATCH 4/6] observability: apply suggestions Co-authored-by: Simon Pasquier --- observability/Makefile | 2 +- observability/README.md | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/observability/Makefile b/observability/Makefile index ae6aca6c8..175f6c3b7 100644 --- a/observability/Makefile +++ b/observability/Makefile @@ -1,6 +1,6 @@ deploy: - kustomize apply deploy + kubectl apply -k deploy/ patch-frontend: kubectl patch -n aro-hcp deployment aro-hcp-frontend --type=json --patch-file=patches/otel-sdk.yaml diff --git a/observability/README.md b/observability/README.md index 27cbef8c6..8681ea0e0 100644 --- a/observability/README.md +++ b/observability/README.md @@ -1,4 +1,6 @@ -# ARO dev observability +# Observability for developer environments + +This page explains how you can enable tracing for ARO HCP in your [development setup](../dev-infrastructure/docs/development-setup.md). ## Tracing @@ -7,6 +9,8 @@ In the current development environment, there is no possibility to inspect trace ### Deploy Jaeger all-in-one testing backend +We will deploy Jaeger with in-memory storage to store and visualize traces received from the ARO-HCP components. + #### Install ``` make deploy @@ -18,6 +22,7 @@ After installation, the `jaeger` service becomes available in the `observability kubectl port-forward -n observability svc/jaeger 16686:16686 ``` +Open http://localhost:16686 in your browser to access the Jaeger UI. The `observability` namespace contains a second service named `ingest` which accepts otlp via gRPC and HTTP. #### Configure instances @@ -27,7 +32,7 @@ The export of the trace information is configured via environment variables. Exi ```diff + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT -+ value: https://.:4318 ++ value: https://ingest.observability:4318 + - name: OTEL_TRACES_EXPORTER + value: otlp ``` From d73d72783eb7c5bc6b0520bb7c1f8bff22793333 Mon Sep 17 00:00:00 2001 From: Benedikt Bongartz Date: Fri, 14 Feb 2025 16:04:13 +0100 Subject: [PATCH 5/6] observability: extend trace desc Signed-off-by: Benedikt Bongartz --- observability/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/observability/README.md b/observability/README.md index 8681ea0e0..628dc32cb 100644 --- a/observability/README.md +++ b/observability/README.md @@ -47,6 +47,12 @@ make patch-clusteservice ### Correlate with ARM requests +#### Generate Traces + +Traces are automatically generated for incoming http requests. A simple way to generate incoming requests is to run the `aro-hcp demo` scripts, e.g. to create a cluster. + +#### Common Span Attributes + The following span attributes are set in the root span and propagated to the next service via baggage: ``` From 99eff87978a4da6464f413bdda51af87a0961464 Mon Sep 17 00:00:00 2001 From: "Ben B." Date: Fri, 14 Feb 2025 17:33:23 +0100 Subject: [PATCH 6/6] Update observability/Makefile Co-authored-by: Simon Pasquier --- observability/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/observability/Makefile b/observability/Makefile index 175f6c3b7..d24985726 100644 --- a/observability/Makefile +++ b/observability/Makefile @@ -1,4 +1,4 @@ - +all: deploy patch-frontend patch-clusterservice deploy: kubectl apply -k deploy/