Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add policy opentelemetry/inject-otel-environment-variable #1212

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: v1
kind: Pod
metadata:
name: inject-before-preexisting-env
namespace: default
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://opentelemetry-collector.opentelemetry-collector:4317
- name: TEST_ENV
value: test
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: v1
kind: Pod
metadata:
name: inject-custom-endpoint-ns
namespace: ns-w-custom-endpoint
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://http-collector:4318
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: v1
kind: Pod
metadata:
annotations:
otel.corp.org/otlp-endpoint: http://http-collector:4318
name: inject-custom-endpoint-pod
namespace: default
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://http-collector:4318
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: v1
kind: Pod
metadata:
namespace: default
name: inject-custom-endpoint
annotations:
otel.corp.org/otlp-endpoint: http://custom-endpoint:4318
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://custom-endpoint:4318
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: v1
kind: Pod
metadata:
name: inject-default
namespace: ns-wo-annotations
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://opentelemetry-collector.opentelemetry-collector:4317
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: v1
kind: Pod
metadata:
name: inject-ns
namespace: ns-w-inject-true
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://opentelemetry-collector.opentelemetry-collector:4317
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: v1
kind: Pod
metadata:
annotations:
otel.corp.org/inject-env-var: "true"
name: inject-pod
namespace: ns-wo-annotations
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://opentelemetry-collector.opentelemetry-collector:4317
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
apiVersion: cli.kyverno.io/v1alpha1
kind: Test
metadata:
name: inject-otel-environment-variable
policies:
- ../inject-otel-environment-variable.yaml
resources:
- resource.yaml
variables: values.yaml
results:
- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- no-inject-pod
kind: Pod
result: skip

- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- no-inject-ns
kind: Pod
result: skip

- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- inject-ns
patchedResources: expected/inject-ns.yaml
kind: Pod
result: pass

- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- inject-pod
patchedResources: expected/inject-pod.yaml
kind: Pod
result: pass

- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- inject-default
patchedResources: expected/inject-default.yaml
kind: Pod
result: pass

- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- inject-before-preexisting-env
patchedResources: expected/inject-before-preexisting-env.yaml
kind: Pod
result: pass

- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- inject-custom-endpoint-pod
patchedResources: expected/inject-custom-endpoint-pod.yaml
kind: Pod
result: pass

- policy: inject-otel-environment-variable
rule: inject-otel-environment-variable
resources:
- inject-custom-endpoint-ns
patchedResources: expected/inject-custom-endpoint-ns.yaml
kind: Pod
result: pass

## Enable for ConfigMap namespace filtering (see rule comments ConfigMap-NS-Filtering for detail)
# - policy: inject-otel-environment-variable
# rule: inject-otel-environment-variable
# resources:
# - no-inject-ns-excluded-by-configmap
# kind: Pod
# result: skip
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
apiVersion: v1
kind: Pod
metadata:
name: no-inject-pod
namespace: ns-wo-annotations
annotations:
otel.corp.org/inject-env-var: "false"
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
---
apiVersion: v1
kind: Pod
metadata:
name: no-inject-ns
namespace: ns-w-inject-false
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
---
apiVersion: v1
kind: Pod
metadata:
name: inject-ns
namespace: ns-w-inject-true
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
---
apiVersion: v1
kind: Pod
metadata:
name: inject-pod
namespace: ns-wo-annotations
annotations:
otel.corp.org/inject-env-var: "true"
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
---
apiVersion: v1
kind: Pod
metadata:
name: inject-default
namespace: ns-wo-annotations
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
---
apiVersion: v1
kind: Pod
metadata:
name: inject-before-preexisting-env
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
env:
- name: TEST_ENV
value: test
---
apiVersion: v1
kind: Pod
metadata:
name: inject-custom-endpoint-pod
annotations:
otel.corp.org/otlp-endpoint: http://http-collector:4318
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
---
apiVersion: v1
kind: Pod
metadata:
name: inject-custom-endpoint-ns
namespace: ns-w-custom-endpoint
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
---
apiVersion: v1
kind: Pod
metadata:
name: no-inject-ns-excluded-by-configmap
namespace: ns-excluded-by-configmap
spec:
containers:
- name: debdiag
image: ghcr.io/babs/debdiag:0
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
apiVersion: cli.kyverno.io/v1alpha1
kind: Values
policies:
- name: inject-otel-environment-variable
rules:
- name: inject-otel-environment-variable
values:
namespacefilters.data.exclude: '["ns-excluded-by-configmap"]'
# Global Reference simulation
namespaces:
- metadata:
name: ns-wo-annotations
- metadata:
name: ns-w-inject-true
annotations:
otel.corp.org/inject-env-var: "true"
- metadata:
name: ns-w-inject-false
annotations:
otel.corp.org/inject-env-var: "false"
- metadata:
name: ns-w-custom-endpoint
annotations:
otel.corp.org/otlp-endpoint: "http://http-collector:4318"
34 changes: 34 additions & 0 deletions opentelemetry/inject-otel-environment-variable/artifacthub-pkg.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: inject-otel-environment-variable
version: 1.0.0
displayName: Inject OpenTelemetry environment variable
createdAt: "2024-12-27T00:00:00.000Z"
description: >-
Injects OpenTelemetry `OTEL_EXPORTER_OTLP_ENDPOINT` env var in `containers` and `initContainers`.
Injection can be controlled by `otel.corp.org/inject-env-var: "false"` annotation on the Pod or at the Namespace level.
Value of the injected env var can also be overridden per Pod or Namespace via `otel.corp.org/otlp-endpoint: "http://xxxx:4317"` annotation.
The env var will be injected first, meaning that if one is already declared, the later will takes precedence over the injected one.
To avoid stress on the control plane API, a GlobalContextEntry containing Namespaces has to be declared (group: '', version: v1, resource: namespaces)
and `cache.kyverno.io/enabled: "true"` should be set on the `namespacefilters` ConfigMap (see rule comments ConfigMap-NS-Filtering).
install: |-
```shell
kubectl apply \
-f https://raw.githubusercontent.com/kyverno/policies/main/opentelemetry/inject-otel-environment-variable/globalcontext.yaml
-f https://raw.githubusercontent.com/kyverno/policies/main/opentelemetry/inject-otel-environment-variable/inject-otel-environment-variable.yaml
```
keywords:
- kyverno
- OpenTelemetry
readme: |
Injects OpenTelemetry `OTEL_EXPORTER_OTLP_ENDPOINT` env var in `containers` and `initContainers`.
Injection can be controlled by `otel.corp.org/inject-env-var: "false"` annotation on the Pod or at the Namespace level.
Value of the injected env var can also be overridden per Pod or Namespace via `otel.corp.org/otlp-endpoint: "http://xxxx:4317"` annotation.
The env var will be injected first, meaning that if one is already declared, the later will takes precedence over the injected one.
To avoid stress on the control plane API, a GlobalContextEntry containing Namespaces has to be declared (group: '', version: v1, resource: namespaces)
and `cache.kyverno.io/enabled: "true"` should be set on the `namespacefilters` ConfigMap (see rule comments ConfigMap-NS-Filtering).

Refer to the documentation for more details on Kyverno annotations: https://artifacthub.io/docs/topics/annotations/kyverno/
annotations:
kyverno/version: "1.13.0"
kyverno/category: "OpenTelemetry"
kyverno/subject: "Pod"
digest: ffa101097cabaa69e4c3ddfa07d7d4526c5e9a4e32005d794ab37f6c118d6df9
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: namespaces
spec:
kubernetesResource:
group: ''
version: v1
resource: namespaces
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: inject-otel-environment-variable
annotations:
policies.kyverno.io/title: Injects OpenTelemetry environment variable
policies.kyverno.io/minversion: 1.13.0
kyverno.io/kyverno-version: 1.13.0
policies.kyverno.io/subject: Namespace, Pod
policies.kyverno.io/category: OpenTelemetry
policies.kyverno.io/description: >-
Injects OpenTelemetry `OTEL_EXPORTER_OTLP_ENDPOINT` env var in `containers` and `initContainers`.
Injection can be controlled by `otel.corp.org/inject-env-var: "false"` annotation on the Pod or at the Namespace level.
Value of the injected env var can also be overridden per Pod or Namespace via `otel.corp.org/otlp-endpoint: "http://xxxx:4317"` annotation.
The env var will be injected first, meaning that if one is already declared, the later will takes precedence over the injected one.
To avoid stress on the control plane API, a GlobalContextEntry containing Namespaces has to be declared (group: '', version: v1, resource: namespaces)
and `cache.kyverno.io/enabled: "true"` should be set on the `namespacefilters` ConfigMap (see rule comments ConfigMap-NS-Filtering).
spec:
background: false
rules:
- name: inject-otel-environment-variable
context:
- name: namespaces
globalReference:
# This is the name of the GlobalContextEntry
#
# apiVersion: kyverno.io/v2alpha1
# kind: GlobalContextEntry
# metadata:
# name: namespaces
# spec:
# kubernetesResource:
# group: ''
# version: v1
# resource: namespaces
name: namespaces
- name: nsannontations
variable:
jmesPath: namespaces[?metadata.name == '{{ request.object.metadata.namespace }}'] | [0].metadata.annotations
default: {}
- name: otlp_endpoint
variable:
jmesPath: request.object.metadata.annotations."otel.corp.org/otlp-endpoint" || nsannontations."otel.corp.org/otlp-endpoint"
default: 'http://opentelemetry-collector.opentelemetry-collector:4317'

# ConfigMap-NS-Filtering
# See https://kyverno.io/docs/writing-policies/external-data-sources/#handling-configmap-array-values
# Don't forget to enable kyverno's cache on the ConfigMap to avoid stress on the control plane
# Enable only if the proper config map has been enabled
#
# apiVersion: v1
# kind: ConfigMap
# metadata:
# namespace: opentelemetry-collector
# name: namespacefilters
# labels:
# cache.kyverno.io/enabled: "true"
# data:
# exclude: '["ns-excluded-by-configmap"]'

## Uncomment the following for configmap namespace filtering, make sure the proper ConfigMap exists!
## Also uncomment bellow in the preconditions section
# - name: namespacefilters
# configMap:
# name: namespacefilters
# namespace: opentelemetry-collector

preconditions:
all:
# By default inject, expect if namespace or pod annotations says otherwise
- key: "{{ request.object.metadata.annotations.\"otel.corp.org/inject-env-var\" || nsannontations.\"otel.corp.org/inject-env-var\" || 'true' }}"
operator: NotEquals
value: 'false'

# ConfigMap-NS-Filtering
# Env variable injection can be disabled on per NS basis using ConfigMap (no per pod override possible)
## Uncomment the following for ConfigMap-based namespace filtering
## Ensure the corresponding context above is uncommented and the appropriate ConfigMap is configured.
# - key: "{{ request.object.metadata.namespace }}"
# operator: AnyNotIn
# value: "{{ namespacefilters.data.exclude || '[]' }}"
match:
any:
- resources:
kinds:
- Pod
operations:
- CREATE
mutate:
foreach:
- list: request.object.spec.containers[]
patchesJson6902: |-
- op: add
path: /spec/containers/{{elementIndex}}/env/0
value:
name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "{{ otlp_endpoint }}"
- list: request.object.spec.initContainers[]
patchesJson6902: |-
- op: add
path: /spec/initContainers/{{elementIndex}}/env/0
value:
name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "{{ otlp_endpoint }}"