Send agent telemetry

.gitignore

@@ -8,3 +8,5 @@
 
 ### Scripts ###
 tmp/
+
+rendered/

charts/k8s-infra/templates/_config.tpl

@@ -19,6 +19,15 @@ Build config file for daemonset OpenTelemetry Collector: OtelAgent
 {{- $values := deepCopy .Values.otelAgent }}
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
 {{- $config := include "otelAgent.baseConfig" $data | fromYaml }}
+{{- if .Values.presets.ownTelemetry.traces.enabled }}
+{{- $config = (include "opentelemetry-collector.applyOwnTracesConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
+{{- if .Values.presets.ownTelemetry.metrics.enabled }}
+{{- $config = (include "opentelemetry-collector.applyOwnMetricsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
+{{- if .Values.presets.ownTelemetry.logs.enabled }}
+{{- $config = (include "opentelemetry-collector.applyOwnLogsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
 {{- if .Values.presets.logsCollection.enabled }}
 {{- $config = (include "opentelemetry-collector.applyLogsCollectionConfig" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
@@ -40,6 +49,9 @@ Build config file for daemonset OpenTelemetry Collector: OtelAgent
 {{- if .Values.presets.loggingExporter.enabled }}
 {{- $config = (include "opentelemetry-collector.applyLoggingExporterConfig" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
+{{- if .Values.presets.ownTelemetry.logs.enabled }}
+{{- $config = (include "opentelemetry-collector.applyOtlpExporterOwnTelemetryConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
 {{- if .Values.presets.otlpExporter.enabled }}
 {{- $config = (include "opentelemetry-collector.applyOtlpExporterConfig" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
@@ -62,6 +74,12 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- $values := deepCopy .Values }}
 {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }}
 {{- $config := include "otelDeployment.baseConfig" $data | fromYaml }}
+{{- if .Values.presets.ownTelemetry.traces.enabled }}
+{{- $config = (include "opentelemetry-collector.applyOwnTracesConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
+{{- if .Values.presets.ownTelemetry.metrics.enabled }}
+{{- $config = (include "opentelemetry-collector.applyOwnMetricsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
 {{- if .Values.presets.resourceDetection.enabled }}
 {{- $config = (include "opentelemetry-collector.applyResourceDetectionConfigForDeployment" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
@@ -86,6 +104,24 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- tpl (toYaml $config) . }}
 {{- end }}
 
+{{- define "opentelemetry-collector.applyOwnTracesConfig" -}}
+{{- $config := .config }}
+{{- $config = mustMergeOverwrite (include "opentelemetry-collector.ownTracesConfig" .Values | fromYaml) $config }}
+{{- $config | toYaml }}
+{{- end }}
+
+{{- define "opentelemetry-collector.applyOwnMetricsConfig" -}}
+{{- $config := .config }}
+{{- $config = mustMergeOverwrite (include "opentelemetry-collector.ownMetricsConfig" .Values | fromYaml) $config }}
+{{- $config | toYaml }}
+{{- end }}
+
+{{- define "opentelemetry-collector.applyOwnLogsConfig" -}}
+{{- $config := .config }}
+{{- $config = mustMergeOverwrite (include "opentelemetry-collector.ownLogsConfig" .Values | fromYaml) $config }}
+{{- $config | toYaml }}
+{{- end }}
+
 {{- define "opentelemetry-collector.applyLoggingExporterConfig" -}}
 {{- $config := mustMergeOverwrite (include "opentelemetry-collector.loggingExporterConfig" .Values | fromYaml) .config }}
 {{- if $config.service.pipelines.logs }}
@@ -131,6 +167,98 @@ exporters:
 {{- $config | toYaml }}
 {{- end }}
 
+{{- define "opentelemetry-collector.applyOtlpExporterOwnTelemetryConfig" -}}
+{{- $config := mustMergeOverwrite (include "opentelemetry-collector.otlpExporterOwnTelemetryConfig" .Values | fromYaml) .config }}
+{{- $config | toYaml }}
+{{- end }}
+
+
+
+{{- define "opentelemetry-collector.otlpExporterOwnTelemetryConfig" -}}
+exporters:
+  otlphttp/own_telemetry:
+    endpoint: http{{ if not .Values.presets.ownTelemetry.insecure }}s{{ end }}://{{ default "${env:OTEL_EXPORTER_OTLP_ENDPOINT}" .Values.presets.ownTelemetry.endpoint }}
+    tls:
+      insecure: {{ default "${env:OTEL_EXPORTER_OTLP_INSECURE}" .Values.presets.ownTelemetry.insecure }}
+      insecure_skip_verify: {{ default "${env:OTEL_EXPORTER_OTLP_INSECURE_SKIP_VERIFY}" .Values.presets.ownTelemetry.insecureSkipVerify }}
+    headers:
+      "signoz-access-token": "${env:SIGNOZ_API_KEY}"
+{{- end }}
+
+{{/*
+Own traces config, if the endpoint is not set in the ownTelemetry config,
+it will use the same endpoint as regular otlp exporter.
+*/}}
+{{- define "opentelemetry-collector.ownTracesConfig" -}}
+service:
+  telemetry:
+    traces:
+      processors:
+      - batch:
+          exporter:
+            otlp:
+              protocol: http/protobuf
+              endpoint:  http{{ if not .Values.presets.ownTelemetry.insecure }}s{{ end }}://{{ default "${env:OTEL_EXPORTER_OTLP_ENDPOINT}" .Values.presets.ownTelemetry.endpoint }}
+              insecure: {{ default "${env:OTEL_EXPORTER_OTLP_INSECURE}" .Values.presets.ownTelemetry.insecure }}
+              compression: gzip
+              headers:
+                "signoz-access-token": "${env:SIGNOZ_API_KEY}"
+      propagators:
+      - tracecontext
+      - b3
+{{- end }}
+
+{{/*
+Own metrics config, if the endpoint is not set in the ownTelemetry config,
+it will use the same endpoint as regular otlp exporter.
+*/}}
+{{- define "opentelemetry-collector.ownMetricsConfig" -}}
+service:
+  telemetry:
+    metrics:
+      level: detailed
+      readers:
+        - periodic:
+            exporter:
+              otlp:
+                protocol: http/protobuf
+                endpoint:  http{{ if not .Values.presets.ownTelemetry.insecure }}s{{ end }}://{{ default "${env:OTEL_EXPORTER_OTLP_ENDPOINT}" .Values.presets.ownTelemetry.endpoint }}
+                insecure: {{ default "${env:OTEL_EXPORTER_OTLP_INSECURE}" .Values.presets.ownTelemetry.insecure }}
+                compression: gzip
+                headers:
+                  "signoz-access-token": "${env:SIGNOZ_API_KEY}"
+{{- end }}
+
+{{/*
+OTEL go doesn't support logs yet, so we use filelog receiver to collect logs,
+if the endpoint is not set in the ownTelemetry config, it will use the same endpoint as regular otlp exporter.
+*/}}
+{{- define "opentelemetry-collector.ownLogsConfig" -}}
+receivers:
+  filelog/own_logs:
+    include:
+      - /var/log/pods/{{ .Release.Namespace }}_{{ .Release.Name }}*-k8s-infra-*/*/*.log
+    start_at: {{ .Values.presets.logsCollection.startAt }}
+    include_file_path: {{ .Values.presets.logsCollection.includeFilePath }}
+    include_file_name: {{ .Values.presets.logsCollection.includeFileName }}
+    operators:
+    {{ range $operators := .Values.presets.logsCollection.operators }}
+      - {{ toYaml $operators | nindent 8 }}
+    {{ end }}
+processors:
+  filter/non_error_logs:
+    logs:
+      log_record:
+        - 'not IsMatch(body, ".*error.*")'
+service:
+  pipelines:
+    logs/own_logs:
+      exporters: [otlphttp/own_telemetry]
+      # we want to send only error logs
+      processors: [filter/non_error_logs]
+      receivers: [filelog/own_logs]
+{{- end }}
+
 {{- define "opentelemetry-collector.otlpExporterConfig" -}}
 exporters:
   otlp:
@@ -254,10 +382,8 @@ receivers:
       # The file format is /var/log/pods/<namespace_name>_<pod_name>_<pod_uid>/<container_name>/<run_id>.log
       {{- if .Values.presets.logsCollection.whitelist.signozLogs }}
       - /var/log/pods/{{ .Release.Namespace }}_{{ .Release.Name }}*-signoz-*/*/*.log
-      - /var/log/pods/{{ .Release.Namespace }}_{{ .Release.Name }}*-k8s-infra-*/*/*.log
       {{- if and .Values.namespace (ne .Release.Namespace .Values.namespace) }}
       - /var/log/pods/{{ .Release.Namespace }}_{{ .Release.Name }}*-signoz-*/*/*.log
-      - /var/log/pods/{{ .Values.namespace }}_{{ .Release.Name }}*-k8s-infra-*/*/*.log
       {{- end }}
       {{- end }}
       {{- range $namespace := $namespaces }}
@@ -366,6 +492,10 @@ processors:
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "processors" (prepend (index (index $config.service.pipelines "metrics/internal") "processors") "resourcedetection" | uniq) }}
 {{- end }}
+# prepend for "logs/own_logs"
+{{- if index $config.service.pipelines "logs/own_logs" }}
+{{- $_ := set (index $config.service.pipelines "logs/own_logs") "processors" (prepend (index (index $config.service.pipelines "logs/own_logs") "processors") "resourcedetection" | uniq) }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -383,6 +513,10 @@ processors:
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "processors" (prepend (index (index $config.service.pipelines "metrics/internal") "processors") "resourcedetection" | uniq) }}
 {{- end }}
+# prepend for "logs/own_logs"
+{{- if index $config.service.pipelines "logs/own_logs" }}
+{{- $_ := set (index $config.service.pipelines "logs/own_logs") "processors" (prepend (index (index $config.service.pipelines "logs/own_logs") "processors") "resourcedetection" | uniq) }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 

charts/k8s-infra/values.yaml

@@ -90,9 +90,22 @@ presets:
     samplingThereafter: 500
   otlpExporter:
     enabled: true
+  ownTelemetry:
+    # OTLP HTTP endpoint to send own telemetry data to.
+    endpoint: ""
+    # Whether to use insecure mode.
+    insecure: true
+    # Whether to skip verifying the certificate.
+    insecureSkipVerify: true
+    traces:
+      enabled: false
+    metrics:
+      enabled: false
+    logs:
+      enabled: false
   logsCollection:
     enabled: true
-    startAt: beginning
+    startAt: end
     includeFilePath: true
     includeFileName: false
     # This include path patterns for log files to be collected.
@@ -104,7 +117,6 @@ presets:
     # By default, kube-system and hotrod, locust pods are excluded.
     blacklist:
       enabled: true
-      signozLogs: true
       namespaces:
         - kube-system
       pods:
@@ -117,7 +129,6 @@ presets:
     # If whitelist is enabled, `include` is ignored.
     whitelist:
       enabled: false
-      signozLogs: true
       namespaces: []
       pods: []
       containers: []