From 656b08bd7b375a191d1254a3d0b41fedde7be079 Mon Sep 17 00:00:00 2001 From: Paolo Chila Date: Tue, 28 Jan 2025 15:14:31 +0100 Subject: [PATCH 1/2] Switch container logs input to /var/log/pod/* path --- .../container_logs/agent/stream/stream.yml.hbs | 13 +------------ .../data_stream/container_logs/manifest.yml | 9 +-------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs b/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs index 5900cece065..3f9552b54a3 100644 --- a/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs +++ b/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs @@ -1,15 +1,4 @@ -{{! - Because we use `${kubernetes.container.id}` in the ID, an instance - of this input will be generated for every container, so `paths` must - always be unique per container otherwise there will be data - duplication, at the extreme this will overload Filebeat and cause - data ingestion issues. - - This ID is also mentioned in the `README.md, so if it is changed, it - needs to be updated there as well. -}} - -id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id} +id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id} paths: {{#each paths}} - {{this}} diff --git a/packages/kubernetes/data_stream/container_logs/manifest.yml b/packages/kubernetes/data_stream/container_logs/manifest.yml index 8bf6f6a9939..d4594a3ac0b 100644 --- a/packages/kubernetes/data_stream/container_logs/manifest.yml +++ b/packages/kubernetes/data_stream/container_logs/manifest.yml @@ -12,14 +12,7 @@ streams: title: Kubernetes container log path multi: true default: - - /var/log/containers/*${kubernetes.container.id}.log - description: >- - For every container the Elastic-Agent can see (usually every - container on the node) an instance of the input will be - created harvesting all paths defined here, even if - the paths contain no variable! Refer to the [integration - documentation](https://www.elastic.co/guide/en/integrations/current/kubernetes.html) - for more details. + - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log - name: symlinks type: bool title: Use Symlinks From b8d679d41ff34218e90a50469fcea6135d75d6df Mon Sep 17 00:00:00 2001 From: Paolo Chila Date: Tue, 28 Jan 2025 15:26:09 +0100 Subject: [PATCH 2/2] Add changelong and bump patch version --- packages/kubernetes/changelog.yml | 5 +++++ packages/kubernetes/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml index dc68e9a4b12..4a8fdbb5c7a 100644 --- a/packages/kubernetes/changelog.yml +++ b/packages/kubernetes/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: 1.81.0 + changes: + - description: Switch k8s input paths to /var/log/pods/* to ingest rotated container logs + type: bugfix + link: https://github.com/elastic/integrations/pull/12500 - version: 1.80.0 changes: - description: Add support for Kibana `9.0.0` diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml index c1df89ccd4d..7d2de2defbc 100644 --- a/packages/kubernetes/manifest.yml +++ b/packages/kubernetes/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.2 name: kubernetes title: Kubernetes -version: 1.80.0 +version: 1.81.0 description: Collect logs and metrics from Kubernetes clusters with Elastic Agent. type: integration categories: