Skip to content

Commit

Permalink
set service_name label for podLogs (#1175)
Browse files Browse the repository at this point in the history 
* add service_name to pod logs discovery

Signed-off-by: Robbie Lankford <[email protected]>

* make build

* add service_namespace, deployment_environment, deployment_environment_name

* update default_test.yaml for pod logs

* fix tests

* fix test

---------

Signed-off-by: Robbie Lankford <[email protected]>
  • Loading branch information
@rlankfo
rlankfo authored Jan 31, 2025
1 parent 0bda801 commit ab802bc
Show file tree
Hide file tree
Showing 63 changed files with 2,659 additions and 57 deletions.
1 change: 1 addition & 0 deletions .gitattributes
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ charts/**/docs/examples/** linguist-generated=true
charts/**/docs/examples/README.md linguist-generated=false
charts/**/docs/examples/**/README.md linguist-generated=false
charts/**/docs/examples/**/values.yaml linguist-generated=false
charts/k8s-monitoring/tests/**/.rendered/output.yaml linguist-generated=true
4 changes: 2 additions & 2 deletions .github/workflows/unit-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,11 @@ jobs:
if [ "${{ matrix.dir }}" == "k8s-monitoring" ]; then
# Skip checking subchart files for k8s-monitoring, which are always modified, even if the contents are identical
if ! git diff --exit-code -- ':!charts/*.tgz'; then
echo "Generated files in charts/${{ matrix.dir }} are not up to date. Please run 'make all' and commit the changes."
echo "Generated files in charts/${{ matrix.dir }} are not up to date. Please run 'make build' and commit the changes."
exit 1
fi
elif ! git diff --exit-code .; then
echo "Generated files in charts/${{ matrix.dir }} are not up to date. Please run 'make all' and commit the changes."
echo "Generated files in charts/${{ matrix.dir }} are not up to date. Please run 'make build' and commit the changes."
exit 1
else
echo "Generated files in charts/${{ matrix.dir }} are up to date."
Expand Down
2 changes: 1 addition & 1 deletion charts/k8s-monitoring/charts/feature-pod-logs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
| annotations | object | `{"job":"k8s.grafana.com/logs.job"}` | Log labels to set with values copied from the Kubernetes Pod annotations. Format: `<log_label>: <kubernetes_annotation>`. |
| extraLogProcessingStages | string | `""` | Stage blocks to be added to the loki.process component for pod logs. ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks)) This value is templated so that you can refer to other values from this file. |
| labels | object | `{"app_kubernetes_io_name":"app.kubernetes.io/name"}` | Log labels to set with values copied from the Kubernetes Pod labels. Format: `<log_label>: <kubernetes_label>`. |
| labelsToKeep | list | `["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name"]` | The list of labels to keep on the logs, all other pipeline labels will be dropped. |
| labelsToKeep | list | `["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","service_namespace","deployment_environment","deployment_environment_name"]` | The list of labels to keep on the logs, all other pipeline labels will be dropped. |
| staticLabels | object | `{}` | Log labels to set with static values. |
| staticLabelsFrom | object | `{}` | Log labels to set with static values, not quoted so it can reference config components. |

Expand Down
42 changes: 42 additions & 0 deletions charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_pod_discovery.alloy.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,48 @@ discovery.relabel "filtered_pods" {
regex = "__meta_kubernetes_pod_annotation_(.+)"
}

// explicitly set service_name. if not set, loki will automatically try to populate a default.
// see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
//
// choose the first value found from the following ordered list:
// - pod.annotation[resource.opentelemetry.io/service.name]
// - pod.label[app.kubernetes.io/name]
// - k8s.pod.name
// - k8s.container.name
rule {
action = "replace"
source_labels = [
"__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_name",
"__meta_kubernetes_pod_container_name",
]
separator = ";"
regex = "^(?:;*)?([^;]+).*$"
replacement = "$1"
target_label = "service_name"
}

// set service_namespace
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace"]
target_label = "service_namespace"
}

// set deployment_environment and deployment_environment_name
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment_name"]
target_label = "deployment_environment_name"
}
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment"]
target_label = "deployment_environment"
}


{{- if .Values.extraDiscoveryRules }}
{{ .Values.extraDiscoveryRules | indent 2 }}
{{- end }}
Expand Down
170 changes: 167 additions & 3 deletions charts/k8s-monitoring/charts/feature-pod-logs/tests/default_test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,47 @@ tests:
action = "labelmap"
regex = "__meta_kubernetes_pod_annotation_(.+)"
}
// explicitly set service_name. if not set, loki will automatically try to populate a default.
// see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
//
// choose the first value found from the following ordered list:
// - pod.annotation[resource.opentelemetry.io/service.name]
// - pod.label[app.kubernetes.io/name]
// - k8s.pod.name
// - k8s.container.name
rule {
action = "replace"
source_labels = [
"__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_name",
"__meta_kubernetes_pod_container_name",
]
separator = ";"
regex = "^(?:;*)?([^;]+).*$"
replacement = "$1"
target_label = "service_name"
}
// set service_namespace
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace"]
target_label = "service_namespace"
}
// set deployment_environment and deployment_environment_name
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment_name"]
target_label = "deployment_environment_name"
}
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment"]
target_label = "deployment_environment"
}
}
discovery.kubernetes "pods" {
Expand Down Expand Up @@ -142,7 +183,7 @@ tests:
// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","service_namespace","deployment_environment","deployment_environment_name","integration"]
}
forward_to = argument.logs_destinations.value
Expand Down Expand Up @@ -222,6 +263,47 @@ tests:
action = "labelmap"
regex = "__meta_kubernetes_pod_annotation_(.+)"
}
// explicitly set service_name. if not set, loki will automatically try to populate a default.
// see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
//
// choose the first value found from the following ordered list:
// - pod.annotation[resource.opentelemetry.io/service.name]
// - pod.label[app.kubernetes.io/name]
// - k8s.pod.name
// - k8s.container.name
rule {
action = "replace"
source_labels = [
"__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_name",
"__meta_kubernetes_pod_container_name",
]
separator = ";"
regex = "^(?:;*)?([^;]+).*$"
replacement = "$1"
target_label = "service_name"
}
// set service_namespace
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace"]
target_label = "service_namespace"
}
// set deployment_environment and deployment_environment_name
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment_name"]
target_label = "deployment_environment_name"
}
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment"]
target_label = "deployment_environment"
}
}
discovery.kubernetes "pods" {
Expand Down Expand Up @@ -371,6 +453,47 @@ tests:
action = "labelmap"
regex = "__meta_kubernetes_pod_annotation_(.+)"
}
// explicitly set service_name. if not set, loki will automatically try to populate a default.
// see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
//
// choose the first value found from the following ordered list:
// - pod.annotation[resource.opentelemetry.io/service.name]
// - pod.label[app.kubernetes.io/name]
// - k8s.pod.name
// - k8s.container.name
rule {
action = "replace"
source_labels = [
"__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_name",
"__meta_kubernetes_pod_container_name",
]
separator = ";"
regex = "^(?:;*)?([^;]+).*$"
replacement = "$1"
target_label = "service_name"
}
// set service_namespace
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace"]
target_label = "service_namespace"
}
// set deployment_environment and deployment_environment_name
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment_name"]
target_label = "deployment_environment_name"
}
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment"]
target_label = "deployment_environment"
}
}
discovery.kubernetes "pods" {
Expand Down Expand Up @@ -449,7 +572,7 @@ tests:
// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","service_namespace","deployment_environment","deployment_environment_name","integration"]
}
forward_to = argument.logs_destinations.value
Expand Down Expand Up @@ -528,6 +651,47 @@ tests:
action = "labelmap"
regex = "__meta_kubernetes_pod_annotation_(.+)"
}
// explicitly set service_name. if not set, loki will automatically try to populate a default.
// see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
//
// choose the first value found from the following ordered list:
// - pod.annotation[resource.opentelemetry.io/service.name]
// - pod.label[app.kubernetes.io/name]
// - k8s.pod.name
// - k8s.container.name
rule {
action = "replace"
source_labels = [
"__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_name",
"__meta_kubernetes_pod_container_name",
]
separator = ";"
regex = "^(?:;*)?([^;]+).*$"
replacement = "$1"
target_label = "service_name"
}
// set service_namespace
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace"]
target_label = "service_namespace"
}
// set deployment_environment and deployment_environment_name
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment_name"]
target_label = "deployment_environment_name"
}
rule {
action = "replace"
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment"]
target_label = "deployment_environment"
}
}
discovery.kubernetes "pods" {
Expand Down Expand Up @@ -606,7 +770,7 @@ tests:
// Only keep the labels that are defined in the `keepLabels` list.
stage.label_keep {
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","service_namespace","deployment_environment","deployment_environment_name","integration"]
}
forward_to = argument.logs_destinations.value
Expand Down
3 changes: 3 additions & 0 deletions charts/k8s-monitoring/charts/feature-pod-logs/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ labelsToKeep:
- namespace
- pod
- service_name
- service_namespace
- deployment_environment
- deployment_environment_name

# -- The structured metadata mappings to set.
# To not set any structured metadata, set this to an empty object (e.g. `{}`)
Expand Down
43 changes: 42 additions & 1 deletion charts/k8s-monitoring/docs/examples/auth/bearer-token/alloy-logs.alloy
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit ab802bc

Please sign in to comment.