remove pod-stuck in terminating and rename important event annotation

Signed-off-by: Jerry Belmonte <[email protected]> Co-authored-by: Jerry Belmonte <[email protected]> Co-authored-by: Yuqi Jin <[email protected]>
wavefrontHQ · Nov 14, 2023 · 1de4627 · 1de4627
1 parent 1791ece
commit 1de4627
Show file tree

Hide file tree

Showing 8 changed files with 9 additions and 72 deletions.
diff --git a/collector/hack/test/files/all-metrics.jsonl b/collector/hack/test/files/all-metrics.jsonl
@@ -356,7 +356,6 @@
 {"Name":"kubernetes.pod.uptime","Tags":{"cluster":"","label.k8s-app":"prom-example","label.name":"prom-example","namespace_name": "collector-targets","nodename":"","pod_name":"","source":"","type":"pod"}}
 {"Name":"kubernetes.pod.uptime","Tags":{"cluster":"","label.k8s-app":"wavefront-collector","namespace_name":"wavefront-collector","nodename":"","pod_name":"","source":"","type":"pod"}}
 {"Name":"kubernetes.pod.uptime","Tags":{"cluster":"","namespace_name":"","nodename":"","pod_name":"","source":"","type":"pod"}}
-{"Name":"kubernetes.pod.terminating","Value":"1","Tags":{"cluster":"","namespace_name":"collector-targets","nodename":"","pod_name":"pod-stuck-in-terminating","source":"","type":"pod", "DeletionTimestamp":"","workload_name":"","reason": ""}}
 {"Name":"kubernetes.pod_container.cpu.limit","Tags":{"cluster":"","container_base_image":"","container_name":"memcached","label.app.kubernetes.io/instance":"memcached-release","label.app.kubernetes.io/managed-by":"Helm","label.app.kubernetes.io/name":"memcached","label.helm.sh/chart":"","namespace_name": "collector-targets","nodename":"","pod_name":"","source":"","type":"pod_container","workload_name":"","workload_kind":""}}
 {"Name":"kubernetes.pod_container.cpu.limit","Tags":{"cluster":"","container_base_image":"","container_name":"mysql","label.app.kubernetes.io/component":"primary","label.app.kubernetes.io/instance":"mysql-release","label.app.kubernetes.io/managed-by":"Helm","label.app.kubernetes.io/name":"mysql","label.helm.sh/chart":"","label.statefulset.kubernetes.io/pod-name":"mysql-release-0","namespace_name": "collector-targets","nodename":"","pod_name":"","source":"","type":"pod_container"}}
 {"Name":"kubernetes.pod_container.cpu.limit", "Value":"200", "Tags":{"cluster":"","container_base_image":"","container_name":"prom-example","label.k8s-app":"prom-example","label.name":"prom-example","namespace_name": "collector-targets","nodename":"","pod_name":"","source":"","type":"pod_container"}}
@@ -523,7 +522,7 @@
 {"Name":"kubernetes.workload.status","Value":"0","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"pod-cannot-be-scheduled","workload_kind":"Pod","available":"0","desired":"1","reason":"Unschedulable","message":"","!type":""}}
 {"Name":"kubernetes.workload.status","Value":"0","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"pod-crash-loop-backoff","workload_kind":"Pod","available":"0","desired":"1","reason":"CrashLoopBackOff","message":"","!type":""}}
 {"Name":"kubernetes.workload.status","Value":"0","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"pod-image-cannot-be-loaded","workload_kind":"Pod","available":"0","desired":"1","reason":"ImagePullBackOff","message":"","!type":""}}
-{"Name":"kubernetes.workload.status","Value":"0","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"pod-stuck-in-terminating","workload_kind":"Pod","available":"0","desired":"1","reason":"Terminating","message":"","!type":""}}
+{"Name":"kubernetes.workload.status","Value":"0","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"pod-stuck-in-terminating","workload_kind":"Pod","available":"0","desired":"1","reason":"Error","!type":""}}
 {"Name":"kubernetes.workload.status","Value":"1","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"pod-large-init-container","workload_kind":"Pod","available":"1","desired":"1","!type":""}}
 {"Name":"kubernetes.workload.status","Value":"1","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"pod-small-init-container","workload_kind":"Pod","available":"1","desired":"1","!type":""}}
 {"Name":"kubernetes.workload.status","Value":"1","Tags":{"cluster":"","namespace_name":"collector-targets","source":"","workload_name":"prom-example","workload_kind":"Pod","available":"1","desired":"1","!type":""}}

diff --git a/collector/plugins/events/eventannotater.go b/collector/plugins/events/eventannotater.go
@@ -42,10 +42,10 @@ func (em *eventMatcher) categorize(event *v1.Event) bool {
 	if em.match(event) {
 		event.ObjectMeta.Annotations["aria/category"] = em.getCategory(event)
 		event.ObjectMeta.Annotations["aria/subcategory"] = em.getSubcategory(event)
-		event.ObjectMeta.Annotations["internal/important"] = "true"
+		event.ObjectMeta.Annotations["important"] = "true"
 		return true
 	} else {
-		event.ObjectMeta.Annotations["internal/important"] = "false"
+		event.ObjectMeta.Annotations["important"] = "false"
 		return false
 	}
 }
@@ -182,20 +182,6 @@ func (ea *EventAnnotator) runtimeMatchers() []eventMatcher {
 			category:    Runtime,
 			subcategory: OOMKilled,
 		},
-		{
-			match: func(event *v1.Event) bool {
-				if event.Type == v1.EventTypeNormal && event.Reason == "Killing" {
-					pod, err := ea.workloadCache.GetPod(event.InvolvedObject.Name, event.InvolvedObject.Namespace)
-					if err != nil {
-						return false
-					}
-					return util.IsStuckInTerminating(pod)
-				}
-				return false
-			},
-			category:    Runtime,
-			subcategory: Terminating,
-		},
 	}
 }
 

diff --git a/collector/plugins/events/eventannotater_test.go b/collector/plugins/events/eventannotater_test.go
@@ -5,7 +5,6 @@ import (
 	"testing"
 
 	"github.com/wavefronthq/observability-for-kubernetes/collector/internal/testhelper"
-	"github.com/wavefronthq/observability-for-kubernetes/collector/internal/util"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/yaml"
@@ -54,12 +53,6 @@ func TestCategorizeMatching(t *testing.T) {
 		validateCategorySubcategory(t, "examples/unhealthy.yaml", Runtime, Unhealthy, "true")
 	})
 
-	t.Run("Pod stuck in Terminating", func(t *testing.T) {
-		ea := setupAnnotator(t)
-		ea.workloadCache = testhelper.NewFakeWorkloadCache("some-workload-name", "some-workload-kind", "some-node-name", util.GetPodStuckInTerminating())
-		validateAnnotations(t, ea, "examples/pod_terminating.yaml", Runtime, Terminating, "true")
-	})
-
 	t.Run("Out-of-memory killed", func(t *testing.T) {
 		validateCategorySubcategory(t, "examples/oom_killed.yaml", Runtime, OOMKilled, "true")
 	})
@@ -92,16 +85,6 @@ func TestCategorizeNonMatching(t *testing.T) {
 		validateCategorySubcategory(t, "examples/normal_pulling_image.yaml", "", "", "false")
 	})
 
-	t.Run("Pod Terminating Gracefully", func(t *testing.T) {
-		ea := setupAnnotator(t)
-		ea.workloadCache = testhelper.NewFakeWorkloadCache("some-workload-name", "some-workload-kind", "some-node-name", fakePod())
-		validateAnnotations(t, ea, "examples/pod_terminating.yaml", "", "", "false")
-	})
-
-	t.Run("Pod Terminating event, pod not found", func(t *testing.T) {
-		validateCategorySubcategory(t, "examples/pod_terminating.yaml", "", "", "false")
-	})
-
 }
 
 func validateCategorySubcategory(t *testing.T, file, category, subcategory, important string) {
@@ -116,7 +99,7 @@ func validateAnnotations(t *testing.T, ea *EventAnnotator, file, category, subca
 		ea.annotate(&event)
 		require.Equal(t, category, event.ObjectMeta.Annotations["aria/category"])
 		require.Equal(t, subcategory, event.ObjectMeta.Annotations["aria/subcategory"])
-		require.Equal(t, important, event.ObjectMeta.Annotations["internal/important"])
+		require.Equal(t, important, event.ObjectMeta.Annotations["important"])
 	}
 }
 

diff --git a/collector/plugins/events/eventrouter.go b/collector/plugins/events/eventrouter.go
@@ -146,7 +146,7 @@ func (er *EventRouter) addEvent(obj interface{}, isInInitialList bool) {
 		"reason":         e.Reason,
 		"component":      e.Source.Component,
 		"type":           e.Type,
-		"important":      e.Annotations["internal/important"],
+		"important":      e.Annotations["important"],
 	}
 
 	resourceName := e.InvolvedObject.Name
@@ -166,7 +166,7 @@ func (er *EventRouter) addEvent(obj interface{}, isInInitialList bool) {
 		filteredEvents.Inc(1)
 		return
 	}
-	delete(e.Annotations, "internal/important")
+	delete(e.Annotations, "important")
 	delete(tags, "important")
 	sentEvents.Inc(1)
 

diff --git a/collector/plugins/events/eventrouter_test.go b/collector/plugins/events/eventrouter_test.go
@@ -124,7 +124,7 @@ func TestAddEvent(t *testing.T) {
 		event.Type = v1.EventTypeWarning
 		er.addEvent(event, false)
 		require.Empty(t, sink.Annotations["important"])
-		require.Empty(t, event.Annotations["internal/important"])
+		require.Empty(t, event.Annotations["important"])
 	})
 }
 

diff --git a/collector/plugins/events/examples/pod_terminating.yaml b/collector/plugins/events/examples/pod_terminating.yaml
diff --git a/collector/plugins/processors/pod_based_enricher.go b/collector/plugins/processors/pod_based_enricher.go
@@ -228,6 +228,7 @@ func (pbe *PodBasedEnricher) addWorkloadStatusMetric(podMs *metrics.Set, pod *ku
 			} else if containerStatus.State.Terminated != nil {
 				workloadStatus = 0
 				workloadMs.Labels[metrics.LabelReason.Key] = containerStatus.State.Terminated.Reason
+				// TODO: add message for pod stuck in terminating
 				workloadMs.Labels[metrics.LabelMessage.Key] = containerStatus.State.Terminated.Message
 				break
 			}

diff --git a/collector/plugins/sources/kstate/non_running_pods.go b/collector/plugins/sources/kstate/non_running_pods.go
@@ -163,6 +163,7 @@ func buildPodTerminatingMetrics(pod *v1.Pod, sharedTags map[string]string, trans
 	tags["reason"] = "Terminating"
 
 	for _, condition := range pod.Status.Conditions {
+		// TODO: remove terminating logic from non_running_pods and put them under pod_based_enricher
 		if condition.Type == v1.PodScheduled && condition.Status == "False" {
 			tags[metrics.LabelNodename.Key] = "none"
 		}