remove kubernetes.pod.terminating metric (#483)

Co-authored-by: Yuqi Jin <[email protected]>
Co-authored-by: Priya Selvaganesan <[email protected]>

collector/plugins/sources/kstate/non_running_pods.go

@@ -33,9 +33,6 @@ func pointsForNonRunningPods(workloadCache util.WorkloadCache) func(item interfa
 		now := time.Now().Unix()
 
 		points := buildPodPhaseMetrics(pod, transforms, sharedTags, now)
-		if pod.DeletionTimestamp != nil {
-			points = append(points, buildPodTerminatingMetrics(pod, sharedTags, transforms, now)...)
-		}
 
 		points = append(points, buildContainerStatusMetrics(pod, sharedTags, transforms, now)...)
 
@@ -155,32 +152,6 @@ func buildPodPhaseMetrics(pod *v1.Pod, transforms configuration.Transforms, shar
 	return points
 }
 
-func buildPodTerminatingMetrics(pod *v1.Pod, sharedTags map[string]string, transforms configuration.Transforms, now int64) []wf.Metric {
-	tags := buildTags("pod_name", pod.Name, pod.Namespace, transforms.Tags)
-	tags[metrics.LabelMetricSetType.Key] = metrics.MetricSetTypePod
-	tags[metrics.LabelPodId.Key] = string(pod.UID)
-	tags["DeletionTimestamp"] = pod.DeletionTimestamp.Format(time.RFC3339)
-	tags["reason"] = "Terminating"
-
-	for _, condition := range pod.Status.Conditions {
-		// TODO: remove terminating logic from non_running_pods and put them under pod_based_enricher
-		if condition.Type == v1.PodScheduled && condition.Status == "False" {
-			tags[metrics.LabelNodename.Key] = "none"
-		}
-	}
-
-	nodeName := pod.Spec.NodeName
-	if len(nodeName) > 0 {
-		sharedTags[metrics.LabelNodename.Key] = nodeName
-	}
-
-	copyTags(sharedTags, tags)
-	points := []wf.Metric{
-		metricPoint(transforms.Prefix, "pod.terminating", 1, now, transforms.Source, tags),
-	}
-	return points
-}
-
 func buildContainerStatusMetrics(pod *v1.Pod, sharedTags map[string]string, transforms configuration.Transforms, now int64) []wf.Metric {
 	statuses := pod.Status.ContainerStatuses
 	if len(statuses) == 0 {

collector/plugins/sources/kstate/non_running_pods_test.go

@@ -1,7 +1,6 @@
 package kstate
 
 import (
-	"fmt"
 	"testing"
 	"time"
 
@@ -240,38 +239,6 @@ func TestPointsForNonRunningPods(t *testing.T) {
 		assert.Equal(t, "some-workload-name", point.Tags()[workloadNameTag])
 	})
 
-	t.Run("test for terminating pod without nodename", func(t *testing.T) {
-		deletionTime := "2023-06-08T18:35:37Z"
-		testPod := setupTerminatingPod(t, deletionTime, "")
-		workloadCache := fakeWorkloadCache()
-		actualWFPoints := pointsForNonRunningPods(workloadCache)(testPod, testTransform)
-		assert.Equal(t, 2, len(actualWFPoints))
-		point := actualWFPoints[1].(*wf.Point)
-		assert.Equal(t, fmt.Sprintf("%spod.terminating", testTransform.Prefix), point.Metric)
-		assert.Equal(t, float64(util.POD_PHASE_PENDING), point.Value)
-		assert.Equal(t, deletionTime, point.Tags()["DeletionTimestamp"])
-		assert.Equal(t, "pod1", point.Tags()["pod_name"])
-		assert.Equal(t, "testLabelName", point.Tags()["label.name"])
-		assert.Equal(t, "none", point.Tags()["nodename"])
-		assert.Equal(t, "Terminating", point.Tags()["reason"])
-	})
-
-	t.Run("test for terminating pod with nodename", func(t *testing.T) {
-		deletionTime := "2023-06-08T18:35:37Z"
-		testPod := setupTerminatingPod(t, deletionTime, "some-node")
-		workloadCache := fakeWorkloadCache()
-		actualWFPoints := pointsForNonRunningPods(workloadCache)(testPod, testTransform)
-		assert.Equal(t, 2, len(actualWFPoints))
-		point := actualWFPoints[1].(*wf.Point)
-		assert.Equal(t, fmt.Sprintf("%spod.terminating", testTransform.Prefix), point.Metric)
-		assert.Equal(t, float64(util.POD_PHASE_PENDING), point.Value)
-		assert.Equal(t, deletionTime, point.Tags()["DeletionTimestamp"])
-		assert.Equal(t, "pod1", point.Tags()["pod_name"])
-		assert.Equal(t, "testLabelName", point.Tags()["label.name"])
-		assert.Equal(t, "some-node", point.Tags()["nodename"])
-		assert.Equal(t, "Terminating", point.Tags()["reason"])
-	})
-
 	t.Run("test for completed pod", func(t *testing.T) {
 		testPod := setupCompletedPod()
 		workloadCache := fakeWorkloadCache()

operator/dev-internal/docs/alerts/templates/pod-stuck-in-terminating.json.tmpl

@@ -13,7 +13,7 @@
     },
     {
       "name": "B",
-      "query": "count(ts(\"kubernetes.pod.terminating\", cluster=\"K8S_CLUSTER_NAME\"), sources, cluster, cluster_uuid, namespace_name, workload_name, workload_kind, pod_name, reason, DeletionTimestamp, type, metrics)",
+      "query": "count(ts(\"kubernetes.pod_container.status\", cluster=\"K8S_CLUSTER_NAME\" AND status=\"terminated\"), sources, cluster, cluster_uuid, namespace_name, workload_name, workload_kind, pod_name, reason, type, metrics)",
       "queryType": "WQL",
       "alertSourceType": ["VARIABLE"],
       "hidden": true