prometheus: drop labels with duplicate nodename entries (#4201)

base/prometheus/prometheus.ConfigMap.yaml

@@ -168,6 +168,11 @@ data:
       - source_labels: [__meta_kubernetes_endpoint_node_name]
         action: replace
         target_label: nodename
+      metric_relabel_configs:
+      # Sourcegraph specific customization. Drop metrics with empty nodename responses from the k8s API
+      - source_labels: [nodename]
+        regex: ^$
+        action: drop
 
     # Example scrape config for probing services via the Blackbox Exporter.
     #
@@ -242,7 +247,7 @@ data:
         target_label: ns
       # Sourcegraph specific customization. We want to add a label to every 
       # metric that indicates the node it came from.
-      - source_labels: [__meta_kubernetes_endpoint_node_name]
+      - source_labels: [__meta_kubernetes_pod_node_name]
         action: replace
         target_label: nodename
 
@@ -265,6 +270,10 @@ data:
         action: replace
         target_label: name
         separator: '-'
+      # Sourcegraph specific customization. Drop metrics with empty nodename responses from the k8s API
+      - source_labels: [nodename]
+        regex: ^$
+        action: drop
 
     # Scrape prometheus itself for metrics.
     - job_name: 'builtin-prometheus'