diff --git a/lib/kube.nix b/lib/kube.nix
index 2f28793..b2d8b08 100644
--- a/lib/kube.nix
+++ b/lib/kube.nix
@@ -92,17 +92,50 @@
     # List of labels that should be removed
     labels:
     # Kubernetes manifest
-    manifest:
-      manifest
-      // {
-        metadata =
-          manifest.metadata
-          // (
-            if manifest.metadata ? labels
-            then {
-              labels = removeAttrs manifest.metadata.labels labels;
-            }
-            else {}
-          );
-      };
+    manifest: let
+      updateFunc = old: removeAttrs old labels;
+
+      hasLabelPath = p: res: (lib.attrByPath p null res) != null;
+
+      specialTemplateKinds = [
+        "Deployment"
+        "ReplicaSet"
+        "StatefulSet"
+        "DaemonSet"
+        "Job"
+      ];
+    in
+      lib.attrsets.updateManyAttrsByPath (
+        # If metadata.labels is present, it should be filtered
+        (lib.optional (manifest.metadata ? labels) {
+          path = ["metadata" "labels"];
+          update = updateFunc;
+        })
+        # If it's one of the special kinds with
+        # spec.template.metadata.labels, that should be filtered
+        # too
+        ++ (lib.optional (
+            (lib.any (k: manifest.kind == k) specialTemplateKinds)
+            && (hasLabelPath ["spec" "template" "metadata" "labels"] manifest)
+          ) {
+            path = ["spec" "template" "metadata" "labels"];
+            update = updateFunc;
+          })
+        # CronJob needs to be filtered differently too
+        ++ (
+          lib.optionals
+          (manifest.kind == "CronJob")
+          (
+            (lib.optional (hasLabelPath ["spec" "jobTemplate" "metadata" "labels"] manifest) {
+              path = ["spec" "jobTemplate" "metadata" "labels"];
+              update = updateFunc;
+            })
+            ++ (lib.optional (hasLabelPath ["spec" "jobTemplate" "spec" "template" "metadata" "labels"] manifest) {
+              path = ["spec" "jobTemplate" "spec" "template" "metadata" "labels"];
+              update = updateFunc;
+            })
+          )
+        )
+      )
+      manifest;
 }
diff --git a/lib/tests.nix b/lib/tests.nix
index 47ad7f2..7ef47d7 100644
--- a/lib/tests.nix
+++ b/lib/tests.nix
@@ -107,6 +107,106 @@ in {
           };
         };
       };
+      testNoLabels = {
+        expr = lib.kube.removeLabels ["helm.sh/chart"] {
+          apiVersion = "apps/v1";
+          kind = "Deployment";
+          metadata = {
+            name = "argocd";
+          };
+        };
+        expected = {
+          apiVersion = "apps/v1";
+          kind = "Deployment";
+          metadata = {
+            name = "argocd";
+          };
+        };
+      };
+      testSpecialTemplateLabels = {
+        expr = lib.kube.removeLabels ["helm.sh/chart"] {
+          apiVersion = "apps/v1";
+          kind = "Deployment";
+          metadata = {
+            name = "test1-chart";
+            namespace = "test1";
+            labels = {
+              "app.kubernetes.io/instance" = "test1";
+              "app.kubernetes.io/managed-by" = "Helm";
+              "app.kubernetes.io/name" = "chart";
+              "helm.sh/chart" = "chart-0.1.0";
+            };
+          };
+          spec = {
+            replicas = 1;
+            selector.matchLabels = {
+              "app.kubernetes.io/instance" = "test1";
+              "app.kubernetes.io/name" = "chart";
+            };
+            template = {
+              metadata.labels = {
+                "app.kubernetes.io/instance" = "test1";
+                "app.kubernetes.io/managed-by" = "Helm";
+                "app.kubernetes.io/name" = "chart";
+                "helm.sh/chart" = "chart-0.1.0";
+              };
+              spec.containers = [
+                {
+                  name = "chart";
+                  image = "nginx:latest";
+                  ports = [
+                    {
+                      name = "http";
+                      containerPort = 80;
+                      protocol = "TCP";
+                    }
+                  ];
+                }
+              ];
+            };
+          };
+        };
+        expected = {
+          apiVersion = "apps/v1";
+          kind = "Deployment";
+          metadata = {
+            name = "test1-chart";
+            namespace = "test1";
+            labels = {
+              "app.kubernetes.io/instance" = "test1";
+              "app.kubernetes.io/managed-by" = "Helm";
+              "app.kubernetes.io/name" = "chart";
+            };
+          };
+          spec = {
+            replicas = 1;
+            selector.matchLabels = {
+              "app.kubernetes.io/instance" = "test1";
+              "app.kubernetes.io/name" = "chart";
+            };
+            template = {
+              metadata.labels = {
+                "app.kubernetes.io/instance" = "test1";
+                "app.kubernetes.io/managed-by" = "Helm";
+                "app.kubernetes.io/name" = "chart";
+              };
+              spec.containers = [
+                {
+                  name = "chart";
+                  image = "nginx:latest";
+                  ports = [
+                    {
+                      name = "http";
+                      containerPort = 80;
+                      protocol = "TCP";
+                    }
+                  ];
+                }
+              ];
+            };
+          };
+        };
+      };
     };
   };
 }
diff --git a/tests/helm/transformer.nix b/tests/helm/transformer.nix
index d4dec37..6c901d9 100644
--- a/tests/helm/transformer.nix
+++ b/tests/helm/transformer.nix
@@ -52,8 +52,6 @@ in {
                 "app.kubernetes.io/instance" = "test1";
                 "app.kubernetes.io/managed-by" = "Helm";
                 "app.kubernetes.io/name" = "chart";
-                "app.kubernetes.io/version" = "1.16.0";
-                "helm.sh/chart" = "chart-0.1.0";
               };
               spec.containers = [
                 {
@@ -142,8 +140,6 @@ in {
                 "app.kubernetes.io/instance" = "test1";
                 "app.kubernetes.io/managed-by" = "Helm";
                 "app.kubernetes.io/name" = "chart";
-                "app.kubernetes.io/version" = "1.16.0";
-                "helm.sh/chart" = "chart-0.1.0";
               };
               spec.containers = [
                 {