diff --git a/pkg/kapp/clusterapply/add_or_update_change.go b/pkg/kapp/clusterapply/add_or_update_change.go
index c569b8d29..bdc53cb28 100644
--- a/pkg/kapp/clusterapply/add_or_update_change.go
+++ b/pkg/kapp/clusterapply/add_or_update_change.go
@@ -12,6 +12,7 @@ import (
 	ctlres "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/resources"
 	"github.com/vmware-tanzu/carvel-kapp/pkg/kapp/util"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 const (
@@ -252,7 +253,7 @@ func (c AddOrUpdateChange) recordAppliedResource(savedRes ctlres.Resource) error
 		}
 
 		// Record last applied change on the latest version of a resource
-		latestResWithHistoryUpdated, madeAnyModifications, err := latestResWithHistory.RecordLastAppliedResource(applyChange)
+		annotationStr, madeAnyModifications, err := latestResWithHistory.RecordLastAppliedResource(applyChange)
 		if err != nil {
 			return true, fmt.Errorf("Recording last applied resource: %w", err)
 		}
@@ -262,12 +263,14 @@ func (c AddOrUpdateChange) recordAppliedResource(savedRes ctlres.Resource) error
 			return true, nil
 		}
 
-		_, err = c.identifiedResources.Update(latestResWithHistoryUpdated)
+		jsonStr := fmt.Sprintf("{\"metadata\": {\"annotations\": %s }}", annotationStr)
+		data := []byte(jsonStr)
+
+		_, err = c.identifiedResources.Patch(savedRes, types.MergePatchType, data)
 		if err != nil {
 			latestResWithHistory = nil // Get again
 			return false, fmt.Errorf("Saving record of last applied resource: %w", err)
 		}
-
 		return true, nil
 	})
 }
diff --git a/pkg/kapp/diff/resource_with_history.go b/pkg/kapp/diff/resource_with_history.go
index 34115aa7f..7d24f740d 100644
--- a/pkg/kapp/diff/resource_with_history.go
+++ b/pkg/kapp/diff/resource_with_history.go
@@ -4,6 +4,7 @@
 package diff
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 
@@ -68,13 +69,13 @@ func (r ResourceWithHistory) AllowsRecordingLastApplied() bool {
 	return !found
 }
 
-func (r ResourceWithHistory) RecordLastAppliedResource(appliedChange Change) (ctlres.Resource, bool, error) {
+func (r ResourceWithHistory) RecordLastAppliedResource(appliedChange Change) (string, bool, error) {
 	// Use compact representation to take as little space as possible
 	// because annotation value max length is 262144 characters
 	// (https://github.com/vmware-tanzu/carvel-kapp/issues/48).
 	appliedResBytes, err := appliedChange.AppliedResource().AsCompactBytes()
 	if err != nil {
-		return nil, true, err
+		return "", true, err
 	}
 
 	diff := appliedChange.OpsDiff()
@@ -84,37 +85,30 @@ func (r ResourceWithHistory) RecordLastAppliedResource(appliedChange Change) (ct
 			r.resource.Description(), diff.MinimalMD5(), diff.MinimalString())
 	}
 
-	annsMod := ctlres.StringMapAppendMod{
-		ResourceMatcher: ctlres.AllMatcher{},
-		Path:            ctlres.NewPathFromStrings([]string{"metadata", "annotations"}),
-		KVs: map[string]string{
-			appliedResAnnKey:        string(appliedResBytes),
-			appliedResDiffMD5AnnKey: diff.MinimalMD5(),
+	annsKVS := map[string]string{
+		appliedResAnnKey:        string(appliedResBytes),
+		appliedResDiffMD5AnnKey: diff.MinimalMD5(),
 
-			// Following fields useful for debugging:
-			//   debugAppliedResDiffAnnKey:     diff.MinimalString(),
-			//   debugAppliedResDiffFullAnnKey: diff.FullString(),
-		},
+		// Following fields useful for debugging:
+		//   debugAppliedResDiffAnnKey:     diff.MinimalString(),
+		//   debugAppliedResDiffFullAnnKey: diff.FullString(),
 	}
 
 	const annValMaxLen = 262144
 
 	// kapp deploy should work without adding disable annotation when annotation value max length exceed
 	// (https://github.com/vmware-tanzu/carvel-kapp/issues/410)
-	for _, annVal := range annsMod.KVs {
+	for _, annVal := range annsKVS {
 		if len(annVal) > annValMaxLen {
-			return nil, false, nil
+			return "", false, nil
 		}
 	}
 
-	resultRes := r.resource.DeepCopy()
-
-	err = annsMod.Apply(resultRes)
+	result, err := json.Marshal(annsKVS)
 	if err != nil {
-		return nil, true, err
+		return "", false, err
 	}
-
-	return resultRes, true, nil
+	return string(result), true, nil
 }
 
 func (r ResourceWithHistory) CalculateChange(appliedRes ctlres.Resource) (Change, error) {