source-mongodb: merge deletion events

This adds the (conditional) reduction strategy to deletion events to merge them into existing documents. For materializations using soft deletions, this will have the effect of only updating the `_meta/op` column to `"d"` for the materialized row, instead of removing all of the values from other non-key columns. This will make soft deletions for MongoDB-captured collections work the same way as other CDC connectors, while still using the top-level "last write wins" reductions strategy for other events, most notably updates, where we don't emit explicit `null` values for removed fields (yet).
estuary · Sep 13, 2024 · 0756999 · 0756999
1 parent 210aa06
commit 0756999
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 13 deletions.
diff --git a/source-mongodb/.snapshots/TestDiscover b/source-mongodb/.snapshots/TestDiscover
@@ -20,7 +20,7 @@ Binding 0:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -106,7 +106,7 @@ Binding 1:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",

diff --git a/source-mongodb/.snapshots/TestDiscoverAllDatabases b/source-mongodb/.snapshots/TestDiscoverAllDatabases
@@ -20,7 +20,7 @@ Binding 0:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -106,7 +106,7 @@ Binding 1:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -192,7 +192,7 @@ Binding 2:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -278,7 +278,7 @@ Binding 3:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -364,7 +364,7 @@ Binding 4:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -450,7 +450,7 @@ Binding 5:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",

diff --git a/source-mongodb/.snapshots/TestDiscoverMultipleDatabases b/source-mongodb/.snapshots/TestDiscoverMultipleDatabases
@@ -20,7 +20,7 @@ Binding 0:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -106,7 +106,7 @@ Binding 1:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -192,7 +192,7 @@ Binding 2:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",
@@ -278,7 +278,7 @@ Binding 3:
       "then": {
         "reduce": {
           "delete": true,
-          "strategy": "lastWriteWins"
+          "strategy": "merge"
         }
       },
       "type": "object",

diff --git a/source-mongodb/discovery.go b/source-mongodb/discovery.go
@@ -85,7 +85,7 @@ func generateMinimalSchema() json.RawMessage {
 		Then: &jsonschema.Schema{
 			Extras: map[string]interface{}{
 				"reduce": map[string]interface{}{
-					"strategy": "lastWriteWins",
+					"strategy": "merge",
 					"delete":   true,
 				},
 			},