From 7c122450f1e25c40025b782bb449ab8ece4c899a Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 18 Sep 2024 17:36:21 +0100 Subject: [PATCH] agent: rename SourceCapture's schemaMode to targetSchema --- ...controllers__test__status_json_schema.snap | 55 ------------------- .../src/integration_tests/source_captures.rs | 2 +- crates/agent/src/resource_configs.rs | 4 +- crates/models/src/source_capture.rs | 4 +- ...a_generation__catalog_schema_snapshot.snap | 52 +++++++++++++++++- flow.schema.json | 52 +++++++++++++++++- 6 files changed, 107 insertions(+), 62 deletions(-) diff --git a/crates/agent/src/controllers/snapshots/agent__controllers__test__status_json_schema.snap b/crates/agent/src/controllers/snapshots/agent__controllers__test__status_json_schema.snap index 73f14f2baa..26fd7b65f4 100644 --- a/crates/agent/src/controllers/snapshots/agent__controllers__test__status_json_schema.snap +++ b/crates/agent/src/controllers/snapshots/agent__controllers__test__status_json_schema.snap @@ -33,14 +33,6 @@ expression: schema ], "type": "object" }, - "Capture": { - "description": "Capture names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.", - "examples": [ - "acmeCo/capture" - ], - "pattern": "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$", - "type": "string" - }, "Collection": { "description": "Collection names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.", "examples": [ @@ -402,47 +394,6 @@ expression: schema ], "type": "object" }, - "SourceCaptureDef": { - "additionalProperties": false, - "description": "SourceCaptureDef specifies configuration for source captures", - "properties": { - "capture": { - "$ref": "#/definitions/Capture", - "description": "Capture name" - }, - "deltaUpdates": { - "description": "When adding new bindings from a source capture to a materialization, should the new bindings be marked as delta updates", - "type": "boolean" - }, - "schemaMode": { - "$ref": "#/definitions/SourceCaptureSchemaMode", - "default": "leaveEmpty", - "description": "When adding new bindings from a source capture to a materialization, how should the schema of the materialization binding be set" - } - }, - "required": [ - "capture" - ], - "type": "object" - }, - "SourceCaptureSchemaMode": { - "oneOf": [ - { - "description": "Leave the materialization binding's schema field empty, therefore falling back to the default schema of the materialization", - "enum": [ - "leaveEmpty" - ], - "type": "string" - }, - { - "description": "Use the 2nd-to-last component of the collection name as the schema of the materialization binding", - "enum": [ - "collectionSchema" - ], - "type": "string" - } - ] - }, "SourceCaptureStatus": { "description": "Status information about the `sourceCapture`", "properties": { @@ -454,17 +405,11 @@ expression: schema "type": "array", "uniqueItems": true }, - "source_capture": { - "$ref": "#/definitions/SourceCaptureDef" - }, "up_to_date": { "description": "Whether the materialization bindings are up-to-date with respect to the `sourceCapture` bindings. In normal operation, this should always be `true`. Otherwise, there will be a controller `error` and the publication status will contain details of why the update failed.", "type": "boolean" } }, - "required": [ - "source_capture" - ], "type": "object" } }, diff --git a/crates/agent/src/integration_tests/source_captures.rs b/crates/agent/src/integration_tests/source_captures.rs index 03826136a3..7b72b0633d 100644 --- a/crates/agent/src/integration_tests/source_captures.rs +++ b/crates/agent/src/integration_tests/source_captures.rs @@ -168,7 +168,7 @@ async fn test_source_captures_collection_name() { "ducks/materializeA": { "sourceCapture": { "capture": "ducks/capture", - "schemaMode": "collectionSchema", + "targetSchema": "fromSourceName", "deltaUpdates": true, }, "endpoint": { diff --git a/crates/agent/src/resource_configs.rs b/crates/agent/src/resource_configs.rs index 2c435d3467..b3f7b0c09e 100644 --- a/crates/agent/src/resource_configs.rs +++ b/crates/agent/src/resource_configs.rs @@ -1,4 +1,4 @@ -use models::{SourceCaptureSchemaMode, SourceCapture, SourceCaptureDef}; +use models::{SourceCaptureSchemaMode, SourceCapture}; use serde_json::Value; /// @@ -35,7 +35,7 @@ pub fn update_materialization_resource_spec( let source_capture_def = source_capture.to_normalized_def(); - if source_capture_def.schema_mode == SourceCaptureSchemaMode::CollectionSchema { + if source_capture_def.target_schema == SourceCaptureSchemaMode::FromSourceName { if let Some(x_schema_name_ptr) = &resource_spec_pointers.x_schema_name { if let Some(x_schema_name_prev) = x_schema_name_ptr.create_value(resource_spec) { let _ = std::mem::replace(x_schema_name_prev, x_schema_name.into()); diff --git a/crates/models/src/source_capture.rs b/crates/models/src/source_capture.rs index 3a60528d25..b606a3d7a2 100644 --- a/crates/models/src/source_capture.rs +++ b/crates/models/src/source_capture.rs @@ -10,7 +10,7 @@ pub enum SourceCaptureSchemaMode { LeaveEmpty, /// Use the 2nd-to-last component of the collection name as the schema of the materialization /// binding - CollectionSchema, + FromSourceName, } impl Default for SourceCaptureSchemaMode { @@ -29,7 +29,7 @@ pub struct SourceCaptureDef { /// When adding new bindings from a source capture to a materialization, how should the schema /// of the materialization binding be set #[serde(default, skip_serializing_if = "super::is_default")] - pub schema_mode: SourceCaptureSchemaMode, + pub target_schema: SourceCaptureSchemaMode, /// When adding new bindings from a source capture to a materialization, should the new /// bindings be marked as delta updates diff --git a/crates/sources/tests/snapshots/schema_generation__catalog_schema_snapshot.snap b/crates/sources/tests/snapshots/schema_generation__catalog_schema_snapshot.snap index 2e1f3d771f..5d201e7603 100644 --- a/crates/sources/tests/snapshots/schema_generation__catalog_schema_snapshot.snap +++ b/crates/sources/tests/snapshots/schema_generation__catalog_schema_snapshot.snap @@ -884,7 +884,7 @@ expression: "&schema" }, "sourceCapture": { "title": "Automatically materialize new bindings from a named capture", - "$ref": "#/definitions/Capture" + "$ref": "#/definitions/SourceCapture" } }, "additionalProperties": false @@ -1217,6 +1217,56 @@ expression: "&schema" } ] }, + "SourceCapture": { + "anyOf": [ + { + "$ref": "#/definitions/Capture" + }, + { + "$ref": "#/definitions/SourceCaptureDef" + } + ] + }, + "SourceCaptureDef": { + "description": "SourceCaptureDef specifies configuration for source captures", + "type": "object", + "required": [ + "capture" + ], + "properties": { + "capture": { + "description": "Capture name", + "$ref": "#/definitions/Capture" + }, + "deltaUpdates": { + "description": "When adding new bindings from a source capture to a materialization, should the new bindings be marked as delta updates", + "type": "boolean" + }, + "targetSchema": { + "description": "When adding new bindings from a source capture to a materialization, how should the schema of the materialization binding be set", + "$ref": "#/definitions/SourceCaptureSchemaMode" + } + }, + "additionalProperties": false + }, + "SourceCaptureSchemaMode": { + "oneOf": [ + { + "description": "Leave the materialization binding's schema field empty, therefore falling back to the default schema of the materialization", + "type": "string", + "enum": [ + "leaveEmpty" + ] + }, + { + "description": "Use the 2nd-to-last component of the collection name as the schema of the materialization binding", + "type": "string", + "enum": [ + "fromSourceName" + ] + } + ] + }, "Test": { "description": "Test names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.", "examples": [ diff --git a/flow.schema.json b/flow.schema.json index 1c17046035..16c83fbe35 100644 --- a/flow.schema.json +++ b/flow.schema.json @@ -880,7 +880,7 @@ }, "sourceCapture": { "title": "Automatically materialize new bindings from a named capture", - "$ref": "#/definitions/Capture" + "$ref": "#/definitions/SourceCapture" } }, "additionalProperties": false @@ -1213,6 +1213,56 @@ } ] }, + "SourceCapture": { + "anyOf": [ + { + "$ref": "#/definitions/Capture" + }, + { + "$ref": "#/definitions/SourceCaptureDef" + } + ] + }, + "SourceCaptureDef": { + "description": "SourceCaptureDef specifies configuration for source captures", + "type": "object", + "required": [ + "capture" + ], + "properties": { + "capture": { + "description": "Capture name", + "$ref": "#/definitions/Capture" + }, + "deltaUpdates": { + "description": "When adding new bindings from a source capture to a materialization, should the new bindings be marked as delta updates", + "type": "boolean" + }, + "targetSchema": { + "description": "When adding new bindings from a source capture to a materialization, how should the schema of the materialization binding be set", + "$ref": "#/definitions/SourceCaptureSchemaMode" + } + }, + "additionalProperties": false + }, + "SourceCaptureSchemaMode": { + "oneOf": [ + { + "description": "Leave the materialization binding's schema field empty, therefore falling back to the default schema of the materialization", + "type": "string", + "enum": [ + "leaveEmpty" + ] + }, + { + "description": "Use the 2nd-to-last component of the collection name as the schema of the materialization binding", + "type": "string", + "enum": [ + "fromSourceName" + ] + } + ] + }, "Test": { "description": "Test names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.", "examples": [