diff --git a/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/TableCatalogFactory.java b/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/CatalogFactory.java similarity index 68% rename from bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/TableCatalogFactory.java rename to bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/CatalogFactory.java index d91229a51..a24b02952 100644 --- a/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/TableCatalogFactory.java +++ b/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/CatalogFactory.java @@ -18,8 +18,7 @@ import com.bytedance.bitsail.base.component.ComponentBuilder; import com.bytedance.bitsail.base.connector.BuilderGroup; -import com.bytedance.bitsail.base.execution.ExecutionEnviron; -import com.bytedance.bitsail.common.catalog.table.TableCatalog; +import com.bytedance.bitsail.common.catalog.table.Catalog; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import java.io.Serializable; @@ -27,16 +26,14 @@ /** * Created 2022/5/23 */ -public interface TableCatalogFactory extends Serializable, ComponentBuilder { +public interface CatalogFactory extends Serializable, ComponentBuilder { /** * Create a table catalog. * - * @param executionEnviron execution environment * @param connectorConfiguration configuration for the reader/writer */ - TableCatalog createTableCatalog(BuilderGroup builderGroup, - ExecutionEnviron executionEnviron, - BitSailConfiguration connectorConfiguration); + Catalog createTableCatalog(BuilderGroup builderGroup, + BitSailConfiguration connectorConfiguration); } diff --git a/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/TableCatalogFactoryHelper.java b/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/CatalogFactoryHelper.java similarity index 77% rename from bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/TableCatalogFactoryHelper.java rename to bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/CatalogFactoryHelper.java index 6e9695ab9..ccb5aaf3c 100644 --- a/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/TableCatalogFactoryHelper.java +++ b/bitsail-base/src/main/java/com/bytedance/bitsail/base/catalog/CatalogFactoryHelper.java @@ -20,11 +20,11 @@ import org.apache.commons.lang3.StringUtils; -public class TableCatalogFactoryHelper { +public class CatalogFactoryHelper { - public static TableCatalogFactory getTableCatalogFactory(String connectorName) { - DefaultComponentBuilderLoader loader = - new DefaultComponentBuilderLoader<>(TableCatalogFactory.class); + public static CatalogFactory getTableCatalogFactory(String connectorName) { + DefaultComponentBuilderLoader loader = + new DefaultComponentBuilderLoader<>(CatalogFactory.class); return loader.loadComponent(StringUtils.lowerCase(connectorName), false); } diff --git a/bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/SupportMultipleSinkTable.java b/bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/SupportMultipleSinkTable.java new file mode 100644 index 000000000..77e589942 --- /dev/null +++ b/bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/SupportMultipleSinkTable.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.base.extension; + +import com.bytedance.bitsail.base.connector.writer.v1.Writer; +import com.bytedance.bitsail.base.connector.writer.v1.WriterCommitter; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; + +import java.io.Serializable; +import java.util.Optional; + +public interface SupportMultipleSinkTable { + + Writer createWriter(Writer.Context context, + BitSailConfiguration templateConfiguration); + + Optional> createCommitter(BitSailConfiguration templateConfiguration); + + BitSailConfiguration applyTableId(BitSailConfiguration template, + TableId tableId); + +} diff --git a/bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/SupportProducedType.java b/bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/SupportProducedType.java new file mode 100644 index 000000000..49e3e00ca --- /dev/null +++ b/bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/SupportProducedType.java @@ -0,0 +1,26 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.base.extension; + +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; + +import java.io.Serializable; + +public interface SupportProducedType extends Serializable { + + RowTypeInfo getProducedType(); +} diff --git a/bitsail-common/pom.xml b/bitsail-common/pom.xml index 592da6c14..8acf05e3c 100644 --- a/bitsail-common/pom.xml +++ b/bitsail-common/pom.xml @@ -31,7 +31,6 @@ false - ${project.parent.basedir} diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/TableCatalogManager.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/TableCatalogManager.java index 59cc885b8..a797c1c03 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/TableCatalogManager.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/TableCatalogManager.java @@ -17,12 +17,12 @@ package com.bytedance.bitsail.common.catalog; import com.bytedance.bitsail.common.BitSailException; +import com.bytedance.bitsail.common.catalog.table.Catalog; import com.bytedance.bitsail.common.catalog.table.CatalogTable; import com.bytedance.bitsail.common.catalog.table.CatalogTableAlterDefinition; import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; -import com.bytedance.bitsail.common.catalog.table.CatalogTableDefinition; import com.bytedance.bitsail.common.catalog.table.CatalogTableSchema; -import com.bytedance.bitsail.common.catalog.table.TableCatalog; +import com.bytedance.bitsail.common.catalog.table.TableId; import com.bytedance.bitsail.common.catalog.table.TableOperation; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.model.ColumnInfo; @@ -50,8 +50,8 @@ public class TableCatalogManager { private final TypeInfoConverter readerTypeInfoConverter; private final TypeInfoConverter writerTypeInfoConverter; - private final TableCatalog readerTableCatalog; - private final TableCatalog writerTableCatalog; + private final Catalog readerCatalog; + private final Catalog writerCatalog; private final BitSailConfiguration commonConfiguration; private final BitSailConfiguration readerConfiguration; @@ -73,15 +73,15 @@ public class TableCatalogManager { @Builder public TableCatalogManager(TypeInfoConverter readerTypeInfoConverter, TypeInfoConverter writerTypeInfoConverter, - TableCatalog readerTableCatalog, - TableCatalog writerTableCatalog, + Catalog readerCatalog, + Catalog writerCatalog, BitSailConfiguration commonConfiguration, BitSailConfiguration readerConfiguration, BitSailConfiguration writerConfiguration) { this.readerTypeInfoConverter = readerTypeInfoConverter; this.writerTypeInfoConverter = writerTypeInfoConverter; - this.readerTableCatalog = readerTableCatalog; - this.writerTableCatalog = writerTableCatalog; + this.readerCatalog = readerCatalog; + this.writerCatalog = writerCatalog; this.commonConfiguration = commonConfiguration; this.readerConfiguration = readerConfiguration; this.writerConfiguration = writerConfiguration; @@ -102,7 +102,7 @@ private void prepareCatalogManager() { } public void alignmentCatalogTable() throws Exception { - if (Objects.isNull(readerTableCatalog) || Objects.isNull(writerTableCatalog)) { + if (Objects.isNull(readerCatalog) || Objects.isNull(writerCatalog)) { return; } @@ -114,28 +114,28 @@ public void alignmentCatalogTable() throws Exception { startTableCatalog(); try { - CatalogTableDefinition readerTableDefinition = readerTableCatalog.createCatalogTableDefinition(); - CatalogTableDefinition writerTableDefinition = readerTableCatalog.createCatalogTableDefinition(); - if (!readerTableCatalog.tableExists(readerTableDefinition)) { + TableId readerTableDefinition = readerCatalog.createCatalogTableDefinition(); + TableId writerTableDefinition = readerCatalog.createCatalogTableDefinition(); + if (!readerCatalog.tableExists(readerTableDefinition)) { throw BitSailException.asBitSailException(TableCatalogErrorCode.TABLE_CATALOG_TABLE_NOT_EXISTS, String.format("Reader table definition %s not exists.", readerTableDefinition)); } // get reader catalog table. - readerCatalogTable = readerTableCatalog.getCatalogTable(readerTableDefinition); + readerCatalogTable = readerCatalog.getCatalogTable(readerTableDefinition); - if (!writerTableCatalog.tableExists(writerTableDefinition)) { + if (!writerCatalog.tableExists(writerTableDefinition)) { if (!tableCatalogCreateTableNotExists) { throw BitSailException.asBitSailException(TableCatalogErrorCode.TABLE_CATALOG_TABLE_NOT_EXISTS, String.format("Writer table definition %s not exists.", writerTableDefinition)); } // try to create table when not exists. - writerTableCatalog.createTable(writerTableDefinition, readerCatalogTable); + writerCatalog.createTable(writerTableDefinition, readerCatalogTable); } // get writer catalog table. - writerCatalogTable = writerTableCatalog.getCatalogTable(writerTableDefinition); + writerCatalogTable = writerCatalog.getCatalogTable(writerTableDefinition); // get base table schema. CatalogTableSchema catalogTableSchema = tableCatalogStrategy @@ -190,7 +190,7 @@ private void alterCatalogSchema(CatalogTableAlterDefinition catalogTableAlterDef CollectionUtils.isNotEmpty(catalogTableAlterDefinition.getPendingAddColumns())) { LOG.info("Writer catalog table {} try to add column: {}.", writerCatalogTable, catalogTableAlterDefinition.getPendingAddColumns()); - writerTableCatalog.alterTableColumns( + writerCatalog.alterTableColumns( TableOperation.ALTER_COLUMNS_ADD, catalogTableAlterDefinition.getPendingAddColumns() ); @@ -200,7 +200,7 @@ private void alterCatalogSchema(CatalogTableAlterDefinition catalogTableAlterDef CollectionUtils.isNotEmpty(catalogTableAlterDefinition.getPendingUpdateColumns())) { LOG.info("Writer catalog table {} try to update column: {}.", writerCatalogTable, catalogTableAlterDefinition.getPendingUpdateColumns()); - writerTableCatalog.alterTableColumns( + writerCatalog.alterTableColumns( TableOperation.ALTER_COLUMNS_UPDATE, catalogTableAlterDefinition.getPendingUpdateColumns() ); @@ -210,7 +210,7 @@ private void alterCatalogSchema(CatalogTableAlterDefinition catalogTableAlterDef CollectionUtils.isNotEmpty(catalogTableAlterDefinition.getPendingDeleteColumns())) { LOG.info("Writer catalog table {} try to delete column: {}.", writerCatalogTable, catalogTableAlterDefinition.getPendingDeleteColumns()); - writerTableCatalog.alterTableColumns( + writerCatalog.alterTableColumns( TableOperation.ALTER_COLUMNS_DELETE, catalogTableAlterDefinition.getPendingDeleteColumns() ); @@ -262,7 +262,7 @@ private CatalogTableAlterDefinition calNecessaryCatalogSchema(CatalogTableSchema TypeInfo baseTypeInfo = catalogTableColumn.getType(); finalCatalogColumns.add(catalogTableColumn); - if (!writerTableCatalog.compareTypeCompatible(writerTypeInfo, baseTypeInfo)) { + if (!writerCatalog.compareTypeCompatible(writerTypeInfo, baseTypeInfo)) { pendingUpdateTableColumns.add(catalogTableColumn); } } else { @@ -279,13 +279,13 @@ private CatalogTableAlterDefinition calNecessaryCatalogSchema(CatalogTableSchema } private void startTableCatalog() { - readerTableCatalog.open(readerTypeInfoConverter); - writerTableCatalog.open(writerTypeInfoConverter); + readerCatalog.open(readerTypeInfoConverter); + writerCatalog.open(writerTypeInfoConverter); } private void closeTableCatalog() { - readerTableCatalog.close(); - writerTableCatalog.close(); + readerCatalog.close(); + writerCatalog.close(); } } diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/TableCatalog.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/Catalog.java similarity index 87% rename from bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/TableCatalog.java rename to bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/Catalog.java index a8cfe1d23..8659b07d4 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/TableCatalog.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/Catalog.java @@ -28,7 +28,7 @@ /** * Table catalog only for the signal table for now. */ -public interface TableCatalog extends Serializable { +public interface Catalog extends Serializable { /** * Open Table catalog @@ -43,22 +43,27 @@ public interface TableCatalog extends Serializable { /** * Get the reference table for the table catalog. */ - CatalogTableDefinition createCatalogTableDefinition(); + TableId createCatalogTableDefinition(); + + /** + * List table under the database + */ + List listTables(); /** * Check the table exits or not. */ - boolean tableExists(CatalogTableDefinition catalogTableDefinition); + boolean tableExists(TableId catalogTableDefinition); /** * Acquire catalog table by the table definition. */ - CatalogTable getCatalogTable(CatalogTableDefinition catalogTableDefinition); + CatalogTable getCatalogTable(TableId catalogTableDefinition); /** * Create table */ - void createTable(CatalogTableDefinition catalogTableDefinition, + void createTable(TableId catalogTableDefinition, CatalogTable catalogTable); /** diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTable.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTable.java index 85d7c2d4e..3deebf791 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTable.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTable.java @@ -25,21 +25,21 @@ @Builder public class CatalogTable implements Serializable { - private CatalogTableDefinition catalogTableDefinition; + private TableId tableId; private final CatalogTableSchema catalogTableSchema; private final String comment; - public CatalogTable(CatalogTableDefinition catalogTableDefinition, + public CatalogTable(TableId tableId, CatalogTableSchema catalogTableSchema) { - this(catalogTableDefinition, catalogTableSchema, null); + this(tableId, catalogTableSchema, null); } - public CatalogTable(CatalogTableDefinition catalogTableDefinition, + public CatalogTable(TableId tableId, CatalogTableSchema catalogTableSchema, String comment) { - this.catalogTableDefinition = catalogTableDefinition; + this.tableId = tableId; this.catalogTableSchema = catalogTableSchema; this.comment = comment; } @@ -47,7 +47,7 @@ public CatalogTable(CatalogTableDefinition catalogTableDefinition, @Override public String toString() { return "CatalogTable{" + - "catalogTableDefinition=" + catalogTableDefinition + + "catalogTableDefinition=" + tableId + ", catalogTableSchema=" + catalogTableSchema + ", comment='" + comment + '\'' + '}'; diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTableColumn.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTableColumn.java index 3911849ec..4e8a0e5f1 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTableColumn.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTableColumn.java @@ -37,6 +37,8 @@ public class CatalogTableColumn implements Serializable { private String comment; + private Object defaultValue; + public CatalogTableColumn(String name, TypeInfo type) { this.name = name; this.type = type; diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTableDefinition.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTableDefinition.java deleted file mode 100644 index 0170fcf22..000000000 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/CatalogTableDefinition.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.bytedance.bitsail.common.catalog.table; - -import lombok.Builder; -import lombok.Getter; - -import java.io.Serializable; - -@Getter -@Builder -public class CatalogTableDefinition implements Serializable { - - private final String database; - private final String schema; - private final String table; - - public CatalogTableDefinition(String database, String schema, String table) { - this.database = database; - this.schema = schema; - this.table = table; - } - - @Override - public String toString() { - return "CatalogTableDefinition{" + - "database='" + database + '\'' + - ", schema='" + schema + '\'' + - ", table='" + table + '\'' + - '}'; - } -} diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/TableId.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/TableId.java new file mode 100644 index 000000000..39000c5c2 --- /dev/null +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/catalog/table/TableId.java @@ -0,0 +1,116 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.common.catalog.table; + +import lombok.Builder; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; + +import java.io.Serializable; +import java.util.Objects; + +@Getter +@Builder +public class TableId implements Serializable { + private static final int DEFAULT_TABLE_ID_LENGTH = 2; + private static final int CONTAINS_SCHEMA_TABLE_ID_LENGTH = 3; + + private final String database; + private final String schema; + private final String table; + + public TableId(String database, String schema, String table) { + this.database = database; + this.schema = schema; + this.table = table; + } + + public static TableId of(String tableId) { + String[] paths = tableId.split("\\."); + + if (paths.length == DEFAULT_TABLE_ID_LENGTH) { + return of(paths[0], paths[1]); + } + if (paths.length == CONTAINS_SCHEMA_TABLE_ID_LENGTH) { + return of(paths[0], paths[1], paths[2]); + } + throw new IllegalArgumentException( + String.format("Can't get table-id from value: %s", tableId)); + } + + public static TableId of(String database, String table) { + return of(database, null, table); + } + + public static TableId of(String database, String schema, String table) { + return new TableId(database, schema, table); + } + + public String getQuotedName() { + return getQuotedName("`"); + } + + public String getQuotedName(String quote) { + StringBuilder builder = new StringBuilder(); + builder.append(quote) + .append(database) + .append(quote) + .append("."); + if (schema != null) { + builder.append(quote) + .append(schema) + .append(quote) + .append("."); + } + builder.append(quote) + .append(table) + .append(quote); + return builder.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TableId tableId = (TableId) o; + return Objects.equals(database, tableId.database) && Objects.equals(schema, tableId.schema) && Objects.equals(table, tableId.table); + } + + @Override + public int hashCode() { + return Objects.hash(database, schema, table); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(database) + .append("."); + + if (StringUtils.isNotEmpty(schema)) { + builder.append(schema) + .append("."); + } + + builder.append(table); + return builder.toString(); + } +} diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/ReaderOptions.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/ReaderOptions.java index f6d6d6306..3b2358262 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/ReaderOptions.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/ReaderOptions.java @@ -107,5 +107,9 @@ interface BaseReaderOptions { ConfigOption CONTENT_TYPE = key(READER_PREFIX + "content_type") .noDefaultValue(String.class); + + ConfigOption MULTIPLE_READER_ENABLED = + key(READER_PREFIX + "multiple_reader_enabled") + .defaultValue(false); } } diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/WriterOptions.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/WriterOptions.java index fe16b550c..f76c54c2b 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/WriterOptions.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/option/WriterOptions.java @@ -108,5 +108,13 @@ interface BaseWriterOptions { ConfigOption WRITE_MODE = key(WRITER_PREFIX + "write_mode") .defaultValue("overwrite"); + + ConfigOption MULTIPLE_TABLE_ENABLED = + key(WRITER_PREFIX + "multiple_table_enabled") + .defaultValue(false); + + ConfigOption TABLE_PATTERN = + key(WRITER_PREFIX + "table_pattern") + .defaultValue(".*"); } } diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/row/MultipleTableRow.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/row/MultipleTableRow.java new file mode 100644 index 000000000..709b7899b --- /dev/null +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/row/MultipleTableRow.java @@ -0,0 +1,119 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.common.row; + +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfos; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.stream.Collectors; + +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Builder +public class MultipleTableRow implements Serializable { + + public static final RowTypeInfo MULTIPLE_TABLE_ROW_TYPE_INFO = + new RowTypeInfo(Arrays.stream(MultipleTableField.values()) + .map(MultipleTableField::getName) + .collect(Collectors.toList()) + .toArray(new String[] {}), + + Arrays.stream(MultipleTableField.values()) + .map(MultipleTableField::getTypeInfo) + .collect(Collectors.toList()) + .toArray(new TypeInfo[] {})); + + private String tableId; + + private String key; + + private String value; + + private String offset; + + private String partition; + + public static MultipleTableRow of(String tableId, + String key, + String value, + String offset, + String partition) { + return MultipleTableRow + .builder() + .tableId(tableId) + .key(key) + .value(value) + .offset(offset) + .partition(partition) + .build(); + } + + public static MultipleTableRow of(Row row) { + return MultipleTableRow + .builder() + .tableId(row.getString(MultipleTableField.TABLE_ID_FIELD.index)) + .key(row.getString(MultipleTableField.KEY_FIELD.index)) + .value(row.getString(MultipleTableField.VALUE_FIELD.index)) + .offset(row.getString(MultipleTableField.OFFSET_FIELD.index)) + .partition(row.getString(MultipleTableField.PARTITION_FIELD.index)) + .build(); + } + + public Row asRow() { + Row row = new Row(MultipleTableField.values().length); + row.setField(MultipleTableField.TABLE_ID_FIELD.index, tableId); + row.setField(MultipleTableField.KEY_FIELD.index, key); + row.setField(MultipleTableField.VALUE_FIELD.index, value); + row.setField(MultipleTableField.OFFSET_FIELD.index, offset); + row.setField(MultipleTableField.PARTITION_FIELD.index, partition); + return row; + } + + public RowTypeInfo getRowTypeInfo() { + return MULTIPLE_TABLE_ROW_TYPE_INFO; + } + + @Getter + private enum MultipleTableField { + TABLE_ID_FIELD("table-id", TypeInfos.STRING_TYPE_INFO, 0), + KEY_FIELD("key", TypeInfos.STRING_TYPE_INFO, 1), + VALUE_FIELD("value", TypeInfos.STRING_TYPE_INFO, 2), + OFFSET_FIELD("offset", TypeInfos.STRING_TYPE_INFO, 3), + PARTITION_FIELD("partition", TypeInfos.STRING_TYPE_INFO, 4); + + private final String name; + + private final TypeInfo typeInfo; + + private final int index; + + MultipleTableField(String name, TypeInfo typeInfo, int index) { + this.name = name; + this.typeInfo = typeInfo; + this.index = index; + } + } +} diff --git a/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/TableCatalogManagerTest.java b/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/CatalogManagerTest.java similarity index 84% rename from bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/TableCatalogManagerTest.java rename to bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/CatalogManagerTest.java index 49ce27901..3d2454576 100644 --- a/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/TableCatalogManagerTest.java +++ b/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/CatalogManagerTest.java @@ -16,9 +16,9 @@ package com.bytedance.bitsail.common.catalog; -import com.bytedance.bitsail.common.catalog.fake.FakeTableCatalog; +import com.bytedance.bitsail.common.catalog.fake.FakeCatalog; import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; -import com.bytedance.bitsail.common.catalog.table.CatalogTableDefinition; +import com.bytedance.bitsail.common.catalog.table.TableId; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.model.ColumnInfo; import com.bytedance.bitsail.common.option.ReaderOptions; @@ -32,7 +32,7 @@ import java.util.List; -public class TableCatalogManagerTest { +public class CatalogManagerTest { private BitSailConfiguration commonConfiguration; private BitSailConfiguration readerConfiguration; @@ -68,16 +68,16 @@ public void testTableCatalogAlignmentIntersect() throws Exception { commonConfiguration.set(TableCatalogOptions.COLUMN_ALIGN_STRATEGY, TableCatalogStrategy.INTERSECT.name()); commonConfiguration.set(TableCatalogOptions.SYNC_DDL, true); - FakeTableCatalog readerFakeTableCatalog = new FakeTableCatalog(readerColumns, CatalogTableDefinition.builder() + FakeCatalog readerFakeTableCatalog = new FakeCatalog(readerColumns, TableId.builder() .database("a") .table("b").build()); - FakeTableCatalog writerFakeTableCatalog = new FakeTableCatalog(writerColumns, CatalogTableDefinition.builder() + FakeCatalog writerFakeTableCatalog = new FakeCatalog(writerColumns, TableId.builder() .database("a") .table("c").build()); TableCatalogManager tableCatalogManager = TableCatalogManager.builder() - .readerTableCatalog(readerFakeTableCatalog) - .writerTableCatalog(writerFakeTableCatalog) + .readerCatalog(readerFakeTableCatalog) + .writerCatalog(writerFakeTableCatalog) .writerTypeInfoConverter(new BitSailTypeInfoConverter()) .readerTypeInfoConverter(new BitSailTypeInfoConverter()) .commonConfiguration(commonConfiguration) @@ -117,16 +117,16 @@ public void testTableCatalogAlignmentSourceOnly() throws Exception { TableCatalogStrategy.SOURCE_ONLY.name()); commonConfiguration.set(TableCatalogOptions.SYNC_DDL, true); - FakeTableCatalog readerFakeTableCatalog = new FakeTableCatalog(readerColumns, CatalogTableDefinition.builder() + FakeCatalog readerFakeTableCatalog = new FakeCatalog(readerColumns, TableId.builder() .database("a") .table("b").build()); - FakeTableCatalog writerFakeTableCatalog = new FakeTableCatalog(writerColumns, CatalogTableDefinition.builder() + FakeCatalog writerFakeTableCatalog = new FakeCatalog(writerColumns, TableId.builder() .database("a") .table("c").build()); TableCatalogManager tableCatalogManager = TableCatalogManager.builder() - .readerTableCatalog(readerFakeTableCatalog) - .writerTableCatalog(writerFakeTableCatalog) + .readerCatalog(readerFakeTableCatalog) + .writerCatalog(writerFakeTableCatalog) .writerTypeInfoConverter(new BitSailTypeInfoConverter()) .readerTypeInfoConverter(new BitSailTypeInfoConverter()) .commonConfiguration(commonConfiguration) @@ -167,16 +167,16 @@ public void testColumnAlignmentUpdate() throws Exception { commonConfiguration.set(TableCatalogOptions.COLUMN_ALIGN_STRATEGY, TableCatalogStrategy.SOURCE_ONLY.name()); commonConfiguration.set(TableCatalogOptions.SYNC_DDL, true); - FakeTableCatalog readerFakeTableCatalog = new FakeTableCatalog(readerColumns, CatalogTableDefinition.builder() + FakeCatalog readerFakeTableCatalog = new FakeCatalog(readerColumns, TableId.builder() .database("a") .table("b").build()); - FakeTableCatalog writerFakeTableCatalog = new FakeTableCatalog(writerColumns, CatalogTableDefinition.builder() + FakeCatalog writerFakeTableCatalog = new FakeCatalog(writerColumns, TableId.builder() .database("a") .table("c").build()); TableCatalogManager tableCatalogManager = TableCatalogManager.builder() - .readerTableCatalog(readerFakeTableCatalog) - .writerTableCatalog(writerFakeTableCatalog) + .readerCatalog(readerFakeTableCatalog) + .writerCatalog(writerFakeTableCatalog) .writerTypeInfoConverter(new BitSailTypeInfoConverter()) .readerTypeInfoConverter(new BitSailTypeInfoConverter()) .commonConfiguration(commonConfiguration) diff --git a/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/fake/FakeTableCatalog.java b/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/fake/FakeCatalog.java similarity index 81% rename from bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/fake/FakeTableCatalog.java rename to bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/fake/FakeCatalog.java index d3dc041fe..bdb904082 100644 --- a/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/fake/FakeTableCatalog.java +++ b/bitsail-common/src/test/java/com/bytedance/bitsail/common/catalog/fake/FakeCatalog.java @@ -16,11 +16,11 @@ package com.bytedance.bitsail.common.catalog.fake; +import com.bytedance.bitsail.common.catalog.table.Catalog; import com.bytedance.bitsail.common.catalog.table.CatalogTable; import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; -import com.bytedance.bitsail.common.catalog.table.CatalogTableDefinition; import com.bytedance.bitsail.common.catalog.table.CatalogTableSchema; -import com.bytedance.bitsail.common.catalog.table.TableCatalog; +import com.bytedance.bitsail.common.catalog.table.TableId; import com.bytedance.bitsail.common.catalog.table.TableOperation; import com.bytedance.bitsail.common.model.ColumnInfo; import com.bytedance.bitsail.common.type.BitSailTypeInfoConverter; @@ -33,11 +33,11 @@ import java.util.List; import java.util.stream.Collectors; -public class FakeTableCatalog implements TableCatalog { +public class FakeCatalog implements Catalog { private final List columnInfos; - private final CatalogTableDefinition tableDefinition; + private final TableId tableDefinition; private final TypeInfoConverter typeInfoConverter; @@ -50,8 +50,8 @@ public class FakeTableCatalog implements TableCatalog { @Getter private final List deletedTableColumns; - public FakeTableCatalog(List columnInfos, - CatalogTableDefinition tableDefinition) { + public FakeCatalog(List columnInfos, + TableId tableDefinition) { this.columnInfos = columnInfos; this.tableDefinition = tableDefinition; this.typeInfoConverter = new BitSailTypeInfoConverter(); @@ -78,29 +78,34 @@ public void close() { } @Override - public CatalogTableDefinition createCatalogTableDefinition() { + public TableId createCatalogTableDefinition() { return tableDefinition; } @Override - public boolean tableExists(CatalogTableDefinition catalogTableDefinition) { + public List listTables() { + return null; + } + + @Override + public boolean tableExists(TableId catalogTableDefinition) { return true; } @Override - public CatalogTable getCatalogTable(CatalogTableDefinition catalogTableDefinition) { + public CatalogTable getCatalogTable(TableId catalogTableDefinition) { CatalogTableSchema tableSchema = CatalogTableSchema.builder() .columns(catalogTableColumns) .primaryKeys(null) .build(); return CatalogTable.builder() - .catalogTableDefinition(catalogTableDefinition) + .tableId(catalogTableDefinition) .catalogTableSchema(tableSchema) .build(); } @Override - public void createTable(CatalogTableDefinition catalogTableDefinition, CatalogTable catalogTable) { + public void createTable(TableId catalogTableDefinition, CatalogTable catalogTable) { } @@ -129,6 +134,6 @@ public boolean compareTypeCompatible(TypeInfo original, TypeInfo compared) @Override public List convertTableColumn(TypeInfoConverter typeInfoConverter, List columnInfos) { - return TableCatalog.super.convertTableColumn(typeInfoConverter, columnInfos); + return Catalog.super.convertTableColumn(typeInfoConverter, columnInfos); } } diff --git a/bitsail-components/bitsail-component-formats-flink/pom.xml b/bitsail-components/bitsail-component-formats-flink/pom.xml index c73cb22a1..bb0f4e585 100644 --- a/bitsail-components/bitsail-component-formats-flink/pom.xml +++ b/bitsail-components/bitsail-component-formats-flink/pom.xml @@ -64,6 +64,12 @@ bitsail-core-flink-1.11-bridge ${revision} provided + + + com.bytedance.bitsail + bitsail-core-common + + diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumDeserializationSchema.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumDeserializationSchema.java index ab2174a03..02d74d72c 100644 --- a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumDeserializationSchema.java +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumDeserializationSchema.java @@ -16,14 +16,19 @@ package com.bytedance.bitsail.component.format.debezium; +import com.bytedance.bitsail.base.extension.SupportProducedType; import com.bytedance.bitsail.base.format.DeserializationSchema; +import com.bytedance.bitsail.common.row.Row; import org.apache.kafka.connect.source.SourceRecord; -public interface DebeziumDeserializationSchema extends DeserializationSchema { +public interface DebeziumDeserializationSchema extends DeserializationSchema, SupportProducedType { + + void open(); + @Override - public SourceRecord deserialize(byte[] message); + Row deserialize(SourceRecord sourceRecord); @Override - public boolean isEndOfStream(SourceRecord nextElement); + boolean isEndOfStream(Row nextElement); } diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumJsonDeserializationSchema.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumJsonDeserializationSchema.java new file mode 100644 index 000000000..73d256551 --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumJsonDeserializationSchema.java @@ -0,0 +1,85 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.component.format.debezium; + +import com.bytedance.bitsail.base.extension.SupportProducedType; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfos; +import com.bytedance.bitsail.component.format.debezium.option.DebeziumWriterOptions; + +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonConverterConfig; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.storage.ConverterConfig; +import org.apache.kafka.connect.storage.ConverterType; + +import java.util.HashMap; + +public class DebeziumJsonDeserializationSchema implements DebeziumDeserializationSchema, SupportProducedType { + public static final String NAME = "debezium-json"; + + public static final RowTypeInfo DEBEZIUM_JSON_ROW_TYPE = + new RowTypeInfo( + new String[] {"topic", "key", "value", "timestamp"}, + new TypeInfo[] { + TypeInfos.STRING_TYPE_INFO, TypeInfos.STRING_TYPE_INFO, TypeInfos.STRING_TYPE_INFO, TypeInfos.LONG_TYPE_INFO + }); + + private BitSailConfiguration jobConf; + private transient JsonConverter jsonConverter; + + public DebeziumJsonDeserializationSchema(BitSailConfiguration jobConf) { + this.jobConf = jobConf; + } + + @Override + public RowTypeInfo getProducedType() { + return DEBEZIUM_JSON_ROW_TYPE; + } + + @Override + public void open() { + this.jsonConverter = new JsonConverter(); + boolean includeSchema = jobConf.get(DebeziumWriterOptions.DEBEZIUM_JSON_INCLUDE_SCHEMA); + final HashMap configs = new HashMap<>(); + configs.put(ConverterConfig.TYPE_CONFIG, ConverterType.VALUE.getName()); + configs.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, includeSchema); + jsonConverter.configure(configs); + } + + @SuppressWarnings("checkstyle:MagicNumber") + @Override + public Row deserialize(SourceRecord sourceRecord) { + byte[] key = jsonConverter.fromConnectData(sourceRecord.topic(), sourceRecord.keySchema(), sourceRecord.key()); + byte[] value = jsonConverter.fromConnectData(sourceRecord.topic(), sourceRecord.valueSchema(), sourceRecord.value()); + + Object[] values = new Object[DEBEZIUM_JSON_ROW_TYPE.getFieldNames().length]; + values[0] = sourceRecord.topic(); + values[1] = new String(key); + values[2] = new String(value); + values[3] = sourceRecord.timestamp(); + return new Row(values); + } + + @Override + public boolean isEndOfStream(Row nextElement) { + return false; + } +} diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowDeserializationSchema.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowDeserializationSchema.java new file mode 100644 index 000000000..bdc8440d1 --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowDeserializationSchema.java @@ -0,0 +1,56 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.component.format.debezium; + +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; + +import org.apache.kafka.connect.source.SourceRecord; + +public class DebeziumRowDeserializationSchema implements DebeziumDeserializationSchema { + + private final BitSailConfiguration jobConf; + + private final RowTypeInfo rowTypeInfo; + + public DebeziumRowDeserializationSchema(BitSailConfiguration jobConf, RowTypeInfo rowTypeInfo) { + this.jobConf = jobConf; + this.rowTypeInfo = rowTypeInfo; + } + + @Override + public void open() { + + } + + @Override + public Row deserialize(SourceRecord sourceRecord) { + //TODO + return null; + } + + @Override + public boolean isEndOfStream(Row nextElement) { + return false; + } + + @Override + public RowTypeInfo getProducedType() { + return rowTypeInfo; + } +} diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowFilterNamesDeserializationSchema.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowFilterNamesDeserializationSchema.java new file mode 100644 index 000000000..aa700631a --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowFilterNamesDeserializationSchema.java @@ -0,0 +1,327 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.component.format.debezium; + +import com.bytedance.bitsail.base.format.DeserializationSchema; +import com.bytedance.bitsail.common.BitSailException; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.exception.CommonErrorCode; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.row.RowKind; +import com.bytedance.bitsail.component.format.debezium.option.DebeziumReaderOptions; + +import io.debezium.data.Enum; +import io.debezium.data.EnumSet; +import io.debezium.data.Envelope; +import io.debezium.data.Json; +import io.debezium.data.SpecialValueDecimal; +import io.debezium.data.VariableScaleDecimal; +import io.debezium.time.Date; +import io.debezium.time.MicroTime; +import io.debezium.time.MicroTimestamp; +import io.debezium.time.NanoTime; +import io.debezium.time.NanoTimestamp; +import io.debezium.time.Time; +import io.debezium.time.Timestamp; +import io.debezium.time.Year; +import io.debezium.time.ZonedTime; +import io.debezium.time.ZonedTimestamp; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Field; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonConverterConfig; +import org.apache.kafka.connect.storage.ConverterConfig; +import org.apache.kafka.connect.storage.ConverterType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.ZoneId; +import java.util.HashMap; +import java.util.Objects; + +import static io.debezium.data.Envelope.FieldName.AFTER; +import static io.debezium.data.Envelope.FieldName.BEFORE; +import static io.debezium.data.Envelope.FieldName.OPERATION; +import static org.apache.kafka.connect.data.Values.convertToDate; +import static org.apache.kafka.connect.data.Values.convertToTime; +import static org.apache.kafka.connect.data.Values.convertToTimestamp; + +public class DebeziumRowFilterNamesDeserializationSchema implements DeserializationSchema { + private static final Logger LOG = LoggerFactory.getLogger(DebeziumRowFilterNamesDeserializationSchema.class); + + private final BitSailConfiguration jobConf; + private final JsonConverter jsonConverter; + + public DebeziumRowFilterNamesDeserializationSchema(BitSailConfiguration jobConf) { + this.jobConf = jobConf; + this.jsonConverter = new JsonConverter(); + boolean includeSchema = jobConf.get(DebeziumReaderOptions.DEBEZIUM_JSON_INCLUDE_SCHEMA); + final HashMap configs = new HashMap<>(); + configs.put(ConverterConfig.TYPE_CONFIG, ConverterType.VALUE.getName()); + configs.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, includeSchema); + jsonConverter.configure(configs); + } + + @Override + public Row deserialize(String message) { + throw new UnsupportedOperationException("Please invoke DeserializationSchema#deserialize(byte[], fieldNames) instead."); + } + + public Row deserialize(String message, String[] fieldNames) { + SchemaAndValue schemaAndValue; + try { + schemaAndValue = jsonConverter.toConnectData(null, message.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + LOG.error("Can't parse content from format [debezium], content: {}.", message, e); + throw BitSailException.asBitSailException(CommonErrorCode.CONVERT_NOT_SUPPORT, + String.format("Can't parse debezium json: %s.", message), e); + } + Struct value = (Struct) schemaAndValue.value(); + Envelope.Operation operation = Envelope.Operation.forCode(value.getString(OPERATION)); + + if (operation == Envelope.Operation.CREATE || operation == Envelope.Operation.READ) { + Struct after = value.getStruct(AFTER); + return convert(after, after.schema(), fieldNames, RowKind.INSERT); + } + + if (operation == Envelope.Operation.DELETE) { + Struct before = value.getStruct(BEFORE); + return convert(before, before.schema(), fieldNames, RowKind.DELETE); + } + + if (operation == Envelope.Operation.UPDATE) { + Struct after = value.getStruct(AFTER); + return convert(after, after.schema(), fieldNames, RowKind.UPDATE_AFTER); + } + + throw BitSailException.asBitSailException(CommonErrorCode.CONVERT_NOT_SUPPORT, + String.format("Not support operation: %s right now.", operation)); + } + + public Row convert(Struct struct, Schema schema, String[] fieldNames, RowKind rowKind) { + Row row = new Row(fieldNames.length); + row.setKind(rowKind); + for (int index = 0; index < fieldNames.length; index++) { + Field field = schema.field(fieldNames[index]); + if (Objects.isNull(field)) { + row.setField(index, null); + } else { + Object withoutDefault = struct.getWithoutDefault(fieldNames[index]); + try { + withoutDefault = Objects.isNull(withoutDefault) ? null + : convert(field.schema(), withoutDefault); + row.setField(index, withoutDefault); + } catch (BitSailException e) { + LOG.error("Failed to parse field {} from value {}.", field.name(), withoutDefault); + throw e; + } + } + } + return row; + } + + private Object convert(Schema fieldSchema, Object withoutDefault) { + if (isPrimitiveType(fieldSchema)) { + return convertPrimitiveType(fieldSchema, withoutDefault); + } else { + //todo support local timestamp zone. + return convertOtherType(fieldSchema, withoutDefault, null); + } + } + + private static boolean isPrimitiveType(Schema fieldSchema) { + return fieldSchema.name() == null; + } + + private Object convertPrimitiveType(Schema fieldSchema, Object fieldValue) { + switch (fieldSchema.type()) { + case BOOLEAN: + return convertToBoolean(fieldValue); + case INT8: + case INT16: + case INT32: + return convertToInteger(fieldValue); + case INT64: + return convertToLong(fieldValue); + case FLOAT32: + return convertToFloat(fieldValue); + case FLOAT64: + return convertToDouble(fieldValue); + case STRING: + return convertToString(fieldValue); + case BYTES: + return convertToBinary(fieldValue); + default: + throw new UnsupportedOperationException("Not support type: " + fieldSchema.type()); + } + } + + private Object convertOtherType(Schema fieldSchema, Object fieldValue, ZoneId serverTimeZone) { + switch (fieldSchema.name()) { + case Enum.LOGICAL_NAME: + case Json.LOGICAL_NAME: + case EnumSet.LOGICAL_NAME: + return convertToString(fieldValue); + case Time.SCHEMA_NAME: + case MicroTime.SCHEMA_NAME: + case NanoTime.SCHEMA_NAME: + return convertToTime(fieldSchema, fieldValue); + case Timestamp.SCHEMA_NAME: + case MicroTimestamp.SCHEMA_NAME: + case NanoTimestamp.SCHEMA_NAME: + return convertToTimestamp(fieldSchema, fieldValue); + case Decimal.LOGICAL_NAME: + return convertToDecimal(fieldSchema, fieldValue); + case Date.SCHEMA_NAME: + return convertToDate(fieldSchema, fieldValue); + case Year.SCHEMA_NAME: + return convertToInteger(fieldValue); + case ZonedTime.SCHEMA_NAME: + case ZonedTimestamp.SCHEMA_NAME: + return convertToZoneTimeStamp(fieldSchema, fieldValue); + default: + throw BitSailException.asBitSailException(CommonErrorCode.CONVERT_NOT_SUPPORT, + String.format("Field name %s not support schema %s.", + fieldSchema.name(), + fieldSchema.schema() + ) + ); + } + } + + private byte[] convertToBinary(Object fieldValue) { + if (fieldValue instanceof byte[]) { + return (byte[]) fieldValue; + } else if (fieldValue instanceof ByteBuffer) { + ByteBuffer byteBuffer = (ByteBuffer) fieldValue; + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + return bytes; + } else { + throw new UnsupportedOperationException( + "Unsupported Binary value type: " + fieldValue.getClass().getSimpleName()); + } + } + + private String convertToString(Object fieldValue) { + return fieldValue.toString(); + } + + private Double convertToDouble(Object fieldValue) { + if (fieldValue instanceof Float) { + return ((Float) fieldValue).doubleValue(); + } else if (fieldValue instanceof Double) { + return (Double) fieldValue; + } else { + return Double.parseDouble(fieldValue.toString()); + } + } + + private Float convertToFloat(Object fieldValue) { + if (fieldValue instanceof Float) { + return (Float) fieldValue; + } else if (fieldValue instanceof Double) { + return ((Double) fieldValue).floatValue(); + } else { + return Float.parseFloat(fieldValue.toString()); + } + } + + private Long convertToLong(Object fieldValue) { + if (fieldValue instanceof Integer) { + return ((Integer) fieldValue).longValue(); + } else if (fieldValue instanceof Long) { + return (Long) fieldValue; + } else { + return Long.parseLong(fieldValue.toString()); + } + } + + private Boolean convertToBoolean(Object fieldValue) { + if (fieldValue instanceof Integer) { + return (Integer) fieldValue != 0; + } else if (fieldValue instanceof Long) { + return (Long) fieldValue != 0; + } else { + String str = fieldValue.toString(); + if (NumberUtils.isNumber(str)) { + return NumberUtils.createNumber(str).intValue() != 0; + } + return Boolean.parseBoolean(fieldValue.toString()); + } + } + + private Integer convertToInteger(Object fieldValue) { + if (fieldValue instanceof Integer) { + return (Integer) (fieldValue); + } else if (fieldValue instanceof Long) { + return ((Long) fieldValue).intValue(); + } else { + return Integer.parseInt(fieldValue.toString()); + } + } + + public Object convertToDecimal(Schema schema, Object fieldValue) { + BigDecimal bigDecimal; + if (fieldValue instanceof byte[]) { + // for decimal.handling.mode=precise + bigDecimal = Decimal.toLogical(schema, (byte[]) fieldValue); + } else if (fieldValue instanceof String) { + // for decimal.handling.mode=string + bigDecimal = new BigDecimal((String) fieldValue); + } else if (fieldValue instanceof Double) { + // for decimal.handling.mode=double + bigDecimal = BigDecimal.valueOf((Double) fieldValue); + } else { + if (VariableScaleDecimal.LOGICAL_NAME.equals(schema.name())) { + SpecialValueDecimal decimal = + VariableScaleDecimal.toLogical((Struct) fieldValue); + bigDecimal = decimal.getDecimalValue().orElse(BigDecimal.ZERO); + } else { + // fallback to string + bigDecimal = new BigDecimal(fieldValue.toString()); + } + } + return bigDecimal; + } + + private Object convertToZoneTimeStamp(Schema fieldSchema, Object fieldValue) { + if (fieldValue instanceof String) { + String str = (String) fieldValue; + Instant instant = Instant.parse(str); + //TODO zone timestamp + } + throw BitSailException.asBitSailException(CommonErrorCode.RUNTIME_ERROR, + String.format("Can't parse field [%s] from value %s to zone timestamp.", + fieldSchema.name(), + fieldValue + )); + } + + @Override + public boolean isEndOfStream(Row nextElement) { + return false; + } +} diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/MultipleDebeziumDeserializationSchema.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/MultipleDebeziumDeserializationSchema.java new file mode 100644 index 000000000..15f4944ef --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/MultipleDebeziumDeserializationSchema.java @@ -0,0 +1,82 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.component.format.debezium; + +import com.bytedance.bitsail.base.extension.SupportProducedType; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.row.MultipleTableRow; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; +import com.bytedance.bitsail.component.format.debezium.option.DebeziumWriterOptions; + +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonConverterConfig; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.storage.ConverterConfig; +import org.apache.kafka.connect.storage.ConverterType; + +import java.util.HashMap; + +import static io.debezium.data.Envelope.FieldName.SOURCE; + +public class MultipleDebeziumDeserializationSchema implements DebeziumDeserializationSchema, SupportProducedType { + + private transient JsonConverter jsonConverter; + private BitSailConfiguration jobConf; + + public MultipleDebeziumDeserializationSchema(BitSailConfiguration jobConf) { + this.jobConf = jobConf; + } + + public void open() { + this.jsonConverter = new JsonConverter(); + boolean includeSchema = jobConf.get(DebeziumWriterOptions.DEBEZIUM_JSON_INCLUDE_SCHEMA); + final HashMap configs = new HashMap<>(); + configs.put(ConverterConfig.TYPE_CONFIG, ConverterType.VALUE.getName()); + configs.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, includeSchema); + jsonConverter.configure(configs); + } + + @Override + public RowTypeInfo getProducedType() { + return MultipleTableRow.MULTIPLE_TABLE_ROW_TYPE_INFO; + } + + @Override + public Row deserialize(SourceRecord sourceRecord) { + Struct valueStruct = (Struct) sourceRecord.value(); + TableId tableId = TableId + .of(valueStruct.getStruct(SOURCE).getString("db"), valueStruct.getStruct(SOURCE).getString("table")); + byte[] connectData = jsonConverter.fromConnectData(sourceRecord.topic(), sourceRecord.valueSchema(), sourceRecord.value()); + MultipleTableRow multipleTableRow = MultipleTableRow + .of(tableId.toString(), + sourceRecord.topic(), + new String(connectData), + StringUtils.EMPTY, + StringUtils.EMPTY); + + return multipleTableRow.asRow(); + } + + @Override + public boolean isEndOfStream(Row nextElement) { + return false; + } +} diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumReaderOptions.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumReaderOptions.java index 1fb616793..87b282cdb 100644 --- a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumReaderOptions.java +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumReaderOptions.java @@ -16,7 +16,15 @@ package com.bytedance.bitsail.component.format.debezium.option; +import com.bytedance.bitsail.common.option.ConfigOption; import com.bytedance.bitsail.common.option.ReaderOptions; +import static com.bytedance.bitsail.common.option.ConfigOptions.key; +import static com.bytedance.bitsail.common.option.ReaderOptions.READER_PREFIX; + public interface DebeziumReaderOptions extends ReaderOptions.BaseReaderOptions { + + ConfigOption DEBEZIUM_JSON_INCLUDE_SCHEMA = + key(READER_PREFIX + "debezium_json_include_schema") + .defaultValue(true); } diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumWriterOptions.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumWriterOptions.java index 0b885e3f3..05321b1cb 100644 --- a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumWriterOptions.java +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/main/java/com/bytedance/bitsail/component/format/debezium/option/DebeziumWriterOptions.java @@ -24,6 +24,6 @@ public interface DebeziumWriterOptions extends WriterOptions.BaseWriterOptions { ConfigOption DEBEZIUM_JSON_INCLUDE_SCHEMA = - key(WRITER_PREFIX + "DEBEZIUM_JSON_INCLUDE_SCHEMA") - .defaultValue(false); + key(WRITER_PREFIX + "debezium_json_include_schema") + .defaultValue(true); } diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowFilterNamesDeserializationSchemaTest.java b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowFilterNamesDeserializationSchemaTest.java new file mode 100644 index 000000000..f9eb29ec7 --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/java/com/bytedance/bitsail/component/format/debezium/DebeziumRowFilterNamesDeserializationSchemaTest.java @@ -0,0 +1,100 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.component.format.debezium; + +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.row.RowKind; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.Date; + +public class DebeziumRowFilterNamesDeserializationSchemaTest { + + private DebeziumRowFilterNamesDeserializationSchema deserializationSchema; + + @Before + public void before() { + deserializationSchema = new DebeziumRowFilterNamesDeserializationSchema(BitSailConfiguration.newDefault()); + } + + @Test + public void test() throws URISyntaxException, IOException { + byte[] bytes = Files.readAllBytes(Paths.get(DebeziumRowFilterNamesDeserializationSchemaTest + .class.getClassLoader().getResource("file/debezium.json") + .toURI().getPath())); + + Row deserialize = deserializationSchema.deserialize(new String(bytes), + new String[] {"double_type"}); + + Assert.assertNotNull(deserialize); + Assert.assertTrue(deserialize.getField(0) instanceof BigDecimal); + } + + @Test + public void testInsert() throws URISyntaxException, IOException { + byte[] bytes = Files.readAllBytes(Paths.get(DebeziumRowFilterNamesDeserializationSchemaTest + .class.getClassLoader().getResource("file/debezium_insert.json") + .toURI().getPath())); + + Row deserialize = deserializationSchema.deserialize(new String(bytes), + new String[] {"double_type"}); + + Assert.assertNotNull(deserialize); + Assert.assertEquals(deserialize.getKind(), RowKind.INSERT); + } + + @Test + public void testUpsert() throws URISyntaxException, IOException { + byte[] bytes = Files.readAllBytes(Paths.get(DebeziumRowFilterNamesDeserializationSchemaTest + .class.getClassLoader().getResource("file/debezium_upsert.json") + .toURI().getPath())); + + Row deserialize = deserializationSchema.deserialize(new String(bytes), + new String[] {"order_date"}); + + Assert.assertNotNull(deserialize); + Assert.assertEquals(LocalDate.ofEpochDay(16816).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli(), + ((Date) deserialize.getField(0)).toInstant().toEpochMilli()); + Assert.assertEquals(deserialize.getKind(), RowKind.UPDATE_AFTER); + } + + @Test + public void testDelete() throws URISyntaxException, IOException { + byte[] bytes = Files.readAllBytes(Paths.get(DebeziumRowFilterNamesDeserializationSchemaTest + .class.getClassLoader().getResource("file/postgres/debezium_pg_delete.json") + .toURI().getPath())); + + Row deserialize = deserializationSchema.deserialize(new String(bytes), + new String[] {"id"}); + + Assert.assertNotNull(deserialize); + Assert.assertEquals(deserialize.getField(0), 10049L); + Assert.assertEquals(deserialize.getKind(), RowKind.DELETE); + } + +} \ No newline at end of file diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium.json b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium.json new file mode 100644 index 000000000..5699f29b4 --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium.json @@ -0,0 +1,257 @@ +{ + "schema":{ + "type":"struct", + "fields":[ + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"int_type" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"4", + "connect.decimal.precision":"20" + }, + "field":"double_type" + }, + { + "type":"string", + "optional":true, + "field":"date_type" + }, + { + "type":"string", + "optional":true, + "default":"", + "field":"varchar_type" + }, + { + "type":"int32", + "optional":false, + "name":"io.debezium.time.Date", + "version":1, + "field":"datetime" + } + ], + "optional":true, + "name":"localhost.test.jdbc_source_test.Value", + "field":"before" + }, + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"int_type" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"4", + "connect.decimal.precision":"20" + }, + "field":"double_type" + }, + { + "type":"string", + "optional":true, + "field":"date_type" + }, + { + "type":"string", + "optional":true, + "default":"", + "field":"varchar_type" + }, + { + "type":"int32", + "optional":false, + "name":"io.debezium.time.Date", + "version":1, + "field":"datetime" + } + ], + "optional":true, + "name":"localhost.test.jdbc_source_test.Value", + "field":"after" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"version" + }, + { + "type":"string", + "optional":false, + "field":"connector" + }, + { + "type":"string", + "optional":false, + "field":"name" + }, + { + "type":"int64", + "optional":false, + "field":"ts_ms" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Enum", + "version":1, + "parameters":{ + "allowed":"true,last,false" + }, + "default":"false", + "field":"snapshot" + }, + { + "type":"string", + "optional":false, + "field":"db" + }, + { + "type":"string", + "optional":true, + "field":"sequence" + }, + { + "type":"string", + "optional":true, + "field":"table" + }, + { + "type":"int64", + "optional":false, + "field":"server_id" + }, + { + "type":"string", + "optional":true, + "field":"gtid" + }, + { + "type":"string", + "optional":false, + "field":"file" + }, + { + "type":"int64", + "optional":false, + "field":"pos" + }, + { + "type":"int32", + "optional":false, + "field":"row" + }, + { + "type":"int64", + "optional":true, + "field":"thread" + }, + { + "type":"string", + "optional":true, + "field":"query" + } + ], + "optional":false, + "name":"io.debezium.connector.mysql.Source", + "field":"source" + }, + { + "type":"string", + "optional":false, + "field":"op" + }, + { + "type":"int64", + "optional":true, + "field":"ts_ms" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":false, + "field":"total_order" + }, + { + "type":"int64", + "optional":false, + "field":"data_collection_order" + } + ], + "optional":true, + "field":"transaction" + } + ], + "optional":false, + "name":"localhost.test.jdbc_source_test.Envelope" + }, + "payload":{ + "before":null, + "after":{ + "id":1, + "int_type":1001, + "double_type":"AbH8", + "date_type":"2022-10-01", + "varchar_type":"varchar_01", + "datetime":19297 + }, + "source":{ + "version":"1.6.4.Final", + "connector":"mysql", + "name":"localhost", + "ts_ms":1682237044000, + "snapshot":"false", + "db":"test", + "sequence":null, + "table":"jdbc_source_test", + "server_id":1, + "gtid":null, + "file":"binlog.000002", + "pos":986, + "row":0, + "thread":null, + "query":null + }, + "op":"c", + "ts_ms":1682237048134, + "transaction":null + } +} \ No newline at end of file diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_delete.json b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_delete.json new file mode 100644 index 000000000..2d7c78abf --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_delete.json @@ -0,0 +1,54 @@ +{ + "before":{ + "id":10047, + "bigint_info":null, + "bigserial_info":0, + "bit_info":null, + "bit_varying_info":null, + "boolean_info":null, + "bytea_info":null, + "character_varying_info":null, + "character_info":null, + "cidr_info":null, + "date":null, + "double_precision_info":null, + "inet_info":null, + "integer_info":null, + "interval_info":null, + "macaddr_info":null, + "money_info":null, + "numeric_info":null, + "decimal_info":null, + "real_info":null, + "point_info":null, + "smallint_info":null, + "smallserial_info":0, + "serial_info":0, + "text_info":null, + "time_info":null, + "timestamp_info":null, + "uuid_info":null, + "xml_info":null, + "json_info":null, + "int_list":null, + "txt_list":null + }, + "after":null, + "source":{ + "version":"1.6.4.Final", + "connector":"postgresql", + "name":"abc", + "ts_ms":1682325644419, + "snapshot":"false", + "db":"test_cdc", + "sequence":"[\"434291792\",\"434291792\"]", + "schema":"public", + "table":"dts_source_all111", + "txId":201659, + "lsn":434293384, + "xmin":null + }, + "op":"d", + "ts_ms":1682325644348, + "transaction":null +} \ No newline at end of file diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_insert.json b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_insert.json new file mode 100644 index 000000000..5699f29b4 --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_insert.json @@ -0,0 +1,257 @@ +{ + "schema":{ + "type":"struct", + "fields":[ + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"int_type" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"4", + "connect.decimal.precision":"20" + }, + "field":"double_type" + }, + { + "type":"string", + "optional":true, + "field":"date_type" + }, + { + "type":"string", + "optional":true, + "default":"", + "field":"varchar_type" + }, + { + "type":"int32", + "optional":false, + "name":"io.debezium.time.Date", + "version":1, + "field":"datetime" + } + ], + "optional":true, + "name":"localhost.test.jdbc_source_test.Value", + "field":"before" + }, + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"int_type" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"4", + "connect.decimal.precision":"20" + }, + "field":"double_type" + }, + { + "type":"string", + "optional":true, + "field":"date_type" + }, + { + "type":"string", + "optional":true, + "default":"", + "field":"varchar_type" + }, + { + "type":"int32", + "optional":false, + "name":"io.debezium.time.Date", + "version":1, + "field":"datetime" + } + ], + "optional":true, + "name":"localhost.test.jdbc_source_test.Value", + "field":"after" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"version" + }, + { + "type":"string", + "optional":false, + "field":"connector" + }, + { + "type":"string", + "optional":false, + "field":"name" + }, + { + "type":"int64", + "optional":false, + "field":"ts_ms" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Enum", + "version":1, + "parameters":{ + "allowed":"true,last,false" + }, + "default":"false", + "field":"snapshot" + }, + { + "type":"string", + "optional":false, + "field":"db" + }, + { + "type":"string", + "optional":true, + "field":"sequence" + }, + { + "type":"string", + "optional":true, + "field":"table" + }, + { + "type":"int64", + "optional":false, + "field":"server_id" + }, + { + "type":"string", + "optional":true, + "field":"gtid" + }, + { + "type":"string", + "optional":false, + "field":"file" + }, + { + "type":"int64", + "optional":false, + "field":"pos" + }, + { + "type":"int32", + "optional":false, + "field":"row" + }, + { + "type":"int64", + "optional":true, + "field":"thread" + }, + { + "type":"string", + "optional":true, + "field":"query" + } + ], + "optional":false, + "name":"io.debezium.connector.mysql.Source", + "field":"source" + }, + { + "type":"string", + "optional":false, + "field":"op" + }, + { + "type":"int64", + "optional":true, + "field":"ts_ms" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":false, + "field":"total_order" + }, + { + "type":"int64", + "optional":false, + "field":"data_collection_order" + } + ], + "optional":true, + "field":"transaction" + } + ], + "optional":false, + "name":"localhost.test.jdbc_source_test.Envelope" + }, + "payload":{ + "before":null, + "after":{ + "id":1, + "int_type":1001, + "double_type":"AbH8", + "date_type":"2022-10-01", + "varchar_type":"varchar_01", + "datetime":19297 + }, + "source":{ + "version":"1.6.4.Final", + "connector":"mysql", + "name":"localhost", + "ts_ms":1682237044000, + "snapshot":"false", + "db":"test", + "sequence":null, + "table":"jdbc_source_test", + "server_id":1, + "gtid":null, + "file":"binlog.000002", + "pos":986, + "row":0, + "thread":null, + "query":null + }, + "op":"c", + "ts_ms":1682237048134, + "transaction":null + } +} \ No newline at end of file diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_upsert.json b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_upsert.json new file mode 100644 index 000000000..6b5a456ab --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/debezium_upsert.json @@ -0,0 +1,207 @@ +{ + "schema":{ + "name":"dbserver1.inventory.orders.Envelope", + "optional":false, + "type":"struct", + "fields":[ + { + "field":"before", + "name":"dbserver1.inventory.orders.Value", + "optional":true, + "type":"struct", + "fields":[ + { + "field":"order_number", + "optional":false, + "type":"int32" + }, + { + "field":"order_date", + "name":"io.debezium.time.Date", + "optional":false, + "type":"int32", + "version":1 + }, + { + "field":"purchaser", + "optional":false, + "type":"int32" + }, + { + "field":"quantity", + "optional":false, + "type":"int32" + }, + { + "field":"product_id", + "optional":false, + "type":"int32" + } + ] + }, + { + "field":"after", + "name":"dbserver1.inventory.orders.Value", + "optional":true, + "type":"struct", + "fields":[ + { + "field":"order_number", + "optional":false, + "type":"int32" + }, + { + "field":"order_date", + "name":"io.debezium.time.Date", + "optional":false, + "type":"int32", + "version":1 + }, + { + "field":"purchaser", + "optional":false, + "type":"int32" + }, + { + "field":"quantity", + "optional":false, + "type":"int32" + }, + { + "field":"product_id", + "optional":false, + "type":"int32" + } + ] + }, + { + "field":"source", + "name":"io.debezium.connector.mysql.Source", + "optional":false, + "type":"struct", + "fields":[ + { + "field":"version", + "optional":false, + "type":"string" + }, + { + "field":"connector", + "optional":false, + "type":"string" + }, + { + "field":"name", + "optional":false, + "type":"string" + }, + { + "field":"ts_ms", + "optional":false, + "type":"int64" + }, + { + "default":"false", + "field":"snapshot", + "name":"io.debezium.data.Enum", + "optional":true, + "type":"string", + "version":1, + "parameters":{ + "allowed":"true,last,false" + } + }, + { + "field":"db", + "optional":false, + "type":"string" + }, + { + "field":"table", + "optional":true, + "type":"string" + }, + { + "field":"server_id", + "optional":false, + "type":"int64" + }, + { + "field":"gtid", + "optional":true, + "type":"string" + }, + { + "field":"file", + "optional":false, + "type":"string" + }, + { + "field":"pos", + "optional":false, + "type":"int64" + }, + { + "field":"row", + "optional":false, + "type":"int32" + }, + { + "field":"thread", + "optional":true, + "type":"int64" + }, + { + "field":"query", + "optional":true, + "type":"string" + } + ] + }, + { + "field":"op", + "optional":false, + "type":"string" + }, + { + "field":"ts_ms", + "optional":true, + "type":"int64" + } + ] + }, + "payload":{ + "op":"u", + "before":{ + "order_date":16816, + "quantity":1, + "purchaser":1001, + "order_number":10001, + "product_id":102 + }, + "after":{ + "order_date":16816, + "quantity":6, + "purchaser":1001, + "order_number":10001, + "product_id":102 + }, + "source":{ + "query":null, + "thread":4, + "server_id":223344, + "version":"1.0.3.Final", + "file":"mysql-bin.000007", + "connector":"mysql", + "pos":354, + "name":"dbserver1", + "gtid":null, + "row":0, + "ts_ms":1591620600000, + "snapshot":"false", + "db":"inventory", + "table":"orders" + }, + "ts_ms":1591620602204 + } +} \ No newline at end of file diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/postgres/debezium_pg_delete.json b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/postgres/debezium_pg_delete.json new file mode 100644 index 000000000..a8d8d962b --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/postgres/debezium_pg_delete.json @@ -0,0 +1,635 @@ +{ + "schema":{ + "type":"struct", + "fields":[ + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":true, + "field":"bigint_info" + }, + { + "type":"int64", + "optional":false, + "default":0, + "field":"bigserial_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"3" + }, + "field":"bit_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"1000" + }, + "field":"bit_varying_info" + }, + { + "type":"boolean", + "optional":true, + "field":"boolean_info" + }, + { + "type":"bytes", + "optional":true, + "field":"bytea_info" + }, + { + "type":"string", + "optional":true, + "field":"character_varying_info" + }, + { + "type":"string", + "optional":true, + "field":"character_info" + }, + { + "type":"string", + "optional":true, + "field":"cidr_info" + }, + { + "type":"int32", + "optional":true, + "name":"io.debezium.time.Date", + "version":1, + "field":"date" + }, + { + "type":"double", + "optional":true, + "field":"double_precision_info" + }, + { + "type":"string", + "optional":true, + "field":"inet_info" + }, + { + "type":"int32", + "optional":true, + "field":"integer_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroDuration", + "version":1, + "field":"interval_info" + }, + { + "type":"string", + "optional":true, + "field":"macaddr_info" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"2" + }, + "field":"money_info" + }, + { + "type":"double", + "optional":true, + "field":"numeric_info" + }, + { + "type":"double", + "optional":true, + "field":"decimal_info" + }, + { + "type":"float", + "optional":true, + "field":"real_info" + }, + { + "type":"struct", + "fields":[ + { + "type":"double", + "optional":false, + "field":"x" + }, + { + "type":"double", + "optional":false, + "field":"y" + }, + { + "type":"bytes", + "optional":true, + "field":"wkb" + }, + { + "type":"int32", + "optional":true, + "field":"srid" + } + ], + "optional":true, + "name":"io.debezium.data.geometry.Point", + "version":1, + "doc":"Geometry (POINT)", + "field":"point_info" + }, + { + "type":"int16", + "optional":true, + "field":"smallint_info" + }, + { + "type":"int16", + "optional":false, + "default":0, + "field":"smallserial_info" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"serial_info" + }, + { + "type":"string", + "optional":true, + "field":"text_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTime", + "version":1, + "field":"time_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTimestamp", + "version":1, + "field":"timestamp_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Uuid", + "version":1, + "field":"uuid_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Xml", + "version":1, + "field":"xml_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Json", + "version":1, + "field":"json_info" + }, + { + "type":"array", + "items":{ + "type":"int32", + "optional":true + }, + "optional":true, + "field":"int_list" + }, + { + "type":"array", + "items":{ + "type":"string", + "optional":true + }, + "optional":true, + "field":"txt_list" + } + ], + "optional":true, + "name":"abc.public.dts_source_all111.Value", + "field":"before" + }, + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":true, + "field":"bigint_info" + }, + { + "type":"int64", + "optional":false, + "default":0, + "field":"bigserial_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"3" + }, + "field":"bit_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"1000" + }, + "field":"bit_varying_info" + }, + { + "type":"boolean", + "optional":true, + "field":"boolean_info" + }, + { + "type":"bytes", + "optional":true, + "field":"bytea_info" + }, + { + "type":"string", + "optional":true, + "field":"character_varying_info" + }, + { + "type":"string", + "optional":true, + "field":"character_info" + }, + { + "type":"string", + "optional":true, + "field":"cidr_info" + }, + { + "type":"int32", + "optional":true, + "name":"io.debezium.time.Date", + "version":1, + "field":"date" + }, + { + "type":"double", + "optional":true, + "field":"double_precision_info" + }, + { + "type":"string", + "optional":true, + "field":"inet_info" + }, + { + "type":"int32", + "optional":true, + "field":"integer_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroDuration", + "version":1, + "field":"interval_info" + }, + { + "type":"string", + "optional":true, + "field":"macaddr_info" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"2" + }, + "field":"money_info" + }, + { + "type":"double", + "optional":true, + "field":"numeric_info" + }, + { + "type":"double", + "optional":true, + "field":"decimal_info" + }, + { + "type":"float", + "optional":true, + "field":"real_info" + }, + { + "type":"struct", + "fields":[ + { + "type":"double", + "optional":false, + "field":"x" + }, + { + "type":"double", + "optional":false, + "field":"y" + }, + { + "type":"bytes", + "optional":true, + "field":"wkb" + }, + { + "type":"int32", + "optional":true, + "field":"srid" + } + ], + "optional":true, + "name":"io.debezium.data.geometry.Point", + "version":1, + "doc":"Geometry (POINT)", + "field":"point_info" + }, + { + "type":"int16", + "optional":true, + "field":"smallint_info" + }, + { + "type":"int16", + "optional":false, + "default":0, + "field":"smallserial_info" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"serial_info" + }, + { + "type":"string", + "optional":true, + "field":"text_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTime", + "version":1, + "field":"time_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTimestamp", + "version":1, + "field":"timestamp_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Uuid", + "version":1, + "field":"uuid_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Xml", + "version":1, + "field":"xml_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Json", + "version":1, + "field":"json_info" + }, + { + "type":"array", + "items":{ + "type":"int32", + "optional":true + }, + "optional":true, + "field":"int_list" + }, + { + "type":"array", + "items":{ + "type":"string", + "optional":true + }, + "optional":true, + "field":"txt_list" + } + ], + "optional":true, + "name":"abc.public.dts_source_all111.Value", + "field":"after" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"version" + }, + { + "type":"string", + "optional":false, + "field":"connector" + }, + { + "type":"string", + "optional":false, + "field":"name" + }, + { + "type":"int64", + "optional":false, + "field":"ts_ms" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Enum", + "version":1, + "parameters":{ + "allowed":"true,last,false" + }, + "default":"false", + "field":"snapshot" + }, + { + "type":"string", + "optional":false, + "field":"db" + }, + { + "type":"string", + "optional":true, + "field":"sequence" + }, + { + "type":"string", + "optional":false, + "field":"schema" + }, + { + "type":"string", + "optional":false, + "field":"table" + }, + { + "type":"int64", + "optional":true, + "field":"txId" + }, + { + "type":"int64", + "optional":true, + "field":"lsn" + }, + { + "type":"int64", + "optional":true, + "field":"xmin" + } + ], + "optional":false, + "name":"io.debezium.connector.postgresql.Source", + "field":"source" + }, + { + "type":"string", + "optional":false, + "field":"op" + }, + { + "type":"int64", + "optional":true, + "field":"ts_ms" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":false, + "field":"total_order" + }, + { + "type":"int64", + "optional":false, + "field":"data_collection_order" + } + ], + "optional":true, + "field":"transaction" + } + ], + "optional":false, + "name":"abc.public.dts_source_all111.Envelope" + }, + "payload":{ + "before":{ + "id":10049, + "bigint_info":null, + "bigserial_info":0, + "bit_info":null, + "bit_varying_info":null, + "boolean_info":null, + "bytea_info":null, + "character_varying_info":null, + "character_info":null, + "cidr_info":null, + "date":null, + "double_precision_info":null, + "inet_info":null, + "integer_info":null, + "interval_info":null, + "macaddr_info":null, + "money_info":null, + "numeric_info":null, + "decimal_info":null, + "real_info":null, + "point_info":null, + "smallint_info":null, + "smallserial_info":0, + "serial_info":0, + "text_info":null, + "time_info":null, + "timestamp_info":null, + "uuid_info":null, + "xml_info":null, + "json_info":null, + "int_list":null, + "txt_list":null + }, + "after":null, + "source":{ + "version":"1.6.4.Final", + "connector":"postgresql", + "name":"abc", + "ts_ms":1682326458976, + "snapshot":"false", + "db":"test_cdc", + "sequence":"[\"434325360\",\"434325360\"]", + "schema":"public", + "table":"dts_source_all111", + "txId":201667, + "lsn":434326896, + "xmin":null + }, + "op":"d", + "ts_ms":1682326459323, + "transaction":null + } +} \ No newline at end of file diff --git a/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/postgres/debezium_pg_upsert.json b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/postgres/debezium_pg_upsert.json new file mode 100644 index 000000000..379e15e8e --- /dev/null +++ b/bitsail-components/bitsail-component-formats/bitsail-component-format-debezium/src/test/resources/file/postgres/debezium_pg_upsert.json @@ -0,0 +1,680 @@ +{ + "schema":{ + "type":"struct", + "fields":[ + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":true, + "field":"bigint_info" + }, + { + "type":"int64", + "optional":false, + "default":0, + "field":"bigserial_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"3" + }, + "field":"bit_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"1000" + }, + "field":"bit_varying_info" + }, + { + "type":"boolean", + "optional":true, + "field":"boolean_info" + }, + { + "type":"bytes", + "optional":true, + "field":"bytea_info" + }, + { + "type":"string", + "optional":true, + "field":"character_varying_info" + }, + { + "type":"string", + "optional":true, + "field":"character_info" + }, + { + "type":"string", + "optional":true, + "field":"cidr_info" + }, + { + "type":"int32", + "optional":true, + "name":"io.debezium.time.Date", + "version":1, + "field":"date" + }, + { + "type":"double", + "optional":true, + "field":"double_precision_info" + }, + { + "type":"string", + "optional":true, + "field":"inet_info" + }, + { + "type":"int32", + "optional":true, + "field":"integer_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroDuration", + "version":1, + "field":"interval_info" + }, + { + "type":"string", + "optional":true, + "field":"macaddr_info" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"2" + }, + "field":"money_info" + }, + { + "type":"double", + "optional":true, + "field":"numeric_info" + }, + { + "type":"double", + "optional":true, + "field":"decimal_info" + }, + { + "type":"float", + "optional":true, + "field":"real_info" + }, + { + "type":"struct", + "fields":[ + { + "type":"double", + "optional":false, + "field":"x" + }, + { + "type":"double", + "optional":false, + "field":"y" + }, + { + "type":"bytes", + "optional":true, + "field":"wkb" + }, + { + "type":"int32", + "optional":true, + "field":"srid" + } + ], + "optional":true, + "name":"io.debezium.data.geometry.Point", + "version":1, + "doc":"Geometry (POINT)", + "field":"point_info" + }, + { + "type":"int16", + "optional":true, + "field":"smallint_info" + }, + { + "type":"int16", + "optional":false, + "default":0, + "field":"smallserial_info" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"serial_info" + }, + { + "type":"string", + "optional":true, + "field":"text_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTime", + "version":1, + "field":"time_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTimestamp", + "version":1, + "field":"timestamp_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Uuid", + "version":1, + "field":"uuid_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Xml", + "version":1, + "field":"xml_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Json", + "version":1, + "field":"json_info" + }, + { + "type":"array", + "items":{ + "type":"int32", + "optional":true + }, + "optional":true, + "field":"int_list" + }, + { + "type":"array", + "items":{ + "type":"string", + "optional":true + }, + "optional":true, + "field":"txt_list" + } + ], + "optional":true, + "name":"abc.public.dts_source_all111.Value", + "field":"before" + }, + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":true, + "field":"bigint_info" + }, + { + "type":"int64", + "optional":false, + "default":0, + "field":"bigserial_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"3" + }, + "field":"bit_info" + }, + { + "type":"bytes", + "optional":true, + "name":"io.debezium.data.Bits", + "version":1, + "parameters":{ + "length":"1000" + }, + "field":"bit_varying_info" + }, + { + "type":"boolean", + "optional":true, + "field":"boolean_info" + }, + { + "type":"bytes", + "optional":true, + "field":"bytea_info" + }, + { + "type":"string", + "optional":true, + "field":"character_varying_info" + }, + { + "type":"string", + "optional":true, + "field":"character_info" + }, + { + "type":"string", + "optional":true, + "field":"cidr_info" + }, + { + "type":"int32", + "optional":true, + "name":"io.debezium.time.Date", + "version":1, + "field":"date" + }, + { + "type":"double", + "optional":true, + "field":"double_precision_info" + }, + { + "type":"string", + "optional":true, + "field":"inet_info" + }, + { + "type":"int32", + "optional":true, + "field":"integer_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroDuration", + "version":1, + "field":"interval_info" + }, + { + "type":"string", + "optional":true, + "field":"macaddr_info" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"2" + }, + "field":"money_info" + }, + { + "type":"double", + "optional":true, + "field":"numeric_info" + }, + { + "type":"double", + "optional":true, + "field":"decimal_info" + }, + { + "type":"float", + "optional":true, + "field":"real_info" + }, + { + "type":"struct", + "fields":[ + { + "type":"double", + "optional":false, + "field":"x" + }, + { + "type":"double", + "optional":false, + "field":"y" + }, + { + "type":"bytes", + "optional":true, + "field":"wkb" + }, + { + "type":"int32", + "optional":true, + "field":"srid" + } + ], + "optional":true, + "name":"io.debezium.data.geometry.Point", + "version":1, + "doc":"Geometry (POINT)", + "field":"point_info" + }, + { + "type":"int16", + "optional":true, + "field":"smallint_info" + }, + { + "type":"int16", + "optional":false, + "default":0, + "field":"smallserial_info" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"serial_info" + }, + { + "type":"string", + "optional":true, + "field":"text_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTime", + "version":1, + "field":"time_info" + }, + { + "type":"int64", + "optional":true, + "name":"io.debezium.time.MicroTimestamp", + "version":1, + "field":"timestamp_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Uuid", + "version":1, + "field":"uuid_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Xml", + "version":1, + "field":"xml_info" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Json", + "version":1, + "field":"json_info" + }, + { + "type":"array", + "items":{ + "type":"int32", + "optional":true + }, + "optional":true, + "field":"int_list" + }, + { + "type":"array", + "items":{ + "type":"string", + "optional":true + }, + "optional":true, + "field":"txt_list" + } + ], + "optional":true, + "name":"abc.public.dts_source_all111.Value", + "field":"after" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"version" + }, + { + "type":"string", + "optional":false, + "field":"connector" + }, + { + "type":"string", + "optional":false, + "field":"name" + }, + { + "type":"int64", + "optional":false, + "field":"ts_ms" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Enum", + "version":1, + "parameters":{ + "allowed":"true,last,false" + }, + "default":"false", + "field":"snapshot" + }, + { + "type":"string", + "optional":false, + "field":"db" + }, + { + "type":"string", + "optional":true, + "field":"sequence" + }, + { + "type":"string", + "optional":false, + "field":"schema" + }, + { + "type":"string", + "optional":false, + "field":"table" + }, + { + "type":"int64", + "optional":true, + "field":"txId" + }, + { + "type":"int64", + "optional":true, + "field":"lsn" + }, + { + "type":"int64", + "optional":true, + "field":"xmin" + } + ], + "optional":false, + "name":"io.debezium.connector.postgresql.Source", + "field":"source" + }, + { + "type":"string", + "optional":false, + "field":"op" + }, + { + "type":"int64", + "optional":true, + "field":"ts_ms" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":false, + "field":"total_order" + }, + { + "type":"int64", + "optional":false, + "field":"data_collection_order" + } + ], + "optional":true, + "field":"transaction" + } + ], + "optional":false, + "name":"abc.public.dts_source_all111.Envelope" + }, + "payload":{ + "before":{ + "id":10048, + "bigint_info":null, + "bigserial_info":0, + "bit_info":null, + "bit_varying_info":null, + "boolean_info":null, + "bytea_info":null, + "character_varying_info":null, + "character_info":null, + "cidr_info":null, + "date":null, + "double_precision_info":null, + "inet_info":null, + "integer_info":null, + "interval_info":null, + "macaddr_info":null, + "money_info":null, + "numeric_info":null, + "decimal_info":null, + "real_info":null, + "point_info":null, + "smallint_info":null, + "smallserial_info":0, + "serial_info":0, + "text_info":null, + "time_info":null, + "timestamp_info":null, + "uuid_info":null, + "xml_info":null, + "json_info":null, + "int_list":null, + "txt_list":null + }, + "after":{ + "id":10048, + "bigint_info":4287444545095, + "bigserial_info":2, + "bit_info":"BQ==", + "bit_varying_info":"BQ==", + "boolean_info":false, + "bytea_info":"XHhERUFEQkVFRg==", + "character_varying_info":"hello world", + "character_info":"hello ", + "cidr_info":"192.168.100.128/32", + "date":19300, + "double_precision_info":12.123123123, + "inet_info":"192.168.100.128", + "integer_info":10000, + "interval_info":31536000000000, + "macaddr_info":"08:00:2b:01:02:03", + "money_info":"BrHASw==", + "numeric_info":1233.12312, + "decimal_info":123123.123123123, + "real_info":123123.125, + "point_info":{ + "x":123, + "y":123, + "wkb":"AQEAAAAAAAAAAMBeQAAAAAAAwF5A", + "srid":null + }, + "smallint_info":10000, + "smallserial_info":0, + "serial_info":0, + "text_info":"我们还是类别品牌重要更新.会员价格原因密码只要.很多最后用户详细结果增加是一.\\n解决还是全部准备产品.数据其实中国文化.\\n电脑那么历史只要功能国家其他.来源任何国际自己人员已经.分析有关我的一下信息应该.\\n学习标题表示网络认为.计划科技得到一样电脑他们搜索一起.对于国际点击搜索研究表示记者.\\n起来教育搜索.自己网站的人免费就是所有更新.", + "time_info":13903000000, + "timestamp_info":1667533903000000, + "uuid_info":"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", + "xml_info":"Manual...", + "json_info":"{\"a\": 1, \"b\": 2}", + "int_list":[ + 100, + 200, + 300 + ], + "txt_list":[ + "[Ljava.lang.String;@5f9c8d5d", + "[Ljava.lang.String;@1c2f1cbe" + ] + }, + "source":{ + "version":"1.6.4.Final", + "connector":"postgresql", + "name":"abc", + "ts_ms":1682325982454, + "snapshot":"false", + "db":"test_cdc", + "sequence":"[null,\"434293792\"]", + "schema":"public", + "table":"dts_source_all111", + "txId":201661, + "lsn":434294064, + "xmin":null + }, + "op":"u", + "ts_ms":1682325984233, + "transaction":null + } +} \ No newline at end of file diff --git a/bitsail-components/bitsail-flink-row-parser/pom.xml b/bitsail-components/bitsail-flink-row-parser/pom.xml index fe413b0ca..f5c5da317 100644 --- a/bitsail-components/bitsail-flink-row-parser/pom.xml +++ b/bitsail-components/bitsail-flink-row-parser/pom.xml @@ -133,6 +133,12 @@ bitsail-core-flink-1.11-bridge ${revision} provided + + + com.bytedance.bitsail + bitsail-core-common + + diff --git a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/pom.xml b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/pom.xml index 50519cdfc..13180d386 100644 --- a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/pom.xml +++ b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/pom.xml @@ -26,6 +26,16 @@ bitsail-connector-hive + + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + 2.10.5 + + + + @@ -97,6 +107,10 @@ jersey-client com.sun.jersey + + jackson-module-jaxb-annotations + com.fasterxml.jackson.module + diff --git a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveTableCatalog.java b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveCatalog.java similarity index 83% rename from bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveTableCatalog.java rename to bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveCatalog.java index 2a98a6380..79388bc87 100644 --- a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveTableCatalog.java +++ b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveCatalog.java @@ -16,11 +16,11 @@ package com.bytedance.bitsail.connector.legacy.hive.common; +import com.bytedance.bitsail.common.catalog.table.Catalog; import com.bytedance.bitsail.common.catalog.table.CatalogTable; import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; -import com.bytedance.bitsail.common.catalog.table.CatalogTableDefinition; import com.bytedance.bitsail.common.catalog.table.CatalogTableSchema; -import com.bytedance.bitsail.common.catalog.table.TableCatalog; +import com.bytedance.bitsail.common.catalog.table.TableId; import com.bytedance.bitsail.common.catalog.table.TableOperation; import com.bytedance.bitsail.common.model.ColumnInfo; import com.bytedance.bitsail.common.type.TypeInfoConverter; @@ -36,8 +36,8 @@ import java.util.List; @Builder -public class HiveTableCatalog implements TableCatalog { - private static final Logger LOG = LoggerFactory.getLogger(HiveTableCatalog.class); +public class HiveCatalog implements Catalog { + private static final Logger LOG = LoggerFactory.getLogger(HiveCatalog.class); private final String namespace; private final String database; @@ -58,8 +58,8 @@ public void close() { } @Override - public CatalogTableDefinition createCatalogTableDefinition() { - return CatalogTableDefinition + public TableId createCatalogTableDefinition() { + return TableId .builder() .database(database) .table(table) @@ -67,20 +67,25 @@ public CatalogTableDefinition createCatalogTableDefinition() { } @Override - public boolean tableExists(CatalogTableDefinition catalogTableDefinition) { + public List listTables() { + return null; + } + + @Override + public boolean tableExists(TableId catalogTableDefinition) { //todo real check. return true; } @Override - public CatalogTable getCatalogTable(CatalogTableDefinition catalogTableDefinition) { + public CatalogTable getCatalogTable(TableId catalogTableDefinition) { try { List columnInfo = HiveMetaClientUtil.getColumnInfo(hiveConf, catalogTableDefinition.getDatabase(), catalogTableDefinition.getTable()); return CatalogTable.builder() - .catalogTableDefinition(catalogTableDefinition) + .tableId(catalogTableDefinition) .catalogTableSchema(getCatalogTableSchema(columnInfo)) .build(); } catch (Exception e) { @@ -90,7 +95,7 @@ public CatalogTable getCatalogTable(CatalogTableDefinition catalogTableDefinitio } @Override - public void createTable(CatalogTableDefinition catalogTableDefinition, CatalogTable catalogTable) { + public void createTable(TableId catalogTableDefinition, CatalogTable catalogTable) { throw new UnsupportedOperationException(); } diff --git a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveTableCatalogFactory.java b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveTableCatalogFactory.java index 5e68ea1e9..5b570fb7b 100644 --- a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveTableCatalogFactory.java +++ b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/java/com/bytedance/bitsail/connector/legacy/hive/common/HiveTableCatalogFactory.java @@ -16,28 +16,26 @@ package com.bytedance.bitsail.connector.legacy.hive.common; -import com.bytedance.bitsail.base.catalog.TableCatalogFactory; +import com.bytedance.bitsail.base.catalog.CatalogFactory; import com.bytedance.bitsail.base.connector.BuilderGroup; -import com.bytedance.bitsail.base.execution.ExecutionEnviron; -import com.bytedance.bitsail.common.catalog.table.TableCatalog; +import com.bytedance.bitsail.common.catalog.table.Catalog; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.exception.FrameworkErrorCode; import com.bytedance.bitsail.connector.legacy.hive.option.HiveReaderOptions; import com.bytedance.bitsail.connector.legacy.hive.option.HiveWriterOptions; import com.bytedance.bitsail.connector.legacy.hive.util.HiveConfUtils; -public class HiveTableCatalogFactory implements TableCatalogFactory { +public class HiveTableCatalogFactory implements CatalogFactory { @Override - public TableCatalog createTableCatalog(BuilderGroup builderGroup, - ExecutionEnviron executionEnviron, - BitSailConfiguration connectorConfiguration) { + public Catalog createTableCatalog(BuilderGroup builderGroup, + BitSailConfiguration connectorConfiguration) { if (BuilderGroup.READER.equals(builderGroup)) { String database = connectorConfiguration .getNecessaryOption(HiveReaderOptions.DB_NAME, FrameworkErrorCode.REQUIRED_VALUE); String table = connectorConfiguration .getNecessaryOption(HiveReaderOptions.TABLE_NAME, FrameworkErrorCode.REQUIRED_VALUE); - return HiveTableCatalog + return HiveCatalog .builder() .database(database) .table(table) @@ -50,7 +48,7 @@ public TableCatalog createTableCatalog(BuilderGroup builderGroup, .getNecessaryOption(HiveWriterOptions.DB_NAME, FrameworkErrorCode.REQUIRED_VALUE); String table = connectorConfiguration .getNecessaryOption(HiveWriterOptions.TABLE_NAME, FrameworkErrorCode.REQUIRED_VALUE); - return HiveTableCatalog + return HiveCatalog .builder() .database(database) .table(table) diff --git a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.TableCatalogFactory b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.CatalogFactory similarity index 100% rename from bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.TableCatalogFactory rename to bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.CatalogFactory diff --git a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLTableCatalog.java b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLCatalog.java similarity index 80% rename from bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLTableCatalog.java rename to bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLCatalog.java index ebbb0fb1b..395c164ca 100644 --- a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLTableCatalog.java +++ b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLCatalog.java @@ -17,11 +17,11 @@ package com.bytedance.bitsail.connector.legacy.jdbc.catalog; import com.bytedance.bitsail.common.BitSailException; +import com.bytedance.bitsail.common.catalog.table.Catalog; import com.bytedance.bitsail.common.catalog.table.CatalogTable; import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; -import com.bytedance.bitsail.common.catalog.table.CatalogTableDefinition; import com.bytedance.bitsail.common.catalog.table.CatalogTableSchema; -import com.bytedance.bitsail.common.catalog.table.TableCatalog; +import com.bytedance.bitsail.common.catalog.table.TableId; import com.bytedance.bitsail.common.catalog.table.TableOperation; import com.bytedance.bitsail.common.type.TypeInfoConverter; import com.bytedance.bitsail.common.typeinfo.TypeInfo; @@ -36,8 +36,8 @@ import java.util.List; -public class MySQLTableCatalog implements TableCatalog { - private static final Logger LOG = LoggerFactory.getLogger(MySQLTableCatalog.class); +public class MySQLCatalog implements Catalog { + private static final Logger LOG = LoggerFactory.getLogger(MySQLCatalog.class); private final String database; private final String table; @@ -50,13 +50,13 @@ public class MySQLTableCatalog implements TableCatalog { private TypeInfoConverter typeInfoConverter; @Builder - public MySQLTableCatalog(String database, - String table, - String schema, - String username, - String password, - String url, - String customizedSQL) { + public MySQLCatalog(String database, + String table, + String schema, + String username, + String password, + String url, + String customizedSQL) { this.database = database; this.table = table; this.schema = schema; @@ -78,8 +78,8 @@ public void close() { } @Override - public CatalogTableDefinition createCatalogTableDefinition() { - return CatalogTableDefinition + public TableId createCatalogTableDefinition() { + return TableId .builder() .database(database) .table(table) @@ -87,13 +87,18 @@ public CatalogTableDefinition createCatalogTableDefinition() { } @Override - public boolean tableExists(CatalogTableDefinition catalogTableDefinition) { + public List listTables() { + return null; + } + + @Override + public boolean tableExists(TableId catalogTableDefinition) { //todo doesn't check return true; } @Override - public CatalogTable getCatalogTable(CatalogTableDefinition catalogTableDefinition) { + public CatalogTable getCatalogTable(TableId catalogTableDefinition) { TableInfo tableInfo; try { if (useCustomizedSQL) { @@ -109,7 +114,7 @@ public CatalogTable getCatalogTable(CatalogTableDefinition catalogTableDefinitio } return CatalogTable .builder() - .catalogTableDefinition(catalogTableDefinition) + .tableId(catalogTableDefinition) .catalogTableSchema(CatalogTableSchema.builder() .columns(convertTableColumn( typeInfoConverter, @@ -119,7 +124,7 @@ public CatalogTable getCatalogTable(CatalogTableDefinition catalogTableDefinitio } @Override - public void createTable(CatalogTableDefinition catalogTableDefinition, CatalogTable catalogTable) { + public void createTable(TableId catalogTableDefinition, CatalogTable catalogTable) { throw new UnsupportedOperationException(); } diff --git a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLTableCatalogFactory.java b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLTableCatalogFactory.java index 27485800f..01f742e91 100644 --- a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLTableCatalogFactory.java +++ b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/catalog/MySQLTableCatalogFactory.java @@ -16,11 +16,10 @@ package com.bytedance.bitsail.connector.legacy.jdbc.catalog; -import com.bytedance.bitsail.base.catalog.TableCatalogFactory; +import com.bytedance.bitsail.base.catalog.CatalogFactory; import com.bytedance.bitsail.base.connector.BuilderGroup; -import com.bytedance.bitsail.base.execution.ExecutionEnviron; import com.bytedance.bitsail.common.BitSailException; -import com.bytedance.bitsail.common.catalog.table.TableCatalog; +import com.bytedance.bitsail.common.catalog.table.Catalog; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.exception.CommonErrorCode; import com.bytedance.bitsail.common.exception.FrameworkErrorCode; @@ -34,19 +33,18 @@ import java.util.List; -public class MySQLTableCatalogFactory implements TableCatalogFactory { +public class MySQLTableCatalogFactory implements CatalogFactory { @Override - public TableCatalog createTableCatalog(BuilderGroup builderGroup, - ExecutionEnviron executionEnviron, - BitSailConfiguration connectorConfiguration) { + public Catalog createTableCatalog(BuilderGroup builderGroup, + BitSailConfiguration connectorConfiguration) { if (BuilderGroup.READER.equals(builderGroup)) { List connections = connectorConfiguration .getNecessaryOption(JdbcReaderOptions.CONNECTIONS, FrameworkErrorCode.REQUIRED_VALUE); - return MySQLTableCatalog + return MySQLCatalog .builder() .database(connectorConfiguration.get(JdbcReaderOptions.DB_NAME)) .table(connectorConfiguration.get(JdbcReaderOptions.TABLE_NAME)) @@ -59,7 +57,7 @@ public TableCatalog createTableCatalog(BuilderGroup builderGroup, .url(getClusterUrl(connections)) .build(); } - return MySQLTableCatalog + return MySQLCatalog .builder() .username(connectorConfiguration.get(JdbcWriterOptions.USER_NAME)) .password(connectorConfiguration.get(JdbcWriterOptions.PASSWORD)) diff --git a/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.TableCatalogFactory b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.CatalogFactory similarity index 100% rename from bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.TableCatalogFactory rename to bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.CatalogFactory diff --git a/bitsail-connectors/bitsail-connectors-legacy/pom.xml b/bitsail-connectors/bitsail-connectors-legacy/pom.xml index c88b9082b..7d8cd45bd 100644 --- a/bitsail-connectors/bitsail-connectors-legacy/pom.xml +++ b/bitsail-connectors/bitsail-connectors-legacy/pom.xml @@ -67,12 +67,29 @@ ${scala-2.11.version} 2.11 + + + + + org.reflections + reflections + 0.9.10 + + + + com.bytedance.bitsail bitsail-core-flink-1.11-bridge ${revision} provided + + + com.bytedance.bitsail + bitsail-core-common + + diff --git a/bitsail-connectors/connector-cdc/connector-cdc-base/pom.xml b/bitsail-connectors/connector-cdc/connector-cdc-base/pom.xml index 13877c03a..ff5a6f01d 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-base/pom.xml +++ b/bitsail-connectors/connector-cdc/connector-cdc-base/pom.xml @@ -33,6 +33,27 @@ + + io.debezium + debezium-embedded + ${debezium.version} + + + jakarta.activation + jakarta.activation-api + + + org.slf4j + slf4j-log4j12 + + + + + + com.bytedance.bitsail + bitsail-component-format-debezium + ${revision} + diff --git a/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/option/BinlogReaderOptions.java b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/option/BinlogReaderOptions.java index 2561b1e28..c59688083 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/option/BinlogReaderOptions.java +++ b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/option/BinlogReaderOptions.java @@ -81,4 +81,8 @@ public interface BinlogReaderOptions extends ReaderOptions.BaseReaderOptions { ConfigOption MAX_QUEUE_SIZE = key(READER_PREFIX + "max_queue_size") .defaultValue(8192); + + ConfigOption FORMAT = + key(READER_PREFIX + "format") + .defaultValue("debezium-json"); } diff --git a/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/schema/DebeziumDeserializationFactory.java b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/schema/DebeziumDeserializationFactory.java new file mode 100644 index 000000000..27609dd3a --- /dev/null +++ b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/schema/DebeziumDeserializationFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.connector.cdc.schema; + +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.model.ColumnInfo; +import com.bytedance.bitsail.common.option.ReaderOptions; +import com.bytedance.bitsail.common.type.TypeInfoConverter; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfoUtils; +import com.bytedance.bitsail.component.format.debezium.DebeziumDeserializationSchema; +import com.bytedance.bitsail.component.format.debezium.DebeziumJsonDeserializationSchema; +import com.bytedance.bitsail.component.format.debezium.DebeziumRowDeserializationSchema; +import com.bytedance.bitsail.component.format.debezium.MultipleDebeziumDeserializationSchema; +import com.bytedance.bitsail.connector.cdc.option.BinlogReaderOptions; + +import java.util.List; + +public class DebeziumDeserializationFactory { + + public static DebeziumDeserializationSchema getDebeziumDeserializationSchema(BitSailConfiguration jobConf, TypeInfoConverter typeInfoConverter) { + Boolean multiple = jobConf.get(ReaderOptions.BaseReaderOptions.MULTIPLE_READER_ENABLED); + if (multiple) { + return new MultipleDebeziumDeserializationSchema(jobConf); + } + String format = jobConf.get(BinlogReaderOptions.FORMAT); + if (DebeziumJsonDeserializationSchema.NAME.equalsIgnoreCase(format)) { + return new DebeziumJsonDeserializationSchema(jobConf); + } + List columnInfos = jobConf.get(ReaderOptions.BaseReaderOptions.COLUMNS); + RowTypeInfo rowTypeInfo = TypeInfoUtils.getRowTypeInfo(typeInfoConverter, columnInfos); + return new DebeziumRowDeserializationSchema(jobConf, rowTypeInfo); + + } +} diff --git a/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/BaseCDCSource.java b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/BaseCDCSource.java index 1d3c5f979..e5c6a1cde 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/BaseCDCSource.java +++ b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/BaseCDCSource.java @@ -22,6 +22,7 @@ import com.bytedance.bitsail.base.connector.reader.v1.SourceSplitCoordinator; import com.bytedance.bitsail.base.execution.ExecutionEnviron; import com.bytedance.bitsail.base.extension.ParallelismComputable; +import com.bytedance.bitsail.base.extension.SupportProducedType; import com.bytedance.bitsail.base.parallelism.ParallelismAdvice; import com.bytedance.bitsail.base.serializer.BinarySerializer; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; @@ -29,6 +30,9 @@ import com.bytedance.bitsail.common.row.Row; import com.bytedance.bitsail.common.type.BitSailTypeInfoConverter; import com.bytedance.bitsail.common.type.TypeInfoConverter; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; +import com.bytedance.bitsail.component.format.debezium.DebeziumDeserializationSchema; +import com.bytedance.bitsail.connector.cdc.schema.DebeziumDeserializationFactory; import com.bytedance.bitsail.connector.cdc.source.coordinator.CDCSourceSplitCoordinator; import com.bytedance.bitsail.connector.cdc.source.coordinator.state.AssignmentStateSerializer; import com.bytedance.bitsail.connector.cdc.source.coordinator.state.BaseAssignmentState; @@ -40,18 +44,20 @@ /** * Source to read mysql binlog. */ -public abstract class BaseCDCSource implements Source, ParallelismComputable { +public abstract class BaseCDCSource implements Source, ParallelismComputable, SupportProducedType { protected BitSailConfiguration commonConf; protected BitSailConfiguration readerConf; protected BaseSplitSerializer splitSerializer; + protected DebeziumDeserializationSchema deserializationSchema; @Override public void configure(ExecutionEnviron execution, BitSailConfiguration readerConfiguration) throws IOException { this.readerConf = readerConfiguration; this.commonConf = execution.getCommonConfiguration(); this.splitSerializer = createSplitSerializer(); + this.deserializationSchema = DebeziumDeserializationFactory.getDebeziumDeserializationSchema(readerConf, createTypeInfoConverter()); } @Override @@ -97,4 +103,9 @@ public ParallelismAdvice getParallelismAdvice(BitSailConfiguration commonConf, B .enforceDownStreamChain(false) .build(); } + + @Override + public RowTypeInfo getProducedType() { + return deserializationSchema.getProducedType(); + } } diff --git a/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/reader/BaseCDCSourceReader.java b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/reader/BaseCDCSourceReader.java index 7b0cb7db5..79733c71e 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/reader/BaseCDCSourceReader.java +++ b/bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/reader/BaseCDCSourceReader.java @@ -22,6 +22,7 @@ import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.option.CommonOptions; import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.component.format.debezium.DebeziumDeserializationSchema; import com.bytedance.bitsail.connector.cdc.source.event.BinlogCompleteAckEvent; import com.bytedance.bitsail.connector.cdc.source.split.BaseCDCSplit; import com.bytedance.bitsail.connector.cdc.source.split.BinlogSplit; @@ -51,12 +52,16 @@ public abstract class BaseCDCSourceReader implements SourceReader(); this.reader = getReader(); this.splitSubmitted = false; diff --git a/bitsail-connectors/connector-cdc/connector-cdc-mysql/pom.xml b/bitsail-connectors/connector-cdc/connector-cdc-mysql/pom.xml index 24d5b97c6..669fe2d70 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-mysql/pom.xml +++ b/bitsail-connectors/connector-cdc/connector-cdc-mysql/pom.xml @@ -31,7 +31,7 @@ 8 8 UTF-8 - 1.6.4.Final + 3.0.8 @@ -42,12 +42,6 @@ ${revision} - - com.bytedance.bitsail - bitsail-component-format-debezium - ${revision} - - io.debezium debezium-connector-mysql diff --git a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/MysqlCDCSource.java b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/MysqlCDCSource.java index 1e99e2daf..8ae308368 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/MysqlCDCSource.java +++ b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/MysqlCDCSource.java @@ -33,10 +33,11 @@ public class MysqlCDCSource extends BaseCDCSource { private static final Logger LOG = LoggerFactory.getLogger(MysqlCDCSource.class); + @Override public SourceReader createReader(SourceReader.Context readerContext) { LOG.info("Create Mysql CDC Source"); - return new MysqlCDCSourceReader(readerConf, commonConf, readerContext); + return new MysqlCDCSourceReader(readerConf, commonConf, readerContext, deserializationSchema); } @Override @@ -48,4 +49,5 @@ public BaseSplitSerializer createSplitSerializer() { public String getReaderName() { return "mysql_cdc"; } + } diff --git a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/debezium/MysqlBinlogSplitReader.java b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/debezium/MysqlBinlogSplitReader.java index 160cf30b6..fc1fb08f6 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/debezium/MysqlBinlogSplitReader.java +++ b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/debezium/MysqlBinlogSplitReader.java @@ -16,10 +16,10 @@ package com.bytedance.bitsail.connector.cdc.mysql.source.debezium; +import com.bytedance.bitsail.base.connector.reader.v1.SourceReader; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; -import com.bytedance.bitsail.common.row.BinlogRow; import com.bytedance.bitsail.common.row.Row; -import com.bytedance.bitsail.component.format.debezium.JsonDebeziumSerializationSchema; +import com.bytedance.bitsail.component.format.debezium.DebeziumDeserializationSchema; import com.bytedance.bitsail.connector.cdc.mysql.source.config.MysqlConfig; import com.bytedance.bitsail.connector.cdc.mysql.source.schema.SchemaUtils; import com.bytedance.bitsail.connector.cdc.mysql.source.schema.TableChangeConverter; @@ -52,8 +52,6 @@ import io.debezium.schema.TopicSelector; import io.debezium.util.Clock; import io.debezium.util.SchemaNameAdjuster; -import org.apache.kafka.connect.data.Field; -import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,22 +113,25 @@ public class MysqlBinlogSplitReader implements BinlogSplitReader { private final int subtaskId; - private final JsonDebeziumSerializationSchema serializer; + private final DebeziumDeserializationSchema deserializationSchema; private final BitSailConfiguration jobConf; - public MysqlBinlogSplitReader(BitSailConfiguration jobConf, int subtaskId) { + public MysqlBinlogSplitReader(BitSailConfiguration jobConf, + SourceReader.Context context, + DebeziumDeserializationSchema deserializationSchema) { this.jobConf = jobConf; this.mysqlConfig = MysqlConfig.fromBitSailConf(jobConf); this.schemaNameAdjuster = SchemaNameAdjuster.create(); // handle configuration this.connectorConfig = mysqlConfig.getDbzMySqlConnectorConfig(); - this.subtaskId = subtaskId; + this.subtaskId = context.getIndexOfSubtask(); ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("mysql-binlog-reader-" + this.subtaskId).build(); this.executorService = Executors.newSingleThreadExecutor(threadFactory); this.offset = new HashMap<>(); - this.serializer = new JsonDebeziumSerializationSchema(jobConf); this.isRunning = false; + this.deserializationSchema = deserializationSchema; + this.deserializationSchema.open(); } public void readSplit(BinlogSplit split) { @@ -282,18 +283,7 @@ public boolean isRunning() { public Row poll() { SourceRecord record = this.recordIterator.next(); this.offset = record.sourceOffset(); - byte[] serialized = this.serializer.serialize(record); - Struct val = (Struct) record.value(); - Field keyField = record.keySchema().fields().get(0); - Row result = new Row(BinlogRow.ROW_SIZE); - result.setField(BinlogRow.DATABASE_INDEX, val.getStruct("source").getString("db")); - result.setField(BinlogRow.TABLE_INDEX, val.getStruct("source").getString("table")); - result.setField(BinlogRow.KEY_INDEX, ((Struct) record.key()).get(keyField).toString()); - result.setField(BinlogRow.TIMESTAMP_INDEX, val.getStruct("source").getInt64("ts_ms").toString()); - result.setField(BinlogRow.DDL_FLAG_INDEX, false); - result.setField(BinlogRow.VERSION_INDEX, 1); - result.setField(BinlogRow.VALUE_INDEX, serialized); - return result; + return deserializationSchema.deserialize(record); } @Override diff --git a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/reader/MysqlCDCSourceReader.java b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/reader/MysqlCDCSourceReader.java index bc2ab9f99..5b7e97cd0 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/reader/MysqlCDCSourceReader.java +++ b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/reader/MysqlCDCSourceReader.java @@ -18,6 +18,7 @@ import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.component.format.debezium.DebeziumDeserializationSchema; import com.bytedance.bitsail.connector.cdc.mysql.source.debezium.DebeziumHelper; import com.bytedance.bitsail.connector.cdc.mysql.source.debezium.MysqlBinlogSplitReader; import com.bytedance.bitsail.connector.cdc.source.offset.BinlogOffset; @@ -36,8 +37,11 @@ public class MysqlCDCSourceReader extends BaseCDCSourceReader { private static final Logger LOG = LoggerFactory.getLogger(MysqlCDCSourceReader.class); - public MysqlCDCSourceReader(BitSailConfiguration readerConf, BitSailConfiguration commonConf, Context readerContext) { - super(readerConf, commonConf, readerContext); + public MysqlCDCSourceReader(BitSailConfiguration readerConf, + BitSailConfiguration commonConf, + Context context, + DebeziumDeserializationSchema deserializationSchema) { + super(readerConf, commonConf, context, deserializationSchema); } @Override @@ -56,6 +60,7 @@ public List snapshotState(long checkpointId) { @Override public BinlogSplitReader getReader() { - return new MysqlBinlogSplitReader(readerConf, readerContext.getIndexOfSubtask()); + return new MysqlBinlogSplitReader(readerConf, readerContext, deserializationSchema); } + } diff --git a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/test/java/com/bytedance/bitsail/connector/cdc/mysql/source/MockConnectionsTest.java b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/test/java/com/bytedance/bitsail/connector/cdc/mysql/source/MockConnectionsTest.java index 79adb706f..30bac45f5 100644 --- a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/test/java/com/bytedance/bitsail/connector/cdc/mysql/source/MockConnectionsTest.java +++ b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/test/java/com/bytedance/bitsail/connector/cdc/mysql/source/MockConnectionsTest.java @@ -16,11 +16,14 @@ package com.bytedance.bitsail.connector.cdc.mysql.source; +import com.bytedance.bitsail.base.connector.reader.v1.SourceReader; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.component.format.debezium.DebeziumJsonDeserializationSchema; import com.bytedance.bitsail.connector.cdc.model.ClusterInfo; import com.bytedance.bitsail.connector.cdc.model.ConnectionInfo; import com.bytedance.bitsail.connector.cdc.mysql.source.container.MySQLContainerMariadbAdapter; +import com.bytedance.bitsail.connector.cdc.mysql.source.context.SourceMockContext; import com.bytedance.bitsail.connector.cdc.mysql.source.debezium.MysqlBinlogSplitReader; import com.bytedance.bitsail.connector.cdc.mysql.source.schema.SchemaUtils; import com.bytedance.bitsail.connector.cdc.option.BinlogReaderOptions; @@ -47,7 +50,6 @@ import org.testcontainers.utility.DockerImageName; import java.time.ZoneId; -import java.util.Arrays; import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -64,6 +66,8 @@ public class MockConnectionsTest { private static final String TEST_USERNAME = "user1"; private static final String TEST_PASSWORD = "password1"; private static MySQLContainer container; + private static SourceReader.Context context; + private static DebeziumJsonDeserializationSchema deserializationSchema; @BeforeClass public static void before() { @@ -74,6 +78,8 @@ public static void before() { .withPassword(TEST_PASSWORD) .withLogConsumer(new Slf4jLogConsumer(LOG)); + deserializationSchema = new DebeziumJsonDeserializationSchema(BitSailConfiguration.newDefault()); + context = new SourceMockContext(1, deserializationSchema.getProducedType()); Startables.deepStart(Stream.of(container)).join(); } @@ -129,14 +135,14 @@ public void testBinlogSplitReader() throws InterruptedException { jobConf.set(BinlogReaderOptions.PASSWORD, TEST_PASSWORD); jobConf.set(BinlogReaderOptions.INITIAL_OFFSET_TYPE, "latest"); - MysqlBinlogSplitReader reader = new MysqlBinlogSplitReader(jobConf, 0); + MysqlBinlogSplitReader reader = new MysqlBinlogSplitReader(jobConf, context, deserializationSchema); BinlogSplit split = new BinlogSplit("split-1", BinlogOffset.earliest(), BinlogOffset.boundless()); reader.readSplit(split); int maxPeriod = 0; while (maxPeriod <= 5) { if (reader.hasNext()) { Row row = reader.poll(); - Arrays.stream(row.getFields()).forEach(o -> LOG.info(o.toString())); + LOG.info("row: {}.", row); maxPeriod++; } TimeUnit.SECONDS.sleep(1); diff --git a/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/test/java/com/bytedance/bitsail/connector/cdc/mysql/source/context/SourceMockContext.java b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/test/java/com/bytedance/bitsail/connector/cdc/mysql/source/context/SourceMockContext.java new file mode 100644 index 000000000..e25bac10b --- /dev/null +++ b/bitsail-connectors/connector-cdc/connector-cdc-mysql/src/test/java/com/bytedance/bitsail/connector/cdc/mysql/source/context/SourceMockContext.java @@ -0,0 +1,46 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.connector.cdc.mysql.source.context; + +import com.bytedance.bitsail.base.connector.reader.v1.SourceReader; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; + +public class SourceMockContext implements SourceReader.Context { + + private final int id; + private final RowTypeInfo rowTypeInfo; + + public SourceMockContext(int id, RowTypeInfo rowTypeInfo) { + this.id = id; + this.rowTypeInfo = rowTypeInfo; + } + + @Override + public RowTypeInfo getRowTypeInfo() { + return rowTypeInfo; + } + + @Override + public int getIndexOfSubtask() { + return id; + } + + @Override + public void sendSplitRequest() { + + } +} diff --git a/bitsail-connectors/connector-cdc/pom.xml b/bitsail-connectors/connector-cdc/pom.xml index 49b71610f..593eb0f3c 100644 --- a/bitsail-connectors/connector-cdc/pom.xml +++ b/bitsail-connectors/connector-cdc/pom.xml @@ -33,13 +33,17 @@ - 8 - 8 - UTF-8 + 1.6.4.Final - - - + + + + io.debezium + debezium-embedded + ${debezium.version} + + + \ No newline at end of file diff --git a/bitsail-connectors/connector-doris/src/main/resources/doris-type-converter.yaml b/bitsail-connectors/connector-doris/src/main/resources/doris-type-converter.yaml index 93ba709cc..9b7c4cb3f 100644 --- a/bitsail-connectors/connector-doris/src/main/resources/doris-type-converter.yaml +++ b/bitsail-connectors/connector-doris/src/main/resources/doris-type-converter.yaml @@ -59,6 +59,9 @@ engine.type.to.bitsail.type.converter: - source.type: date target.type: date.date + - source.type: bit + target.type: byte + - source.type: datetime target.type: date.datetime diff --git a/bitsail-connectors/connector-kafka/src/main/java/com/bytedance/bitsail/connector/kafka/sink/KafkaWriter.java b/bitsail-connectors/connector-kafka/src/main/java/com/bytedance/bitsail/connector/kafka/sink/KafkaWriter.java index 73a2e6659..cb0ce4db0 100644 --- a/bitsail-connectors/connector-kafka/src/main/java/com/bytedance/bitsail/connector/kafka/sink/KafkaWriter.java +++ b/bitsail-connectors/connector-kafka/src/main/java/com/bytedance/bitsail/connector/kafka/sink/KafkaWriter.java @@ -128,7 +128,7 @@ public KafkaWriter(BitSailConfiguration commonConf, BitSailConfiguration writerC public void write(Row record) throws IOException { checkErroneous(); //TODO: refactor this as a format factory - if (format.equals("debezium")) { + if (format.equals("debezium-json")) { writeDebezium(record); } else { String result = jsonConverter.convert(record).toString(); @@ -153,13 +153,7 @@ public void writeDebezium(Row record) { String key = record.getString(BinlogRow.KEY_INDEX); partitionFieldsValues[0] = key; int partitionId = choosePartitionIdByFields(partitionFieldsValues); - Map headers = new HashMap<>(4); - headers.put("db", record.getString(BinlogRow.DATABASE_INDEX)); - headers.put("table", record.getString(BinlogRow.TABLE_INDEX)); - headers.put("ddl_flag", String.valueOf(record.getBoolean(BinlogRow.DDL_FLAG_INDEX))); - headers.put("version", String.valueOf(record.getInt(BinlogRow.VERSION_INDEX))); - byte[] value = record.getBinary(BinlogRow.VALUE_INDEX); - sendWithHeaders(key, value, partitionId, headers); + sendWithHeaders(null, record.getString(2), partitionId, null); } @Override diff --git a/bitsail-cores/bitsail-core-common/pom.xml b/bitsail-cores/bitsail-core-common/pom.xml new file mode 100644 index 000000000..edcb415ac --- /dev/null +++ b/bitsail-cores/bitsail-core-common/pom.xml @@ -0,0 +1,56 @@ + + + + + + bitsail-cores + com.bytedance.bitsail + ${revision} + + 4.0.0 + + bitsail-core-common + + + + com.bytedance.bitsail + bitsail-base + ${revision} + + + + com.bytedance.bitsail + bitsail-common + ${revision} + + + + com.bytedance.bitsail + bitsail-component-format-debezium + ${revision} + + + + com.bytedance.bitsail + connector-print + ${revision} + + + + \ No newline at end of file diff --git a/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableCommitSerializer.java b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableCommitSerializer.java new file mode 100644 index 000000000..43ef553a1 --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableCommitSerializer.java @@ -0,0 +1,96 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.serializer.multiple; + +import com.bytedance.bitsail.base.serializer.BinarySerializer; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.core.common.sink.multiple.comittable.MultipleTableCommit; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.SerializationUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class MultipleTableCommitSerializer implements BinarySerializer> { + + private BinarySerializer original; + + public MultipleTableCommitSerializer(BinarySerializer original) { + this.original = original; + } + + @Override + public int getVersion() { + return original.getVersion(); + } + + @Override + public byte[] serialize(MultipleTableCommit multipleTableCommit) throws IOException { + if (Objects.isNull(multipleTableCommit)) { + return null; + } + + byte[] buffer; + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + TableId tableId = multipleTableCommit.getTableId(); + List commits = multipleTableCommit.getCommits(); + buffer = SerializationUtils.serialize(tableId); + outputStream.write(buffer.length); + outputStream.write(buffer); + for (CommitT commitT : commits) { + buffer = original.serialize(commitT); + outputStream.write(buffer.length); + outputStream.write(buffer); + } + buffer = outputStream.toByteArray(); + } + + return buffer; + } + + @Override + public MultipleTableCommit deserialize(int version, byte[] serialized) throws IOException { + if (serialized == null) { + return null; + } + MultipleTableCommit.MultipleTableCommitBuilder builder = MultipleTableCommit.builder(); + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(serialized)) { + int length = inputStream.read(); + byte[] buffer = new byte[length]; + + inputStream.read(buffer, 0, length); + TableId tableId = SerializationUtils.deserialize(buffer); + builder.tableId(tableId); + + List commits = Lists.newArrayList(); + while ((length = inputStream.read()) != -1) { + buffer = new byte[length]; + inputStream.read(buffer, 0, length); + CommitT deserialize = original.deserialize(version, buffer); + commits.add(deserialize); + } + + builder.commits(commits); + + } + return builder.build(); + } +} diff --git a/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableStateSerializer.java b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableStateSerializer.java new file mode 100644 index 000000000..62c00d64f --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableStateSerializer.java @@ -0,0 +1,96 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.serializer.multiple; + +import com.bytedance.bitsail.base.serializer.BinarySerializer; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.core.common.sink.multiple.state.MultipleTableState; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.SerializationUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class MultipleTableStateSerializer implements BinarySerializer> { + + private BinarySerializer original; + + public MultipleTableStateSerializer(BinarySerializer original) { + this.original = original; + } + + @Override + public int getVersion() { + return original.getVersion(); + } + + @Override + public byte[] serialize(MultipleTableState multipleTableCommit) throws IOException { + if (Objects.isNull(multipleTableCommit)) { + return null; + } + + byte[] buffer; + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + TableId tableId = multipleTableCommit.getTableId(); + List commits = multipleTableCommit.getState(); + buffer = SerializationUtils.serialize(tableId); + outputStream.write(buffer.length); + outputStream.write(buffer); + for (WriterStateT commitT : commits) { + buffer = original.serialize(commitT); + outputStream.write(buffer.length); + outputStream.write(buffer); + } + buffer = outputStream.toByteArray(); + } + + return buffer; + } + + @Override + public MultipleTableState deserialize(int version, byte[] serialized) throws IOException { + if (serialized == null) { + return null; + } + MultipleTableState.MultipleTableStateBuilder builder = MultipleTableState.builder(); + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(serialized)) { + int length = inputStream.read(); + byte[] buffer = new byte[length]; + + inputStream.read(buffer, 0, length); + TableId tableId = SerializationUtils.deserialize(buffer); + builder.tableId(tableId); + + List commits = Lists.newArrayList(); + while ((length = inputStream.read()) != -1) { + buffer = new byte[length]; + inputStream.read(buffer, 0, length); + WriterStateT deserialize = original.deserialize(version, buffer); + commits.add(deserialize); + } + + builder.state(commits); + + } + return builder.build(); + } +} diff --git a/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableCommitter.java b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableCommitter.java new file mode 100644 index 000000000..b7841e03d --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableCommitter.java @@ -0,0 +1,63 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.sink.multiple; + +import com.bytedance.bitsail.base.connector.writer.v1.WriterCommitter; +import com.bytedance.bitsail.base.extension.SupportMultipleSinkTable; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.core.common.sink.multiple.comittable.MultipleTableCommit; + +import com.google.common.collect.Maps; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class MultipleTableCommitter implements WriterCommitter> { + + private final SupportMultipleSinkTable supplier; + private final Map> multipleCommitters; + private final BitSailConfiguration templateConfiguration; + + public MultipleTableCommitter(BitSailConfiguration templateConfiguration, + SupportMultipleSinkTable supplier) { + this.supplier = supplier; + this.multipleCommitters = Maps.newConcurrentMap(); + this.templateConfiguration = templateConfiguration; + } + + @Override + public List> commit(List> committables) throws IOException { + for (MultipleTableCommit committable : committables) { + TableId tableId = committable.getTableId(); + WriterCommitter realWriterCommitter = multipleCommitters.get(tableId); + if (Objects.isNull(realWriterCommitter)) { + BitSailConfiguration configuration = supplier.applyTableId(templateConfiguration, tableId); + Optional> committer = supplier.createCommitter(configuration); + realWriterCommitter = committer.get(); + multipleCommitters.put(tableId, realWriterCommitter); + } + realWriterCommitter.commit(committable.getCommits()); + } + return Collections.emptyList(); + } +} diff --git a/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableSink.java b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableSink.java new file mode 100644 index 000000000..f88c67b9f --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableSink.java @@ -0,0 +1,136 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.sink.multiple; + +import com.bytedance.bitsail.base.catalog.CatalogFactory; +import com.bytedance.bitsail.base.connector.BuilderGroup; +import com.bytedance.bitsail.base.connector.writer.v1.Sink; +import com.bytedance.bitsail.base.connector.writer.v1.Writer; +import com.bytedance.bitsail.base.connector.writer.v1.WriterCommitter; +import com.bytedance.bitsail.base.extension.SupportMultipleSinkTable; +import com.bytedance.bitsail.base.serializer.BinarySerializer; +import com.bytedance.bitsail.common.catalog.table.Catalog; +import com.bytedance.bitsail.common.catalog.table.CatalogTable; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.option.WriterOptions; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.type.TypeInfoConverter; +import com.bytedance.bitsail.common.typeinfo.TypeInfoValueConverter; +import com.bytedance.bitsail.component.format.debezium.DebeziumRowFilterNamesDeserializationSchema; +import com.bytedance.bitsail.core.common.serializer.multiple.MultipleTableCommitSerializer; +import com.bytedance.bitsail.core.common.serializer.multiple.MultipleTableStateSerializer; +import com.bytedance.bitsail.core.common.sink.multiple.comittable.MultipleTableCommit; +import com.bytedance.bitsail.core.common.sink.multiple.state.MultipleTableState; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +public class MultipleTableSink + implements Sink, MultipleTableState> { + + private static final Logger LOG = LoggerFactory.getLogger(MultipleTableSink.class); + + private final Sink realSink; + private final CatalogFactory factory; + + private Catalog catalog; + private Map catalogTables; + private BitSailConfiguration commonConfiguration; + private BitSailConfiguration writerConfiguration; + private Pattern patternOfTable; + + public MultipleTableSink(Sink realSink, + CatalogFactory factory) { + this.realSink = realSink; + this.factory = Preconditions.checkNotNull(factory, + String.format("The sink %s must implement catalog when support multiple table write.", + realSink.getWriterName())); + } + + @Override + public void configure(BitSailConfiguration commonConfiguration, BitSailConfiguration writerConfiguration) throws Exception { + this.commonConfiguration = commonConfiguration; + this.writerConfiguration = writerConfiguration; + this.patternOfTable = Pattern.compile(writerConfiguration.get(WriterOptions.BaseWriterOptions.TABLE_PATTERN)); + this.catalog = factory.createTableCatalog(BuilderGroup.WRITER, writerConfiguration); + this.catalog.open(realSink.createTypeInfoConverter()); + this.catalogTables = Maps.newHashMap(); + + List tableIds = catalog.listTables(); + for (TableId tableId : tableIds) { + if (patternOfTable.matcher(tableId.toString()).find()) { + LOG.info("Match table {} of the pattern {}.", tableId, patternOfTable.pattern()); + CatalogTable catalogTable = catalog.getCatalogTable(tableId); + catalogTables.put(tableId, catalogTable); + } + } + this.realSink.configure(commonConfiguration, writerConfiguration); + } + + @Override + public Writer, MultipleTableState> createWriter(Writer.Context> context) + throws IOException { + return new MultipleTableWriter<>( + writerConfiguration, + context, + new TypeInfoValueConverter(commonConfiguration), + (SupportMultipleSinkTable) realSink, + catalogTables, + patternOfTable, + new DebeziumRowFilterNamesDeserializationSchema(writerConfiguration)); + } + + @Override + public Optional>> createCommitter() { + return Optional.of( + new MultipleTableCommitter<>( + writerConfiguration, + (SupportMultipleSinkTable) realSink + ) + ); + } + + @Override + public String getWriterName() { + return realSink.getWriterName(); + } + + @Override + public TypeInfoConverter createTypeInfoConverter() { + return realSink.createTypeInfoConverter(); + } + + @Override + public BinarySerializer> getCommittableSerializer() { + return new MultipleTableCommitSerializer<>(realSink.getCommittableSerializer()); + } + + @Override + public BinarySerializer> getWriteStateSerializer() { + return new MultipleTableStateSerializer<>(realSink.getWriteStateSerializer()); + } +} diff --git a/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableWriter.java b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableWriter.java new file mode 100644 index 000000000..ecfd5dd3f --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableWriter.java @@ -0,0 +1,307 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.sink.multiple; + +import com.bytedance.bitsail.base.connector.writer.v1.Writer; +import com.bytedance.bitsail.base.extension.SupportMultipleSinkTable; +import com.bytedance.bitsail.common.BitSailException; +import com.bytedance.bitsail.common.catalog.table.CatalogTable; +import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; +import com.bytedance.bitsail.common.catalog.table.CatalogTableSchema; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.exception.CommonErrorCode; +import com.bytedance.bitsail.common.option.WriterOptions; +import com.bytedance.bitsail.common.row.MultipleTableRow; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfoValueConverter; +import com.bytedance.bitsail.component.format.debezium.DebeziumRowFilterNamesDeserializationSchema; +import com.bytedance.bitsail.core.common.sink.multiple.comittable.MultipleTableCommit; +import com.bytedance.bitsail.core.common.sink.multiple.state.MultipleTableState; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.collections.MapUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class MultipleTableWriter + implements Writer, MultipleTableState> { + + private static final Logger LOG = LoggerFactory.getLogger(MultipleTableWriter.class); + + private final SupportMultipleSinkTable supplier; + private final Context> context; + private final BitSailConfiguration templateConfiguration; + private final Map catalogTables; + private final DebeziumRowFilterNamesDeserializationSchema deserializationSchema; + private final TypeInfoValueConverter valueConverter; + private final Pattern patternOfTable; + private final String database; + + private transient Map> restoredMultiTableWriters; + private transient Map> processedMultiTableWriters; + private transient Map tableIdRowTypeInfos; + + public MultipleTableWriter(BitSailConfiguration templateConfiguration, + Context> context, + TypeInfoValueConverter typeInfoValueConverter, + SupportMultipleSinkTable supplier, + Map catalogTables, + Pattern patternOfTable, + DebeziumRowFilterNamesDeserializationSchema deserializationSchema) { + this.templateConfiguration = templateConfiguration; + this.context = context; + this.supplier = supplier; + this.catalogTables = catalogTables; + this.deserializationSchema = deserializationSchema; + this.valueConverter = typeInfoValueConverter; + this.processedMultiTableWriters = Maps.newConcurrentMap(); + this.restoredMultiTableWriters = Maps.newConcurrentMap(); + this.tableIdRowTypeInfos = Maps.newConcurrentMap(); + this.patternOfTable = patternOfTable; + this.database = templateConfiguration.get(WriterOptions.BaseWriterOptions.DB_NAME); + + restore(); + } + + private void restore() { + if (context.isRestored()) { + for (MultipleTableState tableState : context.getRestoreStates()) { + TableId tableId = tableState.getTableId(); + CatalogTable catalogTable = catalogTables.get(tableId); + if (Objects.isNull(catalogTable)) { + LOG.warn("Subtask {} table {} already be deleted, skip restore it from state.", + context.getIndexOfSubTaskId(), tableId); + continue; + } + Context clone = cloneWriterContext(catalogTable, context); + BitSailConfiguration configuration = supplier.applyTableId(templateConfiguration, tableId); + Writer writer = supplier.createWriter(clone, configuration); + restoredMultiTableWriters.put(tableId, writer); + } + } + } + + @Override + public void write(Row element) throws IOException { + MultipleTableRow multipleTableRow = MultipleTableRow.of(element); + TableId tableId = TableId.of(multipleTableRow.getTableId()); + tableId = TableId.of(database, tableId.getTable()); + + if (!(patternOfTable.matcher(tableId.getTable()).find())) { + LOG.debug("Table {} not match with pattern: {}.", tableId.getTable(), patternOfTable.pattern()); + return; + } + + Writer realWriter; + if (processedMultiTableWriters.containsKey(tableId)) { + realWriter = processedMultiTableWriters.get(tableId); + } else if (restoredMultiTableWriters.containsKey(tableId)) { + realWriter = restoredMultiTableWriters.get(tableId); + } else { + BitSailConfiguration current = supplier + .applyTableId(templateConfiguration.clone(), tableId); + + CatalogTable catalogTable = catalogTables.get(tableId); + if (Objects.isNull(catalogTable)) { + //Not support new table in runtime just now. + LOG.warn("Subtask {} discovered new table: {} from input.", context.getIndexOfSubTaskId(), + tableId); + LOG.warn("Subtask {} not support create new table: {} in runtime right now, skip it.", context.getIndexOfSubTaskId(), + tableId); + return; + } + LOG.info("Subtask {} create real writer for the table: {}.", context.getIndexOfSubTaskId(), tableId); + + Context clone = cloneWriterContext(catalogTable, context); + + realWriter = supplier.createWriter(clone, current); + processedMultiTableWriters.put(tableId, realWriter); + } + + RowTypeInfo rowTypeInfo = tableIdRowTypeInfos.get(tableId); + Row deserialize = deserializationSchema.deserialize( + multipleTableRow.getValue(), + rowTypeInfo.getFieldNames()); + + for (int index = 0; index < rowTypeInfo.getTypeInfos().length; index++) { + + try { + //convert field to real writer type info. + deserialize.setField(index, + valueConverter.convertObject( + deserialize.getField(index), + rowTypeInfo.getTypeInfos()[index]) + ); + } catch (Exception e) { + LOG.error("Subtask {} failed to convert field name {}'s value {} to dest type info {}.", + context.getIndexOfSubTaskId(), + rowTypeInfo.getFieldNames()[index], + deserialize.getField(index), + rowTypeInfo.getTypeInfos()[index], e); + //handled as dirty record. + throw BitSailException.asBitSailException(CommonErrorCode.CONVERT_NOT_SUPPORT, + String.format("Subtask %s failed to convert field name %s to dest type info %S.", + context.getIndexOfSubTaskId(), + rowTypeInfo.getFieldNames()[index], + rowTypeInfo.getTypeInfos()[index])); + } + } + realWriter.write((InputT) deserialize); + } + + private Context cloneWriterContext(CatalogTable catalogTable, + Context> context) { + return new Context() { + + @Override + public RowTypeInfo getRowTypeInfo() { + if (tableIdRowTypeInfos.containsKey(catalogTable.getTableId())) { + return tableIdRowTypeInfos.get(catalogTable.getTableId()); + } + CatalogTableSchema catalogTableSchema = catalogTable.getCatalogTableSchema(); + List columns = catalogTableSchema.getColumns(); + String[] fieldNames = columns.stream() + .map(CatalogTableColumn::getName) + .collect(Collectors.toList()) + .toArray(new String[] {}); + + TypeInfo[] fieldTypes = columns.stream() + .map(CatalogTableColumn::getType) + .collect(Collectors.toList()) + .toArray(new TypeInfo[] {}); + + RowTypeInfo rowTypeInfo = new RowTypeInfo(fieldNames, fieldTypes); + tableIdRowTypeInfos.put(catalogTable.getTableId(), rowTypeInfo); + return rowTypeInfo; + } + + @Override + public int getIndexOfSubTaskId() { + return context.getIndexOfSubTaskId(); + } + + @Override + public boolean isRestored() { + for (MultipleTableState tableState : context.getRestoreStates()) { + if (Objects.equals(tableState.getTableId(), catalogTable.getTableId())) { + return true; + } + } + return false; + } + + @Override + public List getRestoreStates() { + for (MultipleTableState tableState : context.getRestoreStates()) { + if (Objects.equals(tableState.getTableId(), catalogTable.getTableId())) { + return tableState.getState(); + } + } + return Collections.emptyList(); + } + }; + } + + @Override + public void flush(boolean endOfInput) throws IOException { + for (Writer writer : processedMultiTableWriters.values()) { + writer.flush(endOfInput); + } + } + + @Override + public List> prepareCommit() throws IOException { + List> prepared = Lists.newArrayList(); + for (TableId tableId : processedMultiTableWriters.keySet()) { + List commit = processedMultiTableWriters.get(tableId).prepareCommit(); + MultipleTableCommit tableCommit = MultipleTableCommit + .builder() + .commits(commit) + .tableId(tableId) + .build(); + prepared.add(tableCommit); + } + if (MapUtils.isNotEmpty(restoredMultiTableWriters)) { + for (TableId tableId : restoredMultiTableWriters.keySet()) { + if (processedMultiTableWriters.containsKey(tableId)) { + continue; + } + Writer writer = restoredMultiTableWriters.get(tableId); + MultipleTableCommit tableCommit = MultipleTableCommit + .builder() + .commits(writer.prepareCommit()) + .tableId(tableId) + .build(); + prepared.add(tableCommit); + //close it. + writer.close(); + } + //clear restored multi writer. + restoredMultiTableWriters.clear(); + } + return prepared; + } + + @Override + public List> snapshotState(long checkpointId) throws IOException { + List> states = Lists.newArrayList(); + for (TableId tableId : processedMultiTableWriters.keySet()) { + List state = processedMultiTableWriters.get(tableId).snapshotState(checkpointId); + + MultipleTableState tableState = MultipleTableState + .builder() + .tableId(tableId) + .state(state) + .build(); + + states.add(tableState); + } + + return states; + } + + @Override + public void close() throws IOException { + for (Writer writer : processedMultiTableWriters.values()) { + writer.close(); + } + } + + @VisibleForTesting + public Map> getProcessedMultiTableWriters() { + return processedMultiTableWriters; + } + + @VisibleForTesting + public Map getTableIdRowTypeInfos() { + return tableIdRowTypeInfos; + } +} diff --git a/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/comittable/MultipleTableCommit.java b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/comittable/MultipleTableCommit.java new file mode 100644 index 000000000..1277f8a69 --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/comittable/MultipleTableCommit.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.sink.multiple.comittable; + +import com.bytedance.bitsail.common.catalog.table.TableId; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Data +@Builder +public class MultipleTableCommit implements Serializable { + + private TableId tableId; + + private List commits; +} diff --git a/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/state/MultipleTableState.java b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/state/MultipleTableState.java new file mode 100644 index 000000000..9ac960c2c --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/main/java/com/bytedance/bitsail/core/common/sink/multiple/state/MultipleTableState.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.sink.multiple.state; + +import com.bytedance.bitsail.common.catalog.table.TableId; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Data +@Builder +public class MultipleTableState implements Serializable { + + private TableId tableId; + + private List state; +} diff --git a/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/catalog/FakeCatalog.java b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/catalog/FakeCatalog.java new file mode 100644 index 000000000..b11a97c2f --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/catalog/FakeCatalog.java @@ -0,0 +1,89 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.catalog; + +import com.bytedance.bitsail.common.catalog.table.Catalog; +import com.bytedance.bitsail.common.catalog.table.CatalogTable; +import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.catalog.table.TableOperation; +import com.bytedance.bitsail.common.type.TypeInfoConverter; +import com.bytedance.bitsail.common.typeinfo.TypeInfo; + +import java.util.List; + +public class FakeCatalog implements Catalog { + + private final List tableIds; + + private final List catalogTables; + + public FakeCatalog(List tableIds, List catalogTables) { + this.tableIds = tableIds; + this.catalogTables = catalogTables; + } + + @Override + public void open(TypeInfoConverter typeInfoConverter) { + + } + + @Override + public void close() { + + } + + @Override + public TableId createCatalogTableDefinition() { + return null; + } + + @Override + public List listTables() { + return tableIds; + } + + @Override + public boolean tableExists(TableId catalogTableDefinition) { + return false; + } + + @Override + public CatalogTable getCatalogTable(TableId tableId) { + return catalogTables.get(catalogTables.indexOf(tableId)); + } + + @Override + public void createTable(TableId catalogTableDefinition, CatalogTable catalogTable) { + + } + + @Override + public void alterTable(TableOperation tableOperation, CatalogTable table) { + + } + + @Override + public void alterTableColumns(TableOperation tableOperation, List catalogTableColumns) { + + } + + @Override + public boolean compareTypeCompatible(TypeInfo original, TypeInfo compared) { + return false; + } +} diff --git a/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableCommitSerializerTest.java b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableCommitSerializerTest.java new file mode 100644 index 000000000..c5ad9f47a --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableCommitSerializerTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.serializer.multiple; + +import com.bytedance.bitsail.base.serializer.SimpleVersionedBinarySerializer; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.core.common.sink.multiple.comittable.MultipleTableCommit; + +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class MultipleTableCommitSerializerTest { + + private MultipleTableCommitSerializer multipleTableCommitSerializer; + + @Before + public void before() { + multipleTableCommitSerializer = new MultipleTableCommitSerializer<>(new SimpleVersionedBinarySerializer()); + } + + @Test + public void testSerializer() throws IOException { + MultipleTableCommit multipleTableCommit = new MultipleTableCommit(); + + multipleTableCommit.setTableId(TableId.of("test", "name")); + multipleTableCommit.setCommits(Lists.newArrayList("a", "b", "c")); + + byte[] serialize = multipleTableCommitSerializer.serialize(multipleTableCommit); + + MultipleTableCommit deserialize = multipleTableCommitSerializer + .deserialize(multipleTableCommitSerializer.getVersion(), serialize); + + Assert.assertEquals(multipleTableCommit.getTableId(), deserialize.getTableId()); + Assert.assertEquals(multipleTableCommit.getCommits(), deserialize.getCommits()); + } + +} \ No newline at end of file diff --git a/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableStateSerializerTest.java b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableStateSerializerTest.java new file mode 100644 index 000000000..3e18261c8 --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/serializer/multiple/MultipleTableStateSerializerTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.serializer.multiple; + +import com.bytedance.bitsail.base.serializer.SimpleVersionedBinarySerializer; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.core.common.sink.multiple.state.MultipleTableState; + +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class MultipleTableStateSerializerTest { + + private MultipleTableStateSerializer multipleTableCommitSerializer; + + @Before + public void before() { + multipleTableCommitSerializer = new MultipleTableStateSerializer<>(new SimpleVersionedBinarySerializer()); + } + + @Test + public void testSerializer() throws IOException { + MultipleTableState multipleTableState = new MultipleTableState(); + + multipleTableState.setTableId(TableId.of("test", "name")); + multipleTableState.setState(Lists.newArrayList("a", "b", "c")); + + byte[] serialize = multipleTableCommitSerializer.serialize(multipleTableState); + + MultipleTableState deserialize = multipleTableCommitSerializer + .deserialize(multipleTableCommitSerializer.getVersion(), serialize); + + Assert.assertEquals(multipleTableState.getTableId(), deserialize.getTableId()); + Assert.assertEquals(multipleTableState.getState(), deserialize.getState()); + } + +} \ No newline at end of file diff --git a/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/sink/MultiTablePrintSink.java b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/sink/MultiTablePrintSink.java new file mode 100644 index 000000000..0b5ac1f69 --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/sink/MultiTablePrintSink.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.sink; + +import com.bytedance.bitsail.base.connector.writer.v1.Writer; +import com.bytedance.bitsail.base.connector.writer.v1.WriterCommitter; +import com.bytedance.bitsail.base.extension.SupportMultipleSinkTable; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.option.WriterOptions; +import com.bytedance.bitsail.connector.print.sink.PrintSink; +import com.bytedance.bitsail.connector.print.sink.PrintWriter; + +import java.util.Optional; + +public class MultiTablePrintSink extends PrintSink + implements SupportMultipleSinkTable { + @Override + public Writer createWriter(Writer.Context context, + BitSailConfiguration templateConfiguration) { + return new PrintWriter(templateConfiguration, context); + } + + @Override + public Optional> createCommitter(BitSailConfiguration templateConfiguration) { + return Optional.empty(); + } + + @Override + public BitSailConfiguration applyTableId(BitSailConfiguration template, TableId tableId) { + return template.set(WriterOptions.BaseWriterOptions.TABLE_NAME, tableId.getTable()); + } +} diff --git a/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableWriterTest.java b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableWriterTest.java new file mode 100644 index 000000000..32dfa0434 --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/test/java/com/bytedance/bitsail/core/common/sink/multiple/MultipleTableWriterTest.java @@ -0,0 +1,184 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.core.common.sink.multiple; + +import com.bytedance.bitsail.base.connector.writer.v1.Writer; +import com.bytedance.bitsail.base.extension.SupportMultipleSinkTable; +import com.bytedance.bitsail.common.catalog.table.CatalogTable; +import com.bytedance.bitsail.common.catalog.table.CatalogTableColumn; +import com.bytedance.bitsail.common.catalog.table.CatalogTableSchema; +import com.bytedance.bitsail.common.catalog.table.TableId; +import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.common.option.WriterOptions; +import com.bytedance.bitsail.common.row.MultipleTableRow; +import com.bytedance.bitsail.common.row.Row; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfo; +import com.bytedance.bitsail.common.typeinfo.TypeInfoValueConverter; +import com.bytedance.bitsail.common.typeinfo.TypeInfos; +import com.bytedance.bitsail.component.format.debezium.DebeziumRowFilterNamesDeserializationSchema; +import com.bytedance.bitsail.core.common.sink.MultiTablePrintSink; +import com.bytedance.bitsail.core.common.sink.multiple.state.MultipleTableState; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class MultipleTableWriterTest { + + private MultiTablePrintSink multiTablePrintSink; + private BitSailConfiguration jobConf; + private Writer.Context> context; + private Map catalogTables; + private TypeInfoValueConverter typeInfoValueConverter; + private MultipleTableWriter multipleTableWriter; + + private CatalogTable catalogTable1; + private CatalogTable catalogTable2; + + @Before + public void before() { + this.multiTablePrintSink = new MultiTablePrintSink(); + this.jobConf = BitSailConfiguration.newDefault(); + this.jobConf.set(WriterOptions.BaseWriterOptions.DB_NAME, "default"); + this.catalogTables = mockCatalogTables(); + this.typeInfoValueConverter = new TypeInfoValueConverter(BitSailConfiguration.newDefault()); + this.context = new Writer.Context>() { + @Override + public RowTypeInfo getRowTypeInfo() { + return null; + } + + @Override + public int getIndexOfSubTaskId() { + return 0; + } + + @Override + public boolean isRestored() { + return false; + } + + @Override + public List> getRestoreStates() { + return null; + } + }; + multipleTableWriter = new MultipleTableWriter<>( + jobConf, + context, + typeInfoValueConverter, + (SupportMultipleSinkTable) multiTablePrintSink, + catalogTables, + Pattern.compile("\\.*"), + new DebeziumRowFilterNamesDeserializationSchema(jobConf)); + } + + private Map mockCatalogTables() { + ConcurrentMap catalogTables = Maps.newConcurrentMap(); + + catalogTable1 = CatalogTable.builder() + .tableId(TableId.of("default", "test1")) + .catalogTableSchema(CatalogTableSchema.builder() + .columns(Lists.newArrayList( + CatalogTableColumn.builder() + .type(TypeInfos.LONG_TYPE_INFO) + .name("int_type") + .build() + )).build() + ).build(); + catalogTable2 = CatalogTable.builder() + .tableId(TableId.of("default", "test2")) + .catalogTableSchema(CatalogTableSchema.builder() + .columns(Lists.newArrayList( + CatalogTableColumn.builder() + .type(TypeInfos.BIG_DECIMAL_TYPE_INFO) + .name("double_type") + .build() + )).build() + ).build(); + + catalogTables.put(catalogTable1.getTableId(), catalogTable1); + catalogTables.put(catalogTable2.getTableId(), catalogTable2); + + return catalogTables; + } + + @Test + public void testMultiTableWriter() throws IOException, URISyntaxException { + for (CatalogTable catalogTable : catalogTables.values()) { + multipleTableWriter.write(mockMultiTableRow(catalogTable)); + } + Map> processedMultiTableWriters = multipleTableWriter.getProcessedMultiTableWriters(); + Assert.assertEquals(processedMultiTableWriters.values().size(), 2); + + validateRowTypeInfo(catalogTable1, multipleTableWriter.getTableIdRowTypeInfos().get(catalogTable1.getTableId())); + validateRowTypeInfo(catalogTable2, multipleTableWriter.getTableIdRowTypeInfos().get(catalogTable2.getTableId())); + } + + public static void validateRowTypeInfo(CatalogTable catalogTable, RowTypeInfo rowTypeInfo) { + Assert.assertNotNull(catalogTable); + Assert.assertNotNull(rowTypeInfo); + + TypeInfo[] fieldTypes = catalogTable.getCatalogTableSchema().getColumns() + .stream() + .map(CatalogTableColumn::getType) + .collect(Collectors.toList()) + .toArray(new TypeInfo[] {}); + + String[] fieldNames = catalogTable.getCatalogTableSchema().getColumns() + .stream() + .map(CatalogTableColumn::getName) + .collect(Collectors.toList()) + .toArray(new String[] {}); + + Assert.assertArrayEquals(fieldTypes, rowTypeInfo.getTypeInfos()); + Assert.assertArrayEquals(fieldNames, rowTypeInfo.getFieldNames()); + } + + private static Row mockMultiTableRow(CatalogTable catalogTable) throws URISyntaxException, IOException { + MultipleTableRow multipleTableRow = MultipleTableRow.of( + catalogTable.getTableId().toString(), + StringUtils.EMPTY, + mockValueJson(), + String.valueOf(Long.MAX_VALUE), + StringUtils.EMPTY + ); + return multipleTableRow.asRow(); + } + + private static String mockValueJson() throws URISyntaxException, IOException { + return new String(Files.readAllBytes(Paths.get(MultipleTableWriterTest.class + .getClassLoader() + .getResource("file/debezium_table1.json") + .toURI() + .getPath()))); + } +} \ No newline at end of file diff --git a/bitsail-cores/bitsail-core-common/src/test/resources/file/debezium_table1.json b/bitsail-cores/bitsail-core-common/src/test/resources/file/debezium_table1.json new file mode 100644 index 000000000..5699f29b4 --- /dev/null +++ b/bitsail-cores/bitsail-core-common/src/test/resources/file/debezium_table1.json @@ -0,0 +1,257 @@ +{ + "schema":{ + "type":"struct", + "fields":[ + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"int_type" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"4", + "connect.decimal.precision":"20" + }, + "field":"double_type" + }, + { + "type":"string", + "optional":true, + "field":"date_type" + }, + { + "type":"string", + "optional":true, + "default":"", + "field":"varchar_type" + }, + { + "type":"int32", + "optional":false, + "name":"io.debezium.time.Date", + "version":1, + "field":"datetime" + } + ], + "optional":true, + "name":"localhost.test.jdbc_source_test.Value", + "field":"before" + }, + { + "type":"struct", + "fields":[ + { + "type":"int64", + "optional":false, + "field":"id" + }, + { + "type":"int32", + "optional":false, + "default":0, + "field":"int_type" + }, + { + "type":"bytes", + "optional":true, + "name":"org.apache.kafka.connect.data.Decimal", + "version":1, + "parameters":{ + "scale":"4", + "connect.decimal.precision":"20" + }, + "field":"double_type" + }, + { + "type":"string", + "optional":true, + "field":"date_type" + }, + { + "type":"string", + "optional":true, + "default":"", + "field":"varchar_type" + }, + { + "type":"int32", + "optional":false, + "name":"io.debezium.time.Date", + "version":1, + "field":"datetime" + } + ], + "optional":true, + "name":"localhost.test.jdbc_source_test.Value", + "field":"after" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"version" + }, + { + "type":"string", + "optional":false, + "field":"connector" + }, + { + "type":"string", + "optional":false, + "field":"name" + }, + { + "type":"int64", + "optional":false, + "field":"ts_ms" + }, + { + "type":"string", + "optional":true, + "name":"io.debezium.data.Enum", + "version":1, + "parameters":{ + "allowed":"true,last,false" + }, + "default":"false", + "field":"snapshot" + }, + { + "type":"string", + "optional":false, + "field":"db" + }, + { + "type":"string", + "optional":true, + "field":"sequence" + }, + { + "type":"string", + "optional":true, + "field":"table" + }, + { + "type":"int64", + "optional":false, + "field":"server_id" + }, + { + "type":"string", + "optional":true, + "field":"gtid" + }, + { + "type":"string", + "optional":false, + "field":"file" + }, + { + "type":"int64", + "optional":false, + "field":"pos" + }, + { + "type":"int32", + "optional":false, + "field":"row" + }, + { + "type":"int64", + "optional":true, + "field":"thread" + }, + { + "type":"string", + "optional":true, + "field":"query" + } + ], + "optional":false, + "name":"io.debezium.connector.mysql.Source", + "field":"source" + }, + { + "type":"string", + "optional":false, + "field":"op" + }, + { + "type":"int64", + "optional":true, + "field":"ts_ms" + }, + { + "type":"struct", + "fields":[ + { + "type":"string", + "optional":false, + "field":"id" + }, + { + "type":"int64", + "optional":false, + "field":"total_order" + }, + { + "type":"int64", + "optional":false, + "field":"data_collection_order" + } + ], + "optional":true, + "field":"transaction" + } + ], + "optional":false, + "name":"localhost.test.jdbc_source_test.Envelope" + }, + "payload":{ + "before":null, + "after":{ + "id":1, + "int_type":1001, + "double_type":"AbH8", + "date_type":"2022-10-01", + "varchar_type":"varchar_01", + "datetime":19297 + }, + "source":{ + "version":"1.6.4.Final", + "connector":"mysql", + "name":"localhost", + "ts_ms":1682237044000, + "snapshot":"false", + "db":"test", + "sequence":null, + "table":"jdbc_source_test", + "server_id":1, + "gtid":null, + "file":"binlog.000002", + "pos":986, + "row":0, + "thread":null, + "query":null + }, + "op":"c", + "ts_ms":1682237048134, + "transaction":null + } +} \ No newline at end of file diff --git a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/pom.xml b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/pom.xml index 2c6f1650f..95ca305fd 100644 --- a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/pom.xml +++ b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/pom.xml @@ -33,6 +33,16 @@ 2.11 + + + + org.reflections + reflections + 0.9.10 + + + + com.bytedance.bitsail diff --git a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/reader/delegate/DelegateFlinkSource.java b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/reader/delegate/DelegateFlinkSource.java index 070e96838..1e1d47a7f 100644 --- a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/reader/delegate/DelegateFlinkSource.java +++ b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/reader/delegate/DelegateFlinkSource.java @@ -18,6 +18,7 @@ import com.bytedance.bitsail.base.connector.reader.v1.SourceSplit; import com.bytedance.bitsail.base.dirty.AbstractDirtyCollector; +import com.bytedance.bitsail.base.extension.SupportProducedType; import com.bytedance.bitsail.base.messenger.Messenger; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.model.ColumnInfo; @@ -62,11 +63,15 @@ public DelegateFlinkSource(com.bytedance.bitsail.base.connector.reader.v1.Source this.source = source; this.commonConfiguration = commonConfiguration; this.readerConfiguration = readerConfiguration; - List columnInfos = readerConfiguration - .get(ReaderOptions.BaseReaderOptions.COLUMNS); - - this.rowTypeInfo = TypeInfoUtils - .getRowTypeInfo(source.createTypeInfoConverter(), columnInfos); + if (source instanceof SupportProducedType) { + this.rowTypeInfo = ((SupportProducedType) source).getProducedType(); + } else { + List columnInfos = readerConfiguration + .get(ReaderOptions.BaseReaderOptions.COLUMNS); + + this.rowTypeInfo = TypeInfoUtils + .getRowTypeInfo(source.createTypeInfoConverter(), columnInfos); + } this.dirtyCollector = dirtyCollector; this.messenger = messenger; } diff --git a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/writer/builder/FlinkWriterBuilder.java b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/writer/builder/FlinkWriterBuilder.java index 48ec8cdc8..becf8555e 100644 --- a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/writer/builder/FlinkWriterBuilder.java +++ b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/java/com/bytedance/bitsail/core/flink/bridge/writer/builder/FlinkWriterBuilder.java @@ -16,8 +16,8 @@ package com.bytedance.bitsail.core.flink.bridge.writer.builder; +import com.bytedance.bitsail.base.catalog.CatalogFactoryHelper; import com.bytedance.bitsail.base.connector.writer.v1.Sink; -import com.bytedance.bitsail.base.connector.writer.v1.WriterCommitter; import com.bytedance.bitsail.base.connector.writer.v1.comittable.CommittableMessage; import com.bytedance.bitsail.base.dirty.AbstractDirtyCollector; import com.bytedance.bitsail.base.dirty.DirtyCollectorFactory; @@ -25,6 +25,7 @@ import com.bytedance.bitsail.base.execution.Mode; import com.bytedance.bitsail.base.execution.ProcessResult; import com.bytedance.bitsail.base.extension.GlobalCommittable; +import com.bytedance.bitsail.base.extension.SupportMultipleSinkTable; import com.bytedance.bitsail.base.extension.TypeInfoConverterFactory; import com.bytedance.bitsail.base.messenger.Messenger; import com.bytedance.bitsail.base.messenger.checker.DirtyRecordChecker; @@ -34,7 +35,9 @@ import com.bytedance.bitsail.base.ratelimit.Channel; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.option.CommonOptions; +import com.bytedance.bitsail.common.option.WriterOptions; import com.bytedance.bitsail.common.type.TypeInfoConverter; +import com.bytedance.bitsail.core.common.sink.multiple.MultipleTableSink; import com.bytedance.bitsail.core.flink.bridge.writer.delegate.DelegateFlinkCommitter; import com.bytedance.bitsail.core.flink.bridge.writer.delegate.DelegateFlinkWriter; import com.bytedance.bitsail.flink.core.execution.FlinkExecutionEnviron; @@ -48,6 +51,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,7 +66,7 @@ public class FlinkWriterBuilder sink; + private Sink sink; private boolean isBatchMode; @@ -89,6 +93,16 @@ public void configure(ExecutionEnviron execution, BitSailConfiguration writerCon this.commonConfiguration = execution.getCommonConfiguration(); this.writerConfiguration = writerConfiguration; + if (writerConfiguration.get(WriterOptions.BaseWriterOptions.MULTIPLE_TABLE_ENABLED)) { + if (sink instanceof SupportMultipleSinkTable) { + sink = new MultipleTableSink<>(sink, CatalogFactoryHelper + .getTableCatalogFactory(sink.getWriterName())); + } else { + LOG.info("Sink {} must implement interface SupportMultipleSinkTable when enabled option {}.", + sink.getWriterName(), + WriterOptions.BaseWriterOptions.MULTIPLE_TABLE_ENABLED.key()); + } + } sink.configure(execution.getCommonConfiguration(), writerConfiguration); this.messengerContext = SimpleMessengerContext.builder() @@ -111,7 +125,7 @@ public void addWriter(DataStream source, int writerParallelism) { .getCheckpointConfig() .isCheckpointingEnabled(); - DelegateFlinkWriter flinkWriter = new DelegateFlinkWriter<>( + DelegateFlinkWriter flinkWriter = new DelegateFlinkWriter<>( commonConfiguration, writerConfiguration, sink, @@ -122,22 +136,22 @@ public void addWriter(DataStream source, int writerParallelism) { flinkWriter.setDirtyCollector(dirtyCollector); flinkWriter.setChannel(channel); - DataStream> writeStream = source.transform(getWriterOperatorName(), - TypeInformation.of(new TypeHint>() { - }), flinkWriter) + DataStream> writeStream = source.transform(getWriterOperatorName(), + (TypeInformation) TypeInformation.of(new TypeHint>() { + }), (OneInputStreamOperator) flinkWriter) .setParallelism(writerParallelism) .name(getWriterOperatorName()) .uid(getWriterOperatorName()); - Optional> committer = sink.createCommitter(); + Optional committer = sink.createCommitter(); if (committer.isPresent()) { LOG.info("Writer enabled committer."); - DataStream> commitStream = writeStream + DataStream> commitStream = writeStream .transform( getWriterCommitterOperatorName(), - TypeInformation.of(new TypeHint>() { + (TypeInformation) TypeInformation.of(new TypeHint>() { }), - new DelegateFlinkCommitter<>(sink, isBatchMode, isCheckpointingEnabled)) + (OneInputStreamOperator) new DelegateFlinkCommitter<>(sink, isBatchMode, isCheckpointingEnabled)) .uid(getWriterCommitterOperatorName()) .name(getWriterCommitterOperatorName()) .setParallelism(writerParallelism); diff --git a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.16-bridge/pom.xml b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.16-bridge/pom.xml index 9b67f455d..d29ddc0c7 100644 --- a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.16-bridge/pom.xml +++ b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.16-bridge/pom.xml @@ -123,6 +123,27 @@ + + org.apache.flink + flink-runtime-web + ${flink.version} + provided + + + org.yaml + snakeyaml + + + org.apache.commons + commons-pool2 + + + org.eclipse.jetty + jetty-http + + + + org.apache.flink flink-table-api-scala-bridge_${scala.binary.version} diff --git a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-base/src/main/java/com/bytedance/bitsail/flink/core/execution/configurer/FlinkDAGBuilderInterceptor.java b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-base/src/main/java/com/bytedance/bitsail/flink/core/execution/configurer/FlinkDAGBuilderInterceptor.java index 5d5728380..a166fcbdc 100644 --- a/bitsail-cores/bitsail-core-flink/bitsail-core-flink-base/src/main/java/com/bytedance/bitsail/flink/core/execution/configurer/FlinkDAGBuilderInterceptor.java +++ b/bitsail-cores/bitsail-core-flink/bitsail-core-flink-base/src/main/java/com/bytedance/bitsail/flink/core/execution/configurer/FlinkDAGBuilderInterceptor.java @@ -16,8 +16,8 @@ package com.bytedance.bitsail.flink.core.execution.configurer; -import com.bytedance.bitsail.base.catalog.TableCatalogFactory; -import com.bytedance.bitsail.base.catalog.TableCatalogFactoryHelper; +import com.bytedance.bitsail.base.catalog.CatalogFactory; +import com.bytedance.bitsail.base.catalog.CatalogFactoryHelper; import com.bytedance.bitsail.base.connector.BuilderGroup; import com.bytedance.bitsail.base.connector.reader.DataReaderDAGBuilder; import com.bytedance.bitsail.base.connector.transform.DataTransformDAGBuilder; @@ -69,10 +69,10 @@ private void alignTableCatalog(List readerBuilders, DataReaderDAGBuilder dataReaderDAGBuilder = readerBuilders.get(0); DataWriterDAGBuilder dataWriterDAGBuilder = writerBuilders.get(0); - TableCatalogFactory readerCatalogFactory = TableCatalogFactoryHelper + CatalogFactory readerCatalogFactory = CatalogFactoryHelper .getTableCatalogFactory(dataReaderDAGBuilder.getReaderName()); - TableCatalogFactory writerCatalogFactory = TableCatalogFactoryHelper + CatalogFactory writerCatalogFactory = CatalogFactoryHelper .getTableCatalogFactory(dataWriterDAGBuilder.getWriterName()); if (Objects.isNull(readerCatalogFactory) || Objects.isNull(writerCatalogFactory)) { @@ -100,8 +100,8 @@ private void alignTableCatalog(List readerBuilders, BitSailConfiguration writerConfiguration = executionEnviron.getWriterConfigurations().get(0); TableCatalogManager catalogManager = TableCatalogManager.builder() - .readerTableCatalog(readerCatalogFactory.createTableCatalog(BuilderGroup.READER, executionEnviron, readerConfiguration)) - .writerTableCatalog(writerCatalogFactory.createTableCatalog(BuilderGroup.WRITER, executionEnviron, writerConfiguration)) + .readerCatalog(readerCatalogFactory.createTableCatalog(BuilderGroup.READER, readerConfiguration)) + .writerCatalog(writerCatalogFactory.createTableCatalog(BuilderGroup.WRITER, writerConfiguration)) .readerTypeInfoConverter(readerTypeInfoConverter) .writerTypeInfoConverter(writerTypeInfoConverter) .commonConfiguration(executionEnviron.getCommonConfiguration()) diff --git a/bitsail-cores/bitsail-core-flink/pom.xml b/bitsail-cores/bitsail-core-flink/pom.xml index bc8ab00a6..7a562acf6 100644 --- a/bitsail-cores/bitsail-core-flink/pom.xml +++ b/bitsail-cores/bitsail-core-flink/pom.xml @@ -226,6 +226,14 @@ + + + com.bytedance.bitsail + bitsail-core-common + ${revision} + + + diff --git a/bitsail-cores/pom.xml b/bitsail-cores/pom.xml index 04e53e04a..86c245e29 100644 --- a/bitsail-cores/pom.xml +++ b/bitsail-cores/pom.xml @@ -34,6 +34,7 @@ bitsail-core-api bitsail-core-entry + bitsail-core-common bitsail-core-flink \ No newline at end of file diff --git a/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/java/com/bytedance/bitsail/test/integration/cdc/mysql/MysqlBinlogSplitReaderContainerITCase.java b/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/java/com/bytedance/bitsail/test/integration/cdc/mysql/MysqlBinlogSplitReaderContainerITCase.java index 2f5742cb1..8b6a531d9 100644 --- a/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/java/com/bytedance/bitsail/test/integration/cdc/mysql/MysqlBinlogSplitReaderContainerITCase.java +++ b/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/java/com/bytedance/bitsail/test/integration/cdc/mysql/MysqlBinlogSplitReaderContainerITCase.java @@ -16,7 +16,10 @@ package com.bytedance.bitsail.test.integration.cdc.mysql; +import com.bytedance.bitsail.base.connector.reader.v1.SourceReader; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; +import com.bytedance.bitsail.component.format.debezium.DebeziumDeserializationSchema; +import com.bytedance.bitsail.component.format.debezium.DebeziumJsonDeserializationSchema; import com.bytedance.bitsail.connector.cdc.model.ClusterInfo; import com.bytedance.bitsail.connector.cdc.model.ConnectionInfo; import com.bytedance.bitsail.connector.cdc.mysql.source.debezium.MysqlBinlogSplitReader; @@ -25,6 +28,7 @@ import com.bytedance.bitsail.connector.cdc.source.split.BinlogSplit; import com.bytedance.bitsail.test.integration.cdc.mysql.container.MySQLContainerMariadbAdapter; import com.bytedance.bitsail.test.integration.cdc.mysql.container.util.TestDatabase; +import com.bytedance.bitsail.test.integration.cdc.mysql.context.SourceMockContext; import com.google.common.collect.Lists; import org.junit.After; @@ -50,6 +54,8 @@ public class MysqlBinlogSplitReaderContainerITCase { private static final String TEST_DATABASE = "test"; private MySQLContainer container; + private DebeziumDeserializationSchema deserializationSchema; + private SourceReader.Context context; @Before public void before() { @@ -63,6 +69,8 @@ public void before() { .withLogConsumer(new Slf4jLogConsumer(LOG)); //container.addParameter("MY_CNF", "container/my.cnf"); + deserializationSchema = new DebeziumJsonDeserializationSchema(BitSailConfiguration.newDefault()); + context = new SourceMockContext(0, deserializationSchema.getProducedType()); Startables.deepStart(Stream.of(container)).join(); } @@ -101,7 +109,7 @@ public void testBinlogReader() throws InterruptedException { jobConf.set("job.reader.debezium.schema.history.internal", "io.debezium.relational.history.MemorySchemaHistory"); jobConf.set("job.reader.debezium.database.history", "io.debezium.relational.history.MemoryDatabaseHistory"); - MysqlBinlogSplitReader reader = new MysqlBinlogSplitReader(jobConf, 0); + MysqlBinlogSplitReader reader = new MysqlBinlogSplitReader(jobConf, context, deserializationSchema); BinlogSplit split = new BinlogSplit("split-1", BinlogOffset.earliest(), BinlogOffset.boundless()); reader.readSplit(split); int maxPeriod = 0; diff --git a/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/java/com/bytedance/bitsail/test/integration/cdc/mysql/context/SourceMockContext.java b/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/java/com/bytedance/bitsail/test/integration/cdc/mysql/context/SourceMockContext.java new file mode 100644 index 000000000..23e87a729 --- /dev/null +++ b/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/java/com/bytedance/bitsail/test/integration/cdc/mysql/context/SourceMockContext.java @@ -0,0 +1,46 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.test.integration.cdc.mysql.context; + +import com.bytedance.bitsail.base.connector.reader.v1.SourceReader; +import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; + +public class SourceMockContext implements SourceReader.Context { + + private final int id; + private final RowTypeInfo rowTypeInfo; + + public SourceMockContext(int id, RowTypeInfo rowTypeInfo) { + this.id = id; + this.rowTypeInfo = rowTypeInfo; + } + + @Override + public RowTypeInfo getRowTypeInfo() { + return rowTypeInfo; + } + + @Override + public int getIndexOfSubtask() { + return id; + } + + @Override + public void sendSplitRequest() { + + } +} diff --git a/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/resources/bitsail_mysql_cdc_kafka.json b/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/resources/bitsail_mysql_cdc_kafka.json index 73e9e8b28..2965f5629 100644 --- a/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/resources/bitsail_mysql_cdc_kafka.json +++ b/bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/resources/bitsail_mysql_cdc_kafka.json @@ -53,7 +53,7 @@ }, "writer": { "class": "com.bytedance.bitsail.connector.kafka.sink.KafkaSink", - "content_type": "debezium", + "content_type": "debezium-json", "columns": [ { "name": "db", diff --git a/bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-jdbc-legacy/src/test/java/com/bytedance/bitsail/test/integration/legacy/mysql/MysqlConnectorITCase.java b/bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-jdbc-legacy/src/test/java/com/bytedance/bitsail/test/integration/legacy/mysql/MysqlConnectorITCase.java index 4e99daa79..a558f3864 100644 --- a/bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-jdbc-legacy/src/test/java/com/bytedance/bitsail/test/integration/legacy/mysql/MysqlConnectorITCase.java +++ b/bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-jdbc-legacy/src/test/java/com/bytedance/bitsail/test/integration/legacy/mysql/MysqlConnectorITCase.java @@ -17,9 +17,9 @@ package com.bytedance.bitsail.test.integration.legacy.mysql; import com.bytedance.bitsail.common.catalog.table.CatalogTable; -import com.bytedance.bitsail.common.catalog.table.CatalogTableDefinition; +import com.bytedance.bitsail.common.catalog.table.TableId; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; -import com.bytedance.bitsail.connector.legacy.jdbc.catalog.MySQLTableCatalog; +import com.bytedance.bitsail.connector.legacy.jdbc.catalog.MySQLCatalog; import com.bytedance.bitsail.connector.legacy.jdbc.converter.JdbcTypeInfoConverter; import com.bytedance.bitsail.connector.legacy.jdbc.model.ClusterInfo; import com.bytedance.bitsail.connector.legacy.jdbc.model.ConnectionInfo; @@ -110,7 +110,7 @@ private void mysqlReader(String filePath) throws Exception { @Test public void testGetCatalogTable() { - MySQLTableCatalog catalog = MySQLTableCatalog + MySQLCatalog catalog = MySQLCatalog .builder() .url(mySQLContainer.getJdbcUrl()) .table(TABLE) @@ -121,7 +121,7 @@ public void testGetCatalogTable() { catalog.open(new JdbcTypeInfoConverter("mysql")); - CatalogTableDefinition catalogTableDefinition = catalog.createCatalogTableDefinition(); + TableId catalogTableDefinition = catalog.createCatalogTableDefinition(); CatalogTable catalogTable = catalog.getCatalogTable(catalogTableDefinition); Assert.assertNotNull(catalogTable.getCatalogTableSchema());