argList) {
+ // set default doris sink config
+ argList.add(SINK_CONF);
+ argList.add(FENODES + "=" + getFenodes());
+ argList.add(SINK_CONF);
+ argList.add(USERNAME + "=" + getDorisUsername());
+ argList.add(SINK_CONF);
+ argList.add(PASSWORD + "=" + getDorisPassword());
+ argList.add(SINK_CONF);
+ argList.add(FENODES + "=" + getFenodes());
+ argList.add(SINK_CONF);
+ argList.add(JDBC_URL + "=" + getDorisQueryUrl());
+ argList.add(SINK_CONF);
+ argList.add(SINK_LABEL_PREFIX + "=" + "label");
+ }
+
+ @AfterClass
+ public static void closeE2EContainers() {
+ LOG.info("Starting to close E2E containers.");
+ closeMySQLContainer();
+ }
+
+ private static void closeMySQLContainer() {
+ if (Objects.isNull(mysqlContainerService)) {
+ return;
+ }
+ singleThreadExecutor.shutdown();
+ mysqlContainerService.close();
+ LOG.info("Mysql container was closed.");
+ }
+}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/CustomerSingleThreadExecutor.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/CustomerSingleThreadExecutor.java
new file mode 100644
index 000000000..066d9a44b
--- /dev/null
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/CustomerSingleThreadExecutor.java
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.autoci.e2e;
+
+import org.apache.doris.flink.exception.DorisRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A thread pool executor that ensures only one job is executed at a time.
+ *
+ * If a job is running, new jobs will wait in the queue until the current job is completed.
+ */
+public class CustomerSingleThreadExecutor {
+ private static final Logger LOG = LoggerFactory.getLogger(CustomerSingleThreadExecutor.class);
+ private final ThreadPoolExecutor executor;
+ private final long timeoutMillis = TimeUnit.HOURS.toMillis(1);
+ private String currentJobName;
+ private Future currentJob;
+
+ public CustomerSingleThreadExecutor() {
+ this.executor =
+ new ThreadPoolExecutor(
+ 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
+ }
+
+ /**
+ * Submits a job to the executor. If there is already a job running, the new job will wait in
+ * the queue until the current job is completed or the current job times out.
+ *
+ * @param jobName The name of the job to be submitted.
+ * @param job The Runnable representing the job.
+ * @return A Future representing the pending completion of the job.
+ */
+ public synchronized Future submitJob(String jobName, Runnable job) {
+ // Wait for the current task to complete, with a timeout of 1 hour
+ long startTime = System.currentTimeMillis();
+ while (currentJob != null && (!currentJob.isDone() || !currentJob.isCancelled())) {
+ try {
+ long elapsed = System.currentTimeMillis() - startTime;
+ long remainingTime = timeoutMillis - elapsed;
+
+ if (remainingTime <= 0) {
+ LOG.warn(
+ "Current job exceeded the maximum timeout of 1 hour and will be canceled. jobName={}",
+ currentJobName);
+ cancelCurrentJob(currentJobName);
+ break;
+ }
+
+ wait(remainingTime);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new DorisRuntimeException(
+ "Thread was interrupted while waiting for the current job to complete.", e);
+ }
+ }
+
+ LOG.info("Submitting a new job. jobName={}", jobName);
+ currentJob =
+ executor.submit(
+ () -> {
+ try {
+ job.run();
+ } finally {
+ synchronized (this) {
+ // Only notify when the job has been cancelled
+ if (currentJob.isCancelled() || currentJob.isDone()) {
+ notifyAll();
+ }
+ }
+ }
+ });
+ currentJobName = jobName;
+ return currentJob;
+ }
+
+ /**
+ * Cancels the currently running job if its name matches the provided job name. The job is
+ * interrupted if it is currently running.
+ *
+ * @param jobName The name of the job to be cancelled.
+ */
+ public synchronized void cancelCurrentJob(String jobName) {
+ if (currentJob != null && !currentJob.isDone() && currentJobName.equals(jobName)) {
+ currentJob.cancel(true);
+ }
+ }
+
+ /**
+ * Shuts down the executor, preventing any new jobs from being submitted. Already submitted jobs
+ * in the queue will be executed before the shutdown.
+ */
+ public void shutdown() {
+ executor.shutdown();
+ }
+}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisDorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
similarity index 52%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisDorisE2ECase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
index 828135060..35e8683a3 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisDorisE2ECase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.tools.cdc;
+package org.apache.doris.flink.autoci.e2e;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -24,27 +24,25 @@
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
-import org.apache.doris.flink.DorisTestBase;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-/** DorisDorisE2ECase. */
-public class DorisDorisE2ECase extends DorisTestBase {
+public class Doris2DorisService extends AbstractE2EService {
+ private static final Logger LOG = LoggerFactory.getLogger(Doris2DorisService.class);
private static final String DATABASE_SOURCE = "test_e2e_source";
private static final String DATABASE_SINK = "test_e2e_sink";
private static final String TABLE = "test_tbl";
@Test
public void testDoris2Doris() throws Exception {
- initializeDorisTable(TABLE);
- printClusterStatus();
+ LOG.info("Start executing the test case of doris to doris.");
+ initializeDorisTable();
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
@@ -83,8 +81,8 @@ public void testDoris2Doris() throws Exception {
+ ")",
getFenodes(),
DATABASE_SOURCE + "." + TABLE,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sourceDDL);
String sinkDDL =
@@ -133,7 +131,8 @@ public void testDoris2Doris() throws Exception {
actual.add(iterator.next().toString());
}
}
- System.out.println(actual);
+ LOG.info("The actual data in the doris sink table is, actual={}", actual);
+
String[] expected =
new String[] {
"+I[1, true, 127, 32767, 2147483647, 9223372036854775807, 123456789012345678901234567890, 3.14, 2.7182818284, 12345.6789, 2023-05-22, 2023-05-22T12:34:56, A, Example text, [item1, item2, item3], {key1=value1, key2=value2}, +I[John Doe, 30], {\"key\":\"value\"}]",
@@ -142,93 +141,15 @@ public void testDoris2Doris() throws Exception {
Assert.assertArrayEquals(expected, actual.toArray(new String[0]));
}
- private void initializeDorisTable(String table) throws Exception {
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE_SOURCE));
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE_SINK));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE_SOURCE, table));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE_SINK, table));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s (\n"
- + " `id` int,\n"
- + " `c1` boolean,\n"
- + " `c2` tinyint,\n"
- + " `c3` smallint,\n"
- + " `c4` int,\n"
- + " `c5` bigint,\n"
- + " `c6` largeint,\n"
- + " `c7` float,\n"
- + " `c8` double,\n"
- + " `c9` decimal(12,4),\n"
- + " `c10` date,\n"
- + " `c11` datetime,\n"
- + " `c12` char(1),\n"
- + " `c13` varchar(256),\n"
- + " `c14` Array,\n"
- + " `c15` Map,\n"
- + " `c16` Struct,\n"
- + " `c17` JSON\n"
- + " )\n"
- + "DUPLICATE KEY(`id`)\n"
- + "DISTRIBUTED BY HASH(`id`) BUCKETS 1\n"
- + "PROPERTIES (\n"
- + "\"replication_num\" = \"1\",\n"
- + "\"light_schema_change\" = \"true\"\n"
- + ");",
- DATABASE_SOURCE, table));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s like %s.%s",
- DATABASE_SINK, table, DATABASE_SOURCE, table));
- statement.execute(
- String.format(
- "INSERT INTO %s.%s \n"
- + "VALUES \n"
- + "(\n"
- + " 1, \n"
- + " TRUE, \n"
- + " 127, \n"
- + " 32767, \n"
- + " 2147483647, \n"
- + " 9223372036854775807, \n"
- + " 123456789012345678901234567890, \n"
- + " 3.14, \n"
- + " 2.7182818284, \n"
- + " 12345.6789, \n"
- + " '2023-05-22', \n"
- + " '2023-05-22 12:34:56', \n"
- + " 'A', \n"
- + " 'Example text', \n"
- + " ['item1', 'item2', 'item3'], \n"
- + " {'key1': 'value1', 'key2': 'value2'}, \n"
- + " STRUCT('John Doe', 30), \n"
- + " '{\"key\": \"value\"}' \n"
- + "),\n"
- + "(\n"
- + " 2,\n"
- + " FALSE,\n"
- + " -128,\n"
- + " -32768,\n"
- + " -2147483648,\n"
- + " -9223372036854775808,\n"
- + " -123456789012345678901234567890,\n"
- + " -3.14,\n"
- + " -2.7182818284,\n"
- + " -12345.6789,\n"
- + " '2024-01-01',\n"
- + " '2024-01-01 00:00:00',\n"
- + " 'B',\n"
- + " 'Another example',\n"
- + " ['item4', 'item5', 'item6'],\n"
- + " {'key3': 'value3', 'key4': 'value4'},\n"
- + " STRUCT('Jane Doe', 25),\n"
- + " '{\"another_key\": \"another_value\"}'\n"
- + ")",
- DATABASE_SOURCE, table));
- }
+ private void initializeDorisTable() {
+ String[] sourceInitSql =
+ E2EContainerUtils.parseFileContentSQL(
+ "autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql");
+ E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sourceInitSql);
+ String[] sinkInitSql =
+ E2EContainerUtils.parseFileContentSQL(
+ "autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql");
+ E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sinkInitSql);
+ LOG.info("Initialization of doris table successful.");
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
new file mode 100644
index 000000000..14108fbb1
--- /dev/null
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.autoci.e2e;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.doris.flink.exception.DorisRuntimeException;
+import org.junit.Assert;
+import org.slf4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class E2EContainerUtils {
+
+ protected static void executeSQLStatement(Connection connection, Logger logger, String... sql) {
+ try (Statement statement = connection.createStatement()) {
+ for (String s : sql) {
+ logger.info("start to execute sql={}", s);
+ statement.execute(s);
+ }
+ } catch (SQLException e) {
+ throw new DorisRuntimeException(e);
+ }
+ }
+
+ protected static String loadFileContent(String resourcePath) {
+ try (InputStream stream =
+ E2EContainerUtils.class.getClassLoader().getResourceAsStream(resourcePath)) {
+ return new BufferedReader(new InputStreamReader(Objects.requireNonNull(stream)))
+ .lines()
+ .collect(Collectors.joining("\n"));
+ } catch (IOException e) {
+ throw new DorisRuntimeException("Failed to read " + resourcePath + " file.", e);
+ }
+ }
+
+ protected static List parseFileArgs(String resourcePath) {
+ String fileContent = E2EContainerUtils.loadFileContent(resourcePath);
+ String[] args = fileContent.split("\n");
+ List argList = new ArrayList<>();
+ for (String arg : args) {
+ String[] split = arg.trim().split("\\s+");
+ List stringList =
+ Arrays.stream(split)
+ .map(E2EContainerUtils::removeQuotes)
+ .collect(Collectors.toList());
+ argList.addAll(stringList);
+ }
+ return argList;
+ }
+
+ private static String removeQuotes(String str) {
+ if (str == null || str.length() < 2) {
+ return str;
+ }
+ if (str.startsWith("\"") && str.endsWith("\"")) {
+ return str.substring(1, str.length() - 1);
+ }
+ if (str.startsWith("\\'") && str.endsWith("\\'")) {
+ return str.substring(1, str.length() - 1);
+ }
+ return str;
+ }
+
+ protected static String[] parseFileContentSQL(String resourcePath) {
+ String fileContent = loadFileContent(resourcePath);
+ return Arrays.stream(fileContent.split(";")).map(String::trim).toArray(String[]::new);
+ }
+
+ public static void checkResult(
+ Connection connection, List expected, String query, int columnSize) {
+ List actual = new ArrayList<>();
+ try (Statement statement = connection.createStatement()) {
+ ResultSet sinkResultSet = statement.executeQuery(query);
+ while (sinkResultSet.next()) {
+ List row = new ArrayList<>();
+ for (int i = 1; i <= columnSize; i++) {
+ Object value = sinkResultSet.getObject(i);
+ if (value == null) {
+ row.add("null");
+ } else {
+ row.add(value.toString());
+ }
+ }
+ actual.add(StringUtils.join(row, ","));
+ }
+ } catch (SQLException e) {
+ throw new DorisRuntimeException(e);
+ }
+ Assert.assertArrayEquals(expected.toArray(), actual.toArray());
+ }
+}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java
new file mode 100644
index 000000000..c28d5ce69
--- /dev/null
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.autoci.e2e;
+
+import org.apache.doris.flink.exception.DorisRuntimeException;
+import org.apache.doris.flink.tools.cdc.DatabaseSyncConfig;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class Mysql2DorisService extends AbstractE2EService {
+ private static final Logger LOG = LoggerFactory.getLogger(Mysql2DorisService.class);
+ private static final String DATABASE = "test_e2e_mysql";
+ private static final String CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS " + DATABASE;
+ private static final String MYSQL_CONF = "--" + DatabaseSyncConfig.MYSQL_CONF;
+
+ private void initialize() {
+ // init mysql table
+ LOG.info("start to init mysql table.");
+ E2EContainerUtils.executeSQLStatement(getMySQLQueryConnection(), LOG, CREATE_DATABASE);
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ E2EContainerUtils.parseFileContentSQL(
+ "autoci/e2e/mysql2doris/initialize/mysql_tbl1.sql"));
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ E2EContainerUtils.parseFileContentSQL(
+ "autoci/e2e/mysql2doris/initialize/mysql_tbl2.sql"));
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ E2EContainerUtils.parseFileContentSQL(
+ "autoci/e2e/mysql2doris/initialize/mysql_tbl3.sql"));
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ E2EContainerUtils.parseFileContentSQL(
+ "autoci/e2e/mysql2doris/initialize/mysql_tbl4.sql"));
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ E2EContainerUtils.parseFileContentSQL(
+ "autoci/e2e/mysql2doris/initialize/mysql_tbl5.sql"));
+
+ // init doris table
+ LOG.info("start to init doris table.");
+ E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, CREATE_DATABASE);
+ E2EContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ "DROP TABLE IF EXISTS test_e2e_mysql.tbl1",
+ "DROP TABLE IF EXISTS test_e2e_mysql.tbl2",
+ "DROP TABLE IF EXISTS test_e2e_mysql.tbl3",
+ "DROP TABLE IF EXISTS test_e2e_mysql.tbl4",
+ "DROP TABLE IF EXISTS test_e2e_mysql.tbl5");
+ }
+
+ private List setMysql2DorisDefaultConfig(List argList) {
+ // set default mysql config
+ argList.add(MYSQL_CONF);
+ argList.add(HOSTNAME + "=" + getMySQLInstanceHost());
+ argList.add(MYSQL_CONF);
+ argList.add(PORT + "=" + getMySQLQueryPort());
+ argList.add(MYSQL_CONF);
+ argList.add(USERNAME + "=" + getMySQLUsername());
+ argList.add(MYSQL_CONF);
+ argList.add(PASSWORD + "=" + getMySQLPassword());
+ argList.add(MYSQL_CONF);
+ argList.add(DATABASE_NAME + "=" + DATABASE);
+
+ // set doris database
+ argList.add(DORIS_DATABASE);
+ argList.add(DATABASE);
+ setSinkConfDefaultConfig(argList);
+ return argList;
+ }
+
+ private void startMysql2DorisJob(String jobName, String resourcePath) {
+ LOG.info("start a mysql to doris job. jobName={}, resourcePath={}", jobName, resourcePath);
+ List argList = E2EContainerUtils.parseFileArgs(resourcePath);
+ String[] args = setMysql2DorisDefaultConfig(argList).toArray(new String[0]);
+ submitE2EJob(jobName, args);
+ verifyInitializeResult();
+ }
+
+ private void verifyInitializeResult() {
+ // wait 2 times checkpoint
+ try {
+ Thread.sleep(20000);
+ LOG.info("Start to verify init result.");
+ List expected =
+ Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3", "doris_5,5");
+ String sql1 =
+ "select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.tbl5) res order by 1";
+ E2EContainerUtils.checkResult(getDorisQueryConnection(), expected, sql1, 2);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testMySQL2Doris() throws Exception {
+ initialize();
+ String jobName = "testMySQL2Doris";
+ String resourcePath = "autoci/e2e/mysql2doris/testMySQL2Doris.txt";
+ startMysql2DorisJob(jobName, resourcePath);
+
+ addIncrementalData();
+ verifyIncrementalDataResult();
+
+ tbl1SchemaChange();
+ verifyTbl1SchemaChange();
+ cancelCurrentE2EJob(jobName);
+ }
+
+ @Test
+ public void testAutoAddTable() throws InterruptedException {
+ initialize();
+ String jobName = "testAutoAddTable";
+ startMysql2DorisJob(jobName, "autoci/e2e/mysql2doris/testAutoAddTable.txt");
+
+ // auto add table
+ LOG.info("starting to create auto_add table.");
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "CREATE TABLE test_e2e_mysql.auto_add ( \n"
+ + "`name` varchar(256) primary key,\n"
+ + "`age` int\n"
+ + ")",
+ "insert into test_e2e_mysql.auto_add values ('doris_4_1',4)",
+ "insert into test_e2e_mysql.auto_add values ('doris_4_2',4)");
+ Thread.sleep(20000);
+ List autoAddResult = Arrays.asList("doris_4_1,4", "doris_4_2,4");
+ String autoAddSql = "select * from test_e2e_mysql.auto_add order by 1";
+ E2EContainerUtils.checkResult(getDorisQueryConnection(), autoAddResult, autoAddSql, 2);
+
+ // incremental data
+ LOG.info("starting to increment data.");
+ addIncrementalData();
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "insert into test_e2e_mysql.auto_add values ('doris_4_3',43)",
+ "delete from test_e2e_mysql.auto_add where name='doris_4_2'",
+ "update test_e2e_mysql.auto_add set age=41 where name='doris_4_1'");
+ Thread.sleep(20000);
+ List incrementDataExpected =
+ Arrays.asList(
+ "doris_1,18",
+ "doris_1_1,10",
+ "doris_2_1,11",
+ "doris_3,3",
+ "doris_3_1,12",
+ "doris_4_1,41",
+ "doris_4_3,43");
+ String incrementDataSql =
+ "select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.auto_add) res order by 1";
+ E2EContainerUtils.checkResult(
+ getDorisQueryConnection(), incrementDataExpected, incrementDataSql, 2);
+
+ // schema change
+ LOG.info("starting to mock schema change.");
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "alter table test_e2e_mysql.auto_add add column c1 varchar(128)",
+ "alter table test_e2e_mysql.auto_add drop column age",
+ "insert into test_e2e_mysql.auto_add values ('doris_4_4','c1_val')");
+ Thread.sleep(20000);
+ List schemaChangeExpected =
+ Arrays.asList("doris_4_1,null", "doris_4_3,null", "doris_4_4,c1_val");
+ String schemaChangeSql = "select * from test_e2e_mysql.auto_add order by 1";
+ E2EContainerUtils.checkResult(
+ getDorisQueryConnection(), schemaChangeExpected, schemaChangeSql, 2);
+ cancelCurrentE2EJob(jobName);
+ }
+
+ @Test
+ public void testMySQL2DorisSQLParse() throws Exception {
+ initialize();
+ String jobName = "testMySQL2DorisSQLParse";
+ String resourcePath = "autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt";
+ startMysql2DorisJob(jobName, resourcePath);
+
+ addIncrementalData();
+ verifyIncrementalDataResult();
+
+ tbl1SchemaChange();
+ verifyTbl1SchemaChange();
+
+ // mock create table
+ LOG.info("start to create table in mysql.");
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "CREATE TABLE test_e2e_mysql.add_tbl (\n"
+ + " `name` varchar(256) primary key,\n"
+ + " `age` int\n"
+ + ");",
+ "insert into test_e2e_mysql.add_tbl values ('doris_1',1)",
+ "insert into test_e2e_mysql.add_tbl values ('doris_2',2)",
+ "insert into test_e2e_mysql.add_tbl values ('doris_3',3)");
+ Thread.sleep(20000);
+ List createTableExpected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3");
+ String createTableSql = "select * from test_e2e_mysql.add_tbl order by 1";
+ E2EContainerUtils.checkResult(
+ getDorisQueryConnection(), createTableExpected, createTableSql, 2);
+ cancelCurrentE2EJob(jobName);
+ }
+
+ private void tbl1SchemaChange() {
+ // mock schema change
+ LOG.info("start to schema change in mysql.");
+ try {
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "alter table test_e2e_mysql.tbl1 add column c1 varchar(128)",
+ "alter table test_e2e_mysql.tbl1 drop column age");
+ Thread.sleep(20000);
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "insert into test_e2e_mysql.tbl1 values ('doris_1_1_1','c1_val')");
+ Thread.sleep(20000);
+ } catch (InterruptedException e) {
+ throw new DorisRuntimeException(e);
+ }
+ }
+
+ private void verifyTbl1SchemaChange() {
+ LOG.info("verify tal1 schema change.");
+ List schemaChangeExpected =
+ Arrays.asList("doris_1,null", "doris_1_1,null", "doris_1_1_1,c1_val");
+ String schemaChangeSql = "select * from test_e2e_mysql.tbl1 order by 1";
+ E2EContainerUtils.checkResult(
+ getDorisQueryConnection(), schemaChangeExpected, schemaChangeSql, 2);
+ }
+
+ private void addIncrementalData() {
+ // add incremental data
+ try {
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "insert into test_e2e_mysql.tbl1 values ('doris_1_1',10)",
+ "insert into test_e2e_mysql.tbl2 values ('doris_2_1',11)",
+ "insert into test_e2e_mysql.tbl3 values ('doris_3_1',12)",
+ "update test_e2e_mysql.tbl1 set age=18 where name='doris_1'",
+ "delete from test_e2e_mysql.tbl2 where name='doris_2'");
+ Thread.sleep(20000);
+ } catch (InterruptedException e) {
+ throw new DorisRuntimeException(e);
+ }
+ }
+
+ private void verifyIncrementalDataResult() {
+ LOG.info("Start to verify incremental data result.");
+ List expected2 =
+ Arrays.asList(
+ "doris_1,18", "doris_1_1,10", "doris_2_1,11", "doris_3,3", "doris_3_1,12");
+ String sql2 =
+ "select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 ) res order by 1";
+ E2EContainerUtils.checkResult(getDorisQueryConnection(), expected2, sql2, 2);
+ }
+
+ @Test
+ public void testMySQL2DorisByDefault() throws InterruptedException {
+ initialize();
+ String jobName = "testMySQL2DorisByDefault";
+ startMysql2DorisJob(jobName, "autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt");
+
+ addIncrementalData();
+ verifyIncrementalDataResult();
+ cancelCurrentE2EJob(jobName);
+ }
+
+ @Test
+ public void testMySQL2DorisEnableDelete() throws Exception {
+ initialize();
+ String jobName = "testMySQL2DorisEnableDelete";
+ startMysql2DorisJob(jobName, "autoci/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt");
+
+ addIncrementalData();
+ E2EContainerUtils.executeSQLStatement(
+ getMySQLQueryConnection(),
+ LOG,
+ "delete from test_e2e_mysql.tbl3 where name='doris_3'",
+ "delete from test_e2e_mysql.tbl5 where name='doris_5'");
+
+ Thread.sleep(20000);
+ List expected =
+ Arrays.asList(
+ "doris_1,18",
+ "doris_1_1,10",
+ "doris_2,2",
+ "doris_2_1,11",
+ "doris_3,3",
+ "doris_3_1,12",
+ "doris_5,5");
+ String sql =
+ "select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.tbl5) res order by 1";
+ E2EContainerUtils.checkResult(getDorisQueryConnection(), expected, sql, 2);
+ cancelCurrentE2EJob(jobName);
+ }
+}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisCatalogITCase.java
similarity index 98%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisCatalogITCase.java
index 712b1881b..779e081f3 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisCatalogITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.catalog;
+package org.apache.doris.flink.autoci.itcase;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.EnvironmentSettings;
@@ -40,7 +40,8 @@
import org.apache.flink.util.CollectionUtil;
import com.google.common.collect.Lists;
-import org.apache.doris.flink.DorisTestBase;
+import org.apache.doris.flink.autoci.DorisTestBase;
+import org.apache.doris.flink.catalog.DorisCatalog;
import org.apache.doris.flink.cfg.DorisConnectionOptions;
import org.junit.Assert;
import org.junit.Before;
@@ -183,13 +184,6 @@ public void setup()
catalog.createTable(new ObjectPath(TEST_DB, TEST_TABLE_SINK_GROUPBY), createTable(), true);
}
- @Test
- @Ignore
- public void testQueryFenodes() {
- String actual = catalog.queryFenodes();
- assertEquals(getFenodes(), actual);
- }
-
@Test
public void testListDatabases() {
List actual = catalog.listDatabases();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisRowDataJdbcLookupFunctionITCase.java
similarity index 97%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisRowDataJdbcLookupFunctionITCase.java
index 0ad1781ae..5f4467d6f 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisRowDataJdbcLookupFunctionITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.table;
+package org.apache.doris.flink.autoci.itcase;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.data.GenericRowData;
@@ -26,9 +26,10 @@
import org.apache.flink.util.Collector;
import com.google.common.cache.Cache;
-import org.apache.doris.flink.DorisTestBase;
+import org.apache.doris.flink.autoci.DorisTestBase;
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.table.DorisRowDataJdbcLookupFunction;
import org.junit.Before;
import org.junit.Test;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSinkITCase.java
similarity index 98%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSinkITCase.java
index de0ef0413..0576e03ac 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSinkITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.sink;
+package org.apache.doris.flink.autoci.itcase;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.RuntimeExecutionMode;
@@ -26,10 +26,12 @@
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.doris.flink.DorisTestBase;
+import org.apache.doris.flink.autoci.DorisTestBase;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.DorisSink;
+import org.apache.doris.flink.sink.DorisSink.Builder;
import org.apache.doris.flink.sink.batch.DorisBatchSink;
import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer;
import org.apache.doris.flink.utils.MockSource;
@@ -110,7 +112,7 @@ public void testSinkJsonFormat() throws Exception {
public void submitJob(String table, Properties properties, String[] records) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
- DorisSink.Builder builder = DorisSink.builder();
+ Builder builder = DorisSink.builder();
final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder();
DorisOptions.Builder dorisBuilder = DorisOptions.builder();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSourceITCase.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSourceITCase.java
index a13e96f7f..87a9b6a6a 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSourceITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.source;
+package org.apache.doris.flink.autoci.itcase;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
@@ -25,11 +25,12 @@
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
-import org.apache.doris.flink.DorisTestBase;
+import org.apache.doris.flink.autoci.DorisTestBase;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisStreamOptions;
import org.apache.doris.flink.datastream.DorisSourceFunction;
import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
+import org.apache.doris.flink.source.DorisSource;
import org.junit.Assert;
import org.junit.Test;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/SchemaManagerITCase.java
similarity index 98%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/SchemaManagerITCase.java
index 3dde08d51..9466cd52a 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/SchemaManagerITCase.java
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.sink.schema;
+package org.apache.doris.flink.autoci.itcase;
-import org.apache.doris.flink.DorisTestBase;
+import org.apache.doris.flink.autoci.DorisTestBase;
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.catalog.doris.FieldSchema;
import org.apache.doris.flink.catalog.doris.TableSchema;
@@ -25,6 +25,7 @@
import org.apache.doris.flink.exception.IllegalArgumentException;
import org.apache.doris.flink.rest.models.Field;
import org.apache.doris.flink.rest.models.Schema;
+import org.apache.doris.flink.sink.schema.SchemaChangeManager;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
index d65deeb0b..951b874a3 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
@@ -63,6 +63,15 @@ public void testParserAlterDDLs() {
}
}
+ @Test
+ public void testParserAlterDDLsa() {
+
+ SourceConnector mysql = SourceConnector.MYSQL;
+ String ddl =
+ "alter table test_hualing add column `order_dt_23312` date DEFAULT(CURRENT_DATE) COMMENT '第三方单行号1号'";
+ List actualDDLs = schemaManager.parseAlterDDLs(mysql, ddl, dorisTable);
+ }
+
@Test
public void testParserAlterDDLsAdd() {
List expectDDLs = new ArrayList<>();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java
deleted file mode 100644
index aeb17c29d..000000000
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MySQLDorisE2ECase.java
+++ /dev/null
@@ -1,779 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.flink.tools.cdc;
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.common.time.Deadline;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.execution.JobClient;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
-import org.apache.doris.flink.DorisTestBase;
-import org.apache.doris.flink.sink.schema.SchemaChangeMode;
-import org.apache.doris.flink.tools.cdc.mysql.MysqlDatabaseSync;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.MySQLContainer;
-import org.testcontainers.lifecycle.Startables;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.stream.Stream;
-
-import static org.apache.flink.api.common.JobStatus.RUNNING;
-
-/**
- * MySQLDorisE2ECase 1. Automatically create tables 2. Schema change event synchronization
- * 3.Synchronization of addition, deletion and modification events 4. CDC multi-table writing.
- */
-public class MySQLDorisE2ECase extends DorisTestBase {
- protected static final Logger LOG = LoggerFactory.getLogger(MySQLDorisE2ECase.class);
- private static final String DATABASE = "test_e2e_mysql";
- private static final String MYSQL_USER = "root";
- private static final String MYSQL_PASSWD = "123456";
- private static final String TABLE_1 = "tbl1";
- private static final String TABLE_2 = "tbl2";
- private static final String TABLE_3 = "tbl3";
- private static final String TABLE_4 = "tbl4";
- private static final String TABLE_5 = "tbl5";
- private static final String TABLE_SQL_PARSE = "tbl_sql_parse";
-
- private static final MySQLContainer MYSQL_CONTAINER =
- new MySQLContainer("mysql:8.0")
- .withDatabaseName(DATABASE)
- .withUsername(MYSQL_USER)
- .withPassword(MYSQL_PASSWD);
-
- @BeforeClass
- public static void startMySQLContainers() {
- MYSQL_CONTAINER.setCommand("--default-time-zone=Asia/Shanghai");
- LOG.info("Starting MySQL containers...");
- Startables.deepStart(Stream.of(MYSQL_CONTAINER)).join();
- LOG.info("MySQL Containers are started.");
- }
-
- @AfterClass
- public static void stopMySQLContainers() {
- LOG.info("Stopping MySQL containers...");
- MYSQL_CONTAINER.stop();
- LOG.info("MySQL Containers are stopped.");
- }
-
- @Test
- public void testMySQL2Doris() throws Exception {
- printClusterStatus();
- initializeMySQLTable();
- initializeDorisTable();
- JobClient jobClient = submitJob();
- // wait 2 times checkpoint
- Thread.sleep(20000);
- List expected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3", "doris_5,5");
- String sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s union all select * from %s.%s) res order by 1";
- String query1 =
- String.format(
- sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3, DATABASE,
- TABLE_5);
- checkResult(expected, query1, 2);
-
- // add incremental data
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format("insert into %s.%s values ('doris_1_1',10)", DATABASE, TABLE_1));
- statement.execute(
- String.format("insert into %s.%s values ('doris_2_1',11)", DATABASE, TABLE_2));
- statement.execute(
- String.format("insert into %s.%s values ('doris_3_1',12)", DATABASE, TABLE_3));
-
- statement.execute(
- String.format(
- "update %s.%s set age=18 where name='doris_1'", DATABASE, TABLE_1));
- statement.execute(
- String.format("delete from %s.%s where name='doris_2'", DATABASE, TABLE_2));
- }
-
- Thread.sleep(20000);
- List expected2 =
- Arrays.asList(
- "doris_1,18", "doris_1_1,10", "doris_2_1,11", "doris_3,3", "doris_3_1,12");
- sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1";
- String query2 = String.format(sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3);
- checkResult(expected2, query2, 2);
-
- // mock schema change
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format(
- "alter table %s.%s add column c1 varchar(128)", DATABASE, TABLE_1));
- statement.execute(
- String.format("alter table %s.%s drop column age", DATABASE, TABLE_1));
- Thread.sleep(20000);
- statement.execute(
- String.format(
- "insert into %s.%s values ('doris_1_1_1','c1_val')",
- DATABASE, TABLE_1));
- }
- Thread.sleep(20000);
- List expected3 =
- Arrays.asList("doris_1,null", "doris_1_1,null", "doris_1_1_1,c1_val");
- sql = "select * from %s.%s order by 1";
- String query3 = String.format(sql, DATABASE, TABLE_1);
- checkResult(expected3, query3, 2);
- jobClient.cancel().get();
- }
-
- @Test
- public void testAutoAddTable() throws Exception {
- printClusterStatus();
- initializeMySQLTable();
- initializeDorisTable();
- JobClient jobClient = submitJob();
- // wait 2 times checkpoint
- Thread.sleep(20000);
- List expected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3");
- String sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1";
- String query1 = String.format(sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3);
- checkResult(expected, query1, 2);
-
- // auto create table4
- addTableTable_4();
- Thread.sleep(20000);
- List expected2 = Arrays.asList("doris_4_1,4", "doris_4_2,4");
- sql = "select * from %s.%s order by 1";
- String query2 = String.format(sql, DATABASE, TABLE_4);
- checkResult(expected2, query2, 2);
-
- // add incremental data
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format("insert into %s.%s values ('doris_1_1',10)", DATABASE, TABLE_1));
- statement.execute(
- String.format("insert into %s.%s values ('doris_2_1',11)", DATABASE, TABLE_2));
- statement.execute(
- String.format("insert into %s.%s values ('doris_3_1',12)", DATABASE, TABLE_3));
- statement.execute(
- String.format("insert into %s.%s values ('doris_4_3',43)", DATABASE, TABLE_4));
-
- statement.execute(
- String.format(
- "update %s.%s set age=18 where name='doris_1'", DATABASE, TABLE_1));
- statement.execute(
- String.format("delete from %s.%s where name='doris_2'", DATABASE, TABLE_2));
- statement.execute(
- String.format("delete from %s.%s where name='doris_4_2'", DATABASE, TABLE_4));
- statement.execute(
- String.format(
- "update %s.%s set age=41 where name='doris_4_1'", DATABASE, TABLE_4));
- }
-
- Thread.sleep(20000);
- List expected3 =
- Arrays.asList(
- "doris_1,18",
- "doris_1_1,10",
- "doris_2_1,11",
- "doris_3,3",
- "doris_3_1,12",
- "doris_4_1,41",
- "doris_4_3,43");
- sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1";
- String query3 =
- String.format(
- sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3, DATABASE,
- TABLE_4);
- checkResult(expected3, query3, 2);
-
- // mock schema change
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format(
- "alter table %s.%s add column c1 varchar(128)", DATABASE, TABLE_4));
- statement.execute(
- String.format("alter table %s.%s drop column age", DATABASE, TABLE_4));
- Thread.sleep(20000);
- statement.execute(
- String.format(
- "insert into %s.%s values ('doris_4_4','c1_val')", DATABASE, TABLE_4));
- }
- Thread.sleep(20000);
- List expected4 =
- Arrays.asList("doris_4_1,null", "doris_4_3,null", "doris_4_4,c1_val");
- sql = "select * from %s.%s order by 1";
- String query4 = String.format(sql, DATABASE, TABLE_4);
- checkResult(expected4, query4, 2);
- jobClient.cancel().get();
- }
-
- @Test
- public void testMySQL2DorisSQLParse() throws Exception {
- printClusterStatus();
- initializeMySQLTable();
- initializeDorisTable();
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- Map flinkMap = new HashMap<>();
- flinkMap.put("execution.checkpointing.interval", "10s");
- flinkMap.put("pipeline.operator-chaining", "false");
- flinkMap.put("parallelism.default", "1");
-
- Configuration configuration = Configuration.fromMap(flinkMap);
- env.configure(configuration);
-
- String database = DATABASE;
- Map mysqlConfig = new HashMap<>();
- mysqlConfig.put("database-name", DATABASE);
- mysqlConfig.put("hostname", MYSQL_CONTAINER.getHost());
- mysqlConfig.put("port", MYSQL_CONTAINER.getMappedPort(3306) + "");
- mysqlConfig.put("username", MYSQL_USER);
- mysqlConfig.put("password", MYSQL_PASSWD);
- mysqlConfig.put("server-time-zone", "Asia/Shanghai");
- Configuration config = Configuration.fromMap(mysqlConfig);
-
- Map sinkConfig = new HashMap<>();
- sinkConfig.put("fenodes", getFenodes());
- sinkConfig.put("username", USERNAME);
- sinkConfig.put("password", PASSWORD);
- sinkConfig.put("jdbc-url", String.format(DorisTestBase.URL, DORIS_CONTAINER.getHost()));
- sinkConfig.put("sink.label-prefix", UUID.randomUUID().toString());
- sinkConfig.put("sink.check-interval", "5000");
- Configuration sinkConf = Configuration.fromMap(sinkConfig);
-
- Map tableConfig = new HashMap<>();
- tableConfig.put("replication_num", "1");
-
- String includingTables = "tbl.*";
- String excludingTables = "";
- DatabaseSync databaseSync = new MysqlDatabaseSync();
- databaseSync
- .setEnv(env)
- .setDatabase(database)
- .setConfig(config)
- .setIncludingTables(includingTables)
- .setExcludingTables(excludingTables)
- .setIgnoreDefaultValue(false)
- .setSinkConfig(sinkConf)
- .setTableConfig(tableConfig)
- .setCreateTableOnly(false)
- .setNewSchemaChange(true)
- .setSchemaChangeMode(SchemaChangeMode.SQL_PARSER.getName())
- // no single sink
- .setSingleSink(true)
- .create();
- databaseSync.build();
- JobClient jobClient = env.executeAsync();
- waitForJobStatus(
- jobClient,
- Collections.singletonList(RUNNING),
- Deadline.fromNow(Duration.ofSeconds(10)));
-
- // wait 2 times checkpoint
- Thread.sleep(20000);
- List expected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3", "doris_5,5");
- String sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s union all select * from %s.%s) res order by 1";
- String query1 =
- String.format(
- sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3, DATABASE,
- TABLE_5);
- checkResult(expected, query1, 2);
-
- // add incremental data
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format("insert into %s.%s values ('doris_1_1',10)", DATABASE, TABLE_1));
- statement.execute(
- String.format("insert into %s.%s values ('doris_2_1',11)", DATABASE, TABLE_2));
- statement.execute(
- String.format("insert into %s.%s values ('doris_3_1',12)", DATABASE, TABLE_3));
-
- statement.execute(
- String.format(
- "update %s.%s set age=18 where name='doris_1'", DATABASE, TABLE_1));
- statement.execute(
- String.format("delete from %s.%s where name='doris_2'", DATABASE, TABLE_2));
- }
-
- Thread.sleep(20000);
- List expected2 =
- Arrays.asList(
- "doris_1,18", "doris_1_1,10", "doris_2_1,11", "doris_3,3", "doris_3_1,12");
- sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1";
- String query2 = String.format(sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3);
- checkResult(expected2, query2, 2);
-
- // mock schema change
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format(
- "alter table %s.%s add column c1 varchar(128)", DATABASE, TABLE_1));
- statement.execute(
- String.format("alter table %s.%s drop column age", DATABASE, TABLE_1));
- Thread.sleep(20000);
- statement.execute(
- String.format(
- "insert into %s.%s values ('doris_1_1_1','c1_val')",
- DATABASE, TABLE_1));
- }
- Thread.sleep(20000);
- List expected3 =
- Arrays.asList("doris_1,null", "doris_1_1,null", "doris_1_1_1,c1_val");
- sql = "select * from %s.%s order by 1";
- String query3 = String.format(sql, DATABASE, TABLE_1);
- checkResult(expected3, query3, 2);
-
- // mock create table
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256) primary key,\n"
- + "`age` int\n"
- + ")",
- DATABASE, TABLE_SQL_PARSE));
- statement.execute(
- String.format(
- "insert into %s.%s values ('doris_1',1)", DATABASE, TABLE_SQL_PARSE));
- statement.execute(
- String.format(
- "insert into %s.%s values ('doris_2',2)", DATABASE, TABLE_SQL_PARSE));
- statement.execute(
- String.format(
- "insert into %s.%s values ('doris_3',3)", DATABASE, TABLE_SQL_PARSE));
- }
- Thread.sleep(20000);
- List expected4 = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3");
- sql = "select * from %s.%s order by 1";
- String query4 = String.format(sql, DATABASE, TABLE_SQL_PARSE);
- checkResult(expected4, query4, 2);
-
- jobClient.cancel().get();
- }
-
- @Test
- public void testMySQL2DorisByDefault() throws Exception {
- printClusterStatus();
- initializeMySQLTable();
- initializeDorisTable();
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- Map flinkMap = new HashMap<>();
- flinkMap.put("execution.checkpointing.interval", "10s");
- flinkMap.put("pipeline.operator-chaining", "false");
- flinkMap.put("parallelism.default", "1");
-
- Configuration configuration = Configuration.fromMap(flinkMap);
- env.configure(configuration);
-
- String database = DATABASE;
- Map mysqlConfig = new HashMap<>();
- mysqlConfig.put("database-name", DATABASE);
- mysqlConfig.put("hostname", MYSQL_CONTAINER.getHost());
- mysqlConfig.put("port", MYSQL_CONTAINER.getMappedPort(3306) + "");
- mysqlConfig.put("username", MYSQL_USER);
- mysqlConfig.put("password", MYSQL_PASSWD);
- mysqlConfig.put("server-time-zone", "Asia/Shanghai");
- Configuration config = Configuration.fromMap(mysqlConfig);
-
- Map sinkConfig = new HashMap<>();
- sinkConfig.put("fenodes", getFenodes());
- sinkConfig.put("username", USERNAME);
- sinkConfig.put("password", PASSWORD);
- sinkConfig.put("jdbc-url", String.format(DorisTestBase.URL, DORIS_CONTAINER.getHost()));
- sinkConfig.put("sink.label-prefix", UUID.randomUUID().toString());
- sinkConfig.put("sink.check-interval", "5000");
- Configuration sinkConf = Configuration.fromMap(sinkConfig);
-
- Map tableConfig = new HashMap<>();
- tableConfig.put("replication_num", "1");
-
- String includingTables = "tbl1|tbl2|tbl3|tbl5";
- String excludingTables = "";
- DatabaseSync databaseSync = new MysqlDatabaseSync();
- databaseSync
- .setEnv(env)
- .setDatabase(database)
- .setConfig(config)
- .setIncludingTables(includingTables)
- .setExcludingTables(excludingTables)
- .setIgnoreDefaultValue(false)
- .setSinkConfig(sinkConf)
- .setTableConfig(tableConfig)
- .setCreateTableOnly(false)
- .setNewSchemaChange(true)
- // no single sink
- .setSingleSink(false)
- .create();
- databaseSync.build();
- JobClient jobClient = env.executeAsync();
- waitForJobStatus(
- jobClient,
- Collections.singletonList(RUNNING),
- Deadline.fromNow(Duration.ofSeconds(10)));
-
- // wait 2 times checkpoint
- Thread.sleep(20000);
- List expected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3", "doris_5,5");
- String sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s union all select * from %s.%s) res order by 1";
- String query1 =
- String.format(
- sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3, DATABASE,
- TABLE_5);
- checkResult(expected, query1, 2);
-
- // add incremental data
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format("insert into %s.%s values ('doris_1_1',10)", DATABASE, TABLE_1));
- statement.execute(
- String.format("insert into %s.%s values ('doris_2_1',11)", DATABASE, TABLE_2));
- statement.execute(
- String.format("insert into %s.%s values ('doris_3_1',12)", DATABASE, TABLE_3));
-
- statement.execute(
- String.format(
- "update %s.%s set age=18 where name='doris_1'", DATABASE, TABLE_1));
- statement.execute(
- String.format("delete from %s.%s where name='doris_2'", DATABASE, TABLE_2));
- }
-
- Thread.sleep(20000);
- List expected2 =
- Arrays.asList(
- "doris_1,18", "doris_1_1,10", "doris_2_1,11", "doris_3,3", "doris_3_1,12");
- sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s ) res order by 1";
- String query2 = String.format(sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3);
- checkResult(expected2, query2, 2);
- jobClient.cancel().get();
- }
-
- @Test
- public void testMySQL2DorisEnableDelete() throws Exception {
- printClusterStatus();
- initializeMySQLTable();
- initializeDorisTable();
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- Map flinkMap = new HashMap<>();
- flinkMap.put("execution.checkpointing.interval", "10s");
- flinkMap.put("pipeline.operator-chaining", "false");
- flinkMap.put("parallelism.default", "1");
-
- Configuration configuration = Configuration.fromMap(flinkMap);
- env.configure(configuration);
-
- String database = DATABASE;
- Map mysqlConfig = new HashMap<>();
- mysqlConfig.put("database-name", DATABASE);
- mysqlConfig.put("hostname", MYSQL_CONTAINER.getHost());
- mysqlConfig.put("port", MYSQL_CONTAINER.getMappedPort(3306) + "");
- mysqlConfig.put("username", MYSQL_USER);
- mysqlConfig.put("password", MYSQL_PASSWD);
- mysqlConfig.put("server-time-zone", "Asia/Shanghai");
- Configuration config = Configuration.fromMap(mysqlConfig);
-
- Map sinkConfig = new HashMap<>();
- sinkConfig.put("fenodes", getFenodes());
- sinkConfig.put("username", USERNAME);
- sinkConfig.put("password", PASSWORD);
- sinkConfig.put("jdbc-url", String.format(DorisTestBase.URL, DORIS_CONTAINER.getHost()));
- sinkConfig.put("sink.label-prefix", UUID.randomUUID().toString());
- sinkConfig.put("sink.check-interval", "5000");
- sinkConfig.put("sink.enable-delete", "false");
- Configuration sinkConf = Configuration.fromMap(sinkConfig);
-
- Map tableConfig = new HashMap<>();
- tableConfig.put("replication_num", "1");
-
- String includingTables = "tbl1|tbl2|tbl3|tbl5";
- String excludingTables = "";
- DatabaseSync databaseSync = new MysqlDatabaseSync();
- databaseSync
- .setEnv(env)
- .setDatabase(database)
- .setConfig(config)
- .setIncludingTables(includingTables)
- .setExcludingTables(excludingTables)
- .setIgnoreDefaultValue(false)
- .setSinkConfig(sinkConf)
- .setTableConfig(tableConfig)
- .setCreateTableOnly(false)
- .setNewSchemaChange(true)
- // no single sink
- .setSingleSink(false)
- .create();
- databaseSync.build();
- JobClient jobClient = env.executeAsync();
- waitForJobStatus(
- jobClient,
- Collections.singletonList(RUNNING),
- Deadline.fromNow(Duration.ofSeconds(10)));
-
- // wait 2 times checkpoint
- Thread.sleep(20000);
- List expected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3", "doris_5,5");
- String sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s union all select * from %s.%s) res order by 1";
- String query1 =
- String.format(
- sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3, DATABASE,
- TABLE_5);
- checkResult(expected, query1, 2);
-
- // add incremental data
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(
- String.format("insert into %s.%s values ('doris_1_1',10)", DATABASE, TABLE_1));
- statement.execute(
- String.format("insert into %s.%s values ('doris_2_1',11)", DATABASE, TABLE_2));
- statement.execute(
- String.format("insert into %s.%s values ('doris_3_1',12)", DATABASE, TABLE_3));
-
- statement.execute(
- String.format(
- "update %s.%s set age=18 where name='doris_1'", DATABASE, TABLE_1));
- statement.execute(
- String.format("delete from %s.%s where name='doris_2'", DATABASE, TABLE_2));
- statement.execute(
- String.format("delete from %s.%s where name='doris_3'", DATABASE, TABLE_3));
- statement.execute(
- String.format("delete from %s.%s where name='doris_5'", DATABASE, TABLE_5));
- }
-
- Thread.sleep(20000);
- List expected2 =
- Arrays.asList(
- "doris_1,18",
- "doris_1_1,10",
- "doris_2,2",
- "doris_2_1,11",
- "doris_3,3",
- "doris_3_1,12",
- "doris_5,5");
- sql =
- "select * from ( select * from %s.%s union all select * from %s.%s union all select * from %s.%s union all select * from %s.%s) res order by 1";
- String query2 =
- String.format(
- sql, DATABASE, TABLE_1, DATABASE, TABLE_2, DATABASE, TABLE_3, DATABASE,
- TABLE_5);
- checkResult(expected2, query2, 2);
- jobClient.cancel().get();
- }
-
- private void initializeDorisTable() throws Exception {
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_1));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_2));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_3));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_4));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_5));
- // create a table in Doris
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256),\n"
- + "`age` int\n"
- + ")\n"
- + "UNIQUE KEY(`name`)\n"
- + "DISTRIBUTED BY HASH(`name`) BUCKETS 1\n"
- + "PROPERTIES ( \n"
- + "\"replication_num\" = \"1\" \n"
- + ");\n",
- DATABASE, TABLE_5));
- }
- }
-
- public JobClient submitJob() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- Map flinkMap = new HashMap<>();
- flinkMap.put("execution.checkpointing.interval", "10s");
- flinkMap.put("pipeline.operator-chaining", "false");
- flinkMap.put("parallelism.default", "1");
-
- Configuration configuration = Configuration.fromMap(flinkMap);
- env.configure(configuration);
-
- String database = DATABASE;
- Map mysqlConfig = new HashMap<>();
- mysqlConfig.put("database-name", DATABASE);
- mysqlConfig.put("hostname", MYSQL_CONTAINER.getHost());
- mysqlConfig.put("port", MYSQL_CONTAINER.getMappedPort(3306) + "");
- mysqlConfig.put("username", MYSQL_USER);
- mysqlConfig.put("password", MYSQL_PASSWD);
- mysqlConfig.put("server-time-zone", "Asia/Shanghai");
- Configuration config = Configuration.fromMap(mysqlConfig);
-
- Map sinkConfig = new HashMap<>();
- sinkConfig.put("fenodes", getFenodes());
- sinkConfig.put("username", USERNAME);
- sinkConfig.put("password", PASSWORD);
- sinkConfig.put("jdbc-url", String.format(DorisTestBase.URL, DORIS_CONTAINER.getHost()));
- sinkConfig.put("sink.label-prefix", UUID.randomUUID().toString());
- Configuration sinkConf = Configuration.fromMap(sinkConfig);
-
- Map tableConfig = new HashMap<>();
- tableConfig.put("replication_num", "1");
-
- String includingTables = "tbl.*";
- String excludingTables = "";
- DatabaseSync databaseSync = new MysqlDatabaseSync();
- databaseSync
- .setEnv(env)
- .setDatabase(database)
- .setConfig(config)
- .setIncludingTables(includingTables)
- .setExcludingTables(excludingTables)
- .setIgnoreDefaultValue(false)
- .setSinkConfig(sinkConf)
- .setTableConfig(tableConfig)
- .setCreateTableOnly(false)
- .setNewSchemaChange(true)
- .setSingleSink(true)
- .setIgnoreDefaultValue(true)
- .create();
- databaseSync.build();
- JobClient jobClient = env.executeAsync();
- waitForJobStatus(
- jobClient,
- Collections.singletonList(RUNNING),
- Deadline.fromNow(Duration.ofSeconds(10)));
- return jobClient;
- }
-
- private void addTableTable_4() throws SQLException {
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_4));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256) primary key,\n"
- + "`age` int\n"
- + ")",
- DATABASE, TABLE_4));
-
- // mock stock data
- statement.execute(
- String.format("insert into %s.%s values ('doris_4_1',4)", DATABASE, TABLE_4));
- statement.execute(
- String.format("insert into %s.%s values ('doris_4_2',4)", DATABASE, TABLE_4));
- }
- }
-
- public void initializeMySQLTable() throws Exception {
- try (Connection connection =
- DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_1));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_2));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_3));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_4));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, TABLE_5));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256) primary key,\n"
- + "`age` int\n"
- + ")",
- DATABASE, TABLE_1));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256) primary key,\n"
- + "`age` int\n"
- + ")",
- DATABASE, TABLE_2));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256) primary key,\n"
- + "`age` int\n"
- + ")",
- DATABASE, TABLE_3));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256) primary key,\n"
- + "`age` int\n"
- + ")",
- DATABASE, TABLE_5));
- // mock stock data
- statement.execute(
- String.format("insert into %s.%s values ('doris_1',1)", DATABASE, TABLE_1));
- statement.execute(
- String.format("insert into %s.%s values ('doris_2',2)", DATABASE, TABLE_2));
- statement.execute(
- String.format("insert into %s.%s values ('doris_3',3)", DATABASE, TABLE_3));
- statement.execute(
- String.format("insert into %s.%s values ('doris_5',5)", DATABASE, TABLE_5));
- }
- }
-}
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql
new file mode 100644
index 000000000..d86d6f718
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql
@@ -0,0 +1,31 @@
+CREATE DATABASE IF NOT EXISTS test_e2e_sink;
+
+DROP TABLE IF EXISTS test_e2e_sink.test_tbl;
+
+CREATE TABLE test_e2e_source.test_tbl (
+ `id` int,
+ `c1` boolean,
+ `c2` tinyint,
+ `c3` smallint,
+ `c4` int,
+ `c5` bigint,
+ `c6` largeint,
+ `c7` float,
+ `c8` double,
+ `c9` decimal(12,4),
+ `c10` date,
+ `c11` datetime,
+ `c12` char(1),
+ `c13` varchar(256),
+ `c14` Array,
+ `c15` Map,
+ `c16` Struct,
+ `c17` JSON
+)
+ DUPLICATE KEY(`id`)
+DISTRIBUTED BY HASH(`id`) BUCKETS 1
+PROPERTIES (
+"replication_num" = "1",
+"light_schema_change" = "true"
+);
+
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql
new file mode 100644
index 000000000..77018f3fc
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql
@@ -0,0 +1,73 @@
+CREATE DATABASE IF NOT EXISTS test_e2e_source;
+
+DROP TABLE IF EXISTS test_e2e_source.test_tbl;
+
+CREATE TABLE test_e2e_source.test_tbl (
+ `id` int,
+ `c1` boolean,
+ `c2` tinyint,
+ `c3` smallint,
+ `c4` int,
+ `c5` bigint,
+ `c6` largeint,
+ `c7` float,
+ `c8` double,
+ `c9` decimal(12,4),
+ `c10` date,
+ `c11` datetime,
+ `c12` char(1),
+ `c13` varchar(256),
+ `c14` Array,
+ `c15` Map,
+ `c16` Struct,
+ `c17` JSON
+)
+DUPLICATE KEY(`id`)
+DISTRIBUTED BY HASH(`id`) BUCKETS 1
+PROPERTIES (
+"replication_num" = "1",
+"light_schema_change" = "true"
+);
+
+INSERT INTO test_e2e_source.test_tbl
+VALUES
+ (
+ 1,
+ TRUE,
+ 127,
+ 32767,
+ 2147483647,
+ 9223372036854775807,
+ 123456789012345678901234567890,
+ 3.14,
+ 2.7182818284,
+ 12345.6789,
+ '2023-05-22',
+ '2023-05-22 12:34:56',
+ 'A',
+ 'Example text',
+ ['item1', 'item2', 'item3'],
+ {'key1': 'value1', 'key2': 'value2'},
+ STRUCT('John Doe', 30),
+ '{"key": "value"}'
+ ),
+ (
+ 2,
+ FALSE,
+ -128,
+ -32768,
+ -2147483648,
+ -9223372036854775808,
+ -123456789012345678901234567890,
+ -3.14,
+ -2.7182818284,
+ -12345.6789,
+ '2024-01-01',
+ '2024-01-01 00:00:00',
+ 'B',
+ 'Another example',
+ ['item4', 'item5', 'item6'],
+ {'key3': 'value3', 'key4': 'value4'},
+ STRUCT('Jane Doe', 25),
+ '{"another_key": "another_value"}'
+);
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/doris_tbl5.sql b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/doris_tbl5.sql
new file mode 100644
index 000000000..ebe163b2c
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/doris_tbl5.sql
@@ -0,0 +1,11 @@
+DROP TABLE IF EXISTS test_e2e_mysql.tbl5;
+
+CREATE TABLE test_e2e_mysql.tbl5 (
+ `name` varchar(256) primary key,
+ `age` int
+)
+UNIQUE KEY(`name`)
+DISTRIBUTED BY HASH(`name`) BUCKETS 1
+PROPERTIES (
+ "replication_num"="1"
+);
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl1.sql b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl1.sql
new file mode 100644
index 000000000..bc5849a84
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl1.sql
@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS test_e2e_mysql.tbl1;
+
+CREATE TABLE test_e2e_mysql.tbl1 (
+ `name` varchar(256) primary key,
+ `age` int
+);
+
+insert into test_e2e_mysql.tbl1 values ('doris_1',1);
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl2.sql b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl2.sql
new file mode 100644
index 000000000..8222d6c45
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl2.sql
@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS test_e2e_mysql.tbl2;
+
+CREATE TABLE test_e2e_mysql.tbl2 (
+ `name` varchar(256) primary key,
+ `age` int
+);
+
+insert into test_e2e_mysql.tbl2 values ('doris_2',2);
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl3.sql b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl3.sql
new file mode 100644
index 000000000..da1118ab7
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl3.sql
@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS test_e2e_mysql.tbl3;
+
+CREATE TABLE test_e2e_mysql.tbl3 (
+ `name` varchar(256) primary key,
+ `age` int
+);
+
+insert into test_e2e_mysql.tbl3 values ('doris_3',3);
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl4.sql b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl4.sql
new file mode 100644
index 000000000..320e7e3a0
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl4.sql
@@ -0,0 +1,6 @@
+DROP TABLE IF EXISTS test_e2e_mysql.tbl4;
+
+CREATE TABLE test_e2e_mysql.tbl4 (
+ `name` varchar(256) primary key,
+ `age` int
+);
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl5.sql b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl5.sql
new file mode 100644
index 000000000..0658d7e9e
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/initialize/mysql_tbl5.sql
@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS test_e2e_mysql.tbl5;
+
+CREATE TABLE test_e2e_mysql.tbl5 (
+ `name` varchar(256) primary key,
+ `age` int
+);
+
+insert into test_e2e_mysql.tbl5 values ('doris_5',5);
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testAutoAddTable.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testAutoAddTable.txt
new file mode 100644
index 000000000..e5c03a968
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testAutoAddTable.txt
@@ -0,0 +1,6 @@
+mysql-sync-database
+ --including-tables "tbl.*|auto_add"
+ --table-conf replication_num=1
+ --sink-conf sink.check-interval=5000
+ --single-sink true
+ --ignore-default-value true
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2Doris.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2Doris.txt
new file mode 100644
index 000000000..601d08311
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2Doris.txt
@@ -0,0 +1,5 @@
+mysql-sync-database
+ --including-tables "tbl.*"
+ --table-conf replication_num=1
+ --single-sink true
+ --ignore-default-value false
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt
new file mode 100644
index 000000000..922538143
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt
@@ -0,0 +1,4 @@
+mysql-sync-database
+ --including-tables "tbl1|tbl2|tbl3|tbl5"
+ --table-conf replication_num=1
+ --sink-conf sink.check-interval=5000
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
new file mode 100644
index 000000000..1048916ca
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
@@ -0,0 +1,5 @@
+mysql-sync-database
+ --including-tables "tbl1|tbl2|tbl3|tbl5"
+ --table-conf replication_num=1
+ --sink-conf sink.enable-delete=false
+ --sink-conf sink.check-interval=5000
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
new file mode 100644
index 000000000..1190d10de
--- /dev/null
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
@@ -0,0 +1,6 @@
+mysql-sync-database
+ --including-tables "tbl.*|add_tbl"
+ --table-conf replication_num=1
+ --sink-conf sink.check-interval=5000
+ --schema-change-mode sql_parser
+ --single-sink true
\ No newline at end of file
From ebb21da0edc872c6a19e85a0ed27e8fe4af12025 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 11:31:36 +0800
Subject: [PATCH 02/48] fix
---
.../e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql
index d86d6f718..f569df113 100644
--- a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql
@@ -2,7 +2,7 @@ CREATE DATABASE IF NOT EXISTS test_e2e_sink;
DROP TABLE IF EXISTS test_e2e_sink.test_tbl;
-CREATE TABLE test_e2e_source.test_tbl (
+CREATE TABLE test_e2e_sink.test_tbl (
`id` int,
`c1` boolean,
`c2` tinyint,
From a812ae521fb104fbcd48669129690ad531f1a625 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 11:43:29 +0800
Subject: [PATCH 03/48] fix
---
.../org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
index 14108fbb1..0ba964de4 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
@@ -39,6 +39,9 @@
public class E2EContainerUtils {
protected static void executeSQLStatement(Connection connection, Logger logger, String... sql) {
+ if (Objects.isNull(sql) || sql.length == 0) {
+ return;
+ }
try (Statement statement = connection.createStatement()) {
for (String s : sql) {
logger.info("start to execute sql={}", s);
From 3b0d92ece57c2e616f1acc8ae7198d99202ad9de Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 11:50:32 +0800
Subject: [PATCH 04/48] fix
---
.../org/apache/doris/flink/autoci/AbstractAutoCITestBase.java | 3 +--
.../org/apache/doris/flink/autoci/e2e/AbstractE2EService.java | 3 +--
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
index 9efff651e..38d388c55 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
@@ -19,7 +19,6 @@
import org.apache.doris.flink.autoci.container.ContainerService;
import org.apache.doris.flink.autoci.container.DorisContainerService;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +75,7 @@ protected String getDorisInstanceHost() {
return dorisContainerService.getInstanceHost();
}
- @AfterClass
+ // @AfterClass
public static void closeContainers() {
LOG.info("Starting to close auto ci containers.");
closeDorisContainer();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
index 378bfec73..a8e4d226b 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
@@ -27,7 +27,6 @@
import org.apache.doris.flink.exception.DorisRuntimeException;
import org.apache.doris.flink.tools.cdc.CdcTools;
import org.apache.doris.flink.tools.cdc.DatabaseSyncConfig;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -157,7 +156,7 @@ protected void setSinkConfDefaultConfig(List argList) {
argList.add(SINK_LABEL_PREFIX + "=" + "label");
}
- @AfterClass
+ // @AfterClass
public static void closeE2EContainers() {
LOG.info("Starting to close E2E containers.");
closeMySQLContainer();
From 1788ae3c51dcece7d844a23eff4da1625e65c3e5 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 12:07:06 +0800
Subject: [PATCH 05/48] fix
---
.../flink/autoci/AbstractAutoCITestBase.java | 1 -
.../flink/autoci/e2e/AbstractE2EService.java | 21 -------------------
.../flink/autoci/e2e/Doris2DorisService.java | 8 +++----
...sql => test_doris2doris_sink_test_tbl.sql} | 6 +++---
...l => test_doris2doris_source_test_tbl.sql} | 8 +++----
5 files changed, 11 insertions(+), 33 deletions(-)
rename flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/{test_e2e_sink_test_tbl.sql => test_doris2doris_sink_test_tbl.sql} (78%)
rename flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/{test_e2e_source_test_tbl.sql => test_doris2doris_source_test_tbl.sql} (87%)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
index 38d388c55..6d36d9ed5 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
@@ -75,7 +75,6 @@ protected String getDorisInstanceHost() {
return dorisContainerService.getInstanceHost();
}
- // @AfterClass
public static void closeContainers() {
LOG.info("Starting to close auto ci containers.");
closeDorisContainer();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
index a8e4d226b..3e5ada554 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
@@ -36,15 +36,12 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
public abstract class AbstractE2EService extends AbstractAutoCITestBase {
private static final Logger LOG = LoggerFactory.getLogger(AbstractE2EService.class);
private static ContainerService mysqlContainerService;
private static final CustomerSingleThreadExecutor singleThreadExecutor =
new CustomerSingleThreadExecutor();
- private static final BlockingQueue jobQueue = new LinkedBlockingQueue<>();
protected static final String SINK_CONF = "--" + DatabaseSyncConfig.SINK_CONF;
protected static final String DORIS_DATABASE = "--database";
protected static final String HOSTNAME = "hostname";
@@ -56,23 +53,6 @@ public abstract class AbstractE2EService extends AbstractAutoCITestBase {
protected static final String JDBC_URL = "jdbc-url";
protected static final String SINK_LABEL_PREFIX = "sink.label-prefix";
- static {
- // 启动一个线程来顺序执行任务
- new Thread(
- () -> {
- while (true) {
- try {
- Runnable job = jobQueue.take();
- job.run();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.error("Task execution interrupted.", e);
- }
- }
- })
- .start();
- }
-
@BeforeClass
public static void initE2EContainers() {
LOG.info("Starting to init E2E containers.");
@@ -156,7 +136,6 @@ protected void setSinkConfDefaultConfig(List argList) {
argList.add(SINK_LABEL_PREFIX + "=" + "label");
}
- // @AfterClass
public static void closeE2EContainers() {
LOG.info("Starting to close E2E containers.");
closeMySQLContainer();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
index 35e8683a3..580460bfd 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
@@ -35,8 +35,8 @@
public class Doris2DorisService extends AbstractE2EService {
private static final Logger LOG = LoggerFactory.getLogger(Doris2DorisService.class);
- private static final String DATABASE_SOURCE = "test_e2e_source";
- private static final String DATABASE_SINK = "test_e2e_sink";
+ private static final String DATABASE_SOURCE = "test_doris2doris_source";
+ private static final String DATABASE_SINK = "test_doris2doris_sink";
private static final String TABLE = "test_tbl";
@Test
@@ -144,11 +144,11 @@ public void testDoris2Doris() throws Exception {
private void initializeDorisTable() {
String[] sourceInitSql =
E2EContainerUtils.parseFileContentSQL(
- "autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql");
+ "autoci/e2e/doris2doris/initialize/test_doris2doris_source_test_tbl.sql");
E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sourceInitSql);
String[] sinkInitSql =
E2EContainerUtils.parseFileContentSQL(
- "autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql");
+ "autoci/e2e/doris2doris/initialize/test_doris2doris_sink_test_tbl.sql");
E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sinkInitSql);
LOG.info("Initialization of doris table successful.");
}
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_doris2doris_sink_test_tbl.sql
similarity index 78%
rename from flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql
rename to flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_doris2doris_sink_test_tbl.sql
index f569df113..20ffb525e 100644
--- a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_sink_test_tbl.sql
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_doris2doris_sink_test_tbl.sql
@@ -1,8 +1,8 @@
-CREATE DATABASE IF NOT EXISTS test_e2e_sink;
+CREATE DATABASE IF NOT EXISTS test_doris2doris_sink;
-DROP TABLE IF EXISTS test_e2e_sink.test_tbl;
+DROP TABLE IF EXISTS test_doris2doris_sink.test_tbl;
-CREATE TABLE test_e2e_sink.test_tbl (
+CREATE TABLE test_doris2doris_sink.test_tbl (
`id` int,
`c1` boolean,
`c2` tinyint,
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_doris2doris_source_test_tbl.sql
similarity index 87%
rename from flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql
rename to flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_doris2doris_source_test_tbl.sql
index 77018f3fc..5e57b50ba 100644
--- a/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_e2e_source_test_tbl.sql
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/doris2doris/initialize/test_doris2doris_source_test_tbl.sql
@@ -1,8 +1,8 @@
-CREATE DATABASE IF NOT EXISTS test_e2e_source;
+CREATE DATABASE IF NOT EXISTS test_doris2doris_source;
-DROP TABLE IF EXISTS test_e2e_source.test_tbl;
+DROP TABLE IF EXISTS test_doris2doris_source.test_tbl;
-CREATE TABLE test_e2e_source.test_tbl (
+CREATE TABLE test_doris2doris_source.test_tbl (
`id` int,
`c1` boolean,
`c2` tinyint,
@@ -29,7 +29,7 @@ PROPERTIES (
"light_schema_change" = "true"
);
-INSERT INTO test_e2e_source.test_tbl
+INSERT INTO test_doris2doris_source.test_tbl
VALUES
(
1,
From b3592a956af5659e1e8430e89c2275cafbf4e98c Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 12:11:33 +0800
Subject: [PATCH 06/48] fix
---
.../apache/doris/flink/autoci/e2e/E2EContainerUtils.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
index 0ba964de4..210f527b3 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
@@ -44,8 +44,10 @@ protected static void executeSQLStatement(Connection connection, Logger logger,
}
try (Statement statement = connection.createStatement()) {
for (String s : sql) {
- logger.info("start to execute sql={}", s);
- statement.execute(s);
+ if (StringUtils.isNotEmpty(s)) {
+ logger.info("start to execute sql={}", s);
+ statement.execute(s);
+ }
}
} catch (SQLException e) {
throw new DorisRuntimeException(e);
From 4932b6521ca9cf612c0721db99ae9df7ef0c1d02 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 12:38:36 +0800
Subject: [PATCH 07/48] fix
---
.../org/apache/doris/flink/autoci/e2e/Doris2DorisService.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
index 580460bfd..f9aed1250 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
@@ -118,8 +118,8 @@ public void testDoris2Doris() throws Exception {
+ ")",
getFenodes(),
DATABASE_SINK + "." + TABLE,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sinkDDL);
tEnv.executeSql("INSERT INTO doris_sink SELECT * FROM doris_source").await();
From d33add343802af8f7d5242e8b1634c11861405a6 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 15:34:18 +0800
Subject: [PATCH 08/48] mini fix
---
.../flink/autoci/container/MySQLContainerService.java | 1 -
.../doris/flink/autoci/e2e/Mysql2DorisService.java | 9 +++------
2 files changed, 3 insertions(+), 7 deletions(-)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/MySQLContainerService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/MySQLContainerService.java
index 7261cc52f..883f6a46a 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/MySQLContainerService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/MySQLContainerService.java
@@ -46,7 +46,6 @@ private MySQLContainer createContainer() {
@Override
public void startContainer() {
- // mysqlcontainer.setCommand("--default-time-zone=Asia/Shanghai");
LOG.info("Starting MySQL container.");
Startables.deepStart(Stream.of(mysqlcontainer)).join();
LOG.info("MySQL Container was started.");
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java
index c28d5ce69..26e425156 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java
@@ -19,6 +19,7 @@
import org.apache.doris.flink.exception.DorisRuntimeException;
import org.apache.doris.flink.tools.cdc.DatabaseSyncConfig;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +33,8 @@ public class Mysql2DorisService extends AbstractE2EService {
private static final String CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS " + DATABASE;
private static final String MYSQL_CONF = "--" + DatabaseSyncConfig.MYSQL_CONF;
- private void initialize() {
+ @Before
+ public void initialize() {
// init mysql table
LOG.info("start to init mysql table.");
E2EContainerUtils.executeSQLStatement(getMySQLQueryConnection(), LOG, CREATE_DATABASE);
@@ -120,7 +122,6 @@ private void verifyInitializeResult() {
@Test
public void testMySQL2Doris() throws Exception {
- initialize();
String jobName = "testMySQL2Doris";
String resourcePath = "autoci/e2e/mysql2doris/testMySQL2Doris.txt";
startMysql2DorisJob(jobName, resourcePath);
@@ -135,7 +136,6 @@ public void testMySQL2Doris() throws Exception {
@Test
public void testAutoAddTable() throws InterruptedException {
- initialize();
String jobName = "testAutoAddTable";
startMysql2DorisJob(jobName, "autoci/e2e/mysql2doris/testAutoAddTable.txt");
@@ -198,7 +198,6 @@ public void testAutoAddTable() throws InterruptedException {
@Test
public void testMySQL2DorisSQLParse() throws Exception {
- initialize();
String jobName = "testMySQL2DorisSQLParse";
String resourcePath = "autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt";
startMysql2DorisJob(jobName, resourcePath);
@@ -287,7 +286,6 @@ private void verifyIncrementalDataResult() {
@Test
public void testMySQL2DorisByDefault() throws InterruptedException {
- initialize();
String jobName = "testMySQL2DorisByDefault";
startMysql2DorisJob(jobName, "autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt");
@@ -298,7 +296,6 @@ public void testMySQL2DorisByDefault() throws InterruptedException {
@Test
public void testMySQL2DorisEnableDelete() throws Exception {
- initialize();
String jobName = "testMySQL2DorisEnableDelete";
startMysql2DorisJob(jobName, "autoci/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt");
From 704c355c3417a1740ffec28768e4359c59e3aa48 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 15:41:36 +0800
Subject: [PATCH 09/48] mini fix
---
.../flink/sink/schema/SQLParserSchemaManagerTest.java | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
index 951b874a3..d65deeb0b 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
@@ -63,15 +63,6 @@ public void testParserAlterDDLs() {
}
}
- @Test
- public void testParserAlterDDLsa() {
-
- SourceConnector mysql = SourceConnector.MYSQL;
- String ddl =
- "alter table test_hualing add column `order_dt_23312` date DEFAULT(CURRENT_DATE) COMMENT '第三方单行号1号'";
- List actualDDLs = schemaManager.parseAlterDDLs(mysql, ddl, dorisTable);
- }
-
@Test
public void testParserAlterDDLsAdd() {
List expectDDLs = new ArrayList<>();
From 8007a9292dc9d13aa7d304229a4f1c8192f124e1 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 17:02:43 +0800
Subject: [PATCH 10/48] fix comment
---
.github/workflows/run-e2ecase.yml | 2 +-
.../apache/doris/flink/autoci/AbstractAutoCITestBase.java | 3 ++-
.../doris/flink/autoci/container/DorisContainerService.java | 5 -----
.../apache/doris/flink/autoci/e2e/AbstractE2EService.java | 3 ++-
.../e2e/{Doris2DorisService.java => Doris2DorisE2ECase.java} | 4 ++--
.../e2e/{Mysql2DorisService.java => Mysql2DorisE2ECase.java} | 4 ++--
6 files changed, 9 insertions(+), 12 deletions(-)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/{Doris2DorisService.java => Doris2DorisE2ECase.java} (98%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/{Mysql2DorisService.java => Mysql2DorisE2ECase.java} (99%)
diff --git a/.github/workflows/run-e2ecase.yml b/.github/workflows/run-e2ecase.yml
index 189173ca1..e6318f8f9 100644
--- a/.github/workflows/run-e2ecase.yml
+++ b/.github/workflows/run-e2ecase.yml
@@ -40,5 +40,5 @@ jobs:
- name: Run E2ECases
run: |
- cd flink-doris-connector && mvn test -Dtest="*DorisService" -Dimage="apache/doris:doris-all-in-one-2.1.0"
+ cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="apache/doris:doris-all-in-one-2.1.0"
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
index 6d36d9ed5..059161a25 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
@@ -32,12 +32,13 @@ public abstract class AbstractAutoCITestBase {
@BeforeClass
public static void initContainers() {
- LOG.info("Starting to init auto ci containers.");
+ LOG.info("Trying to start auto ci containers.");
initDorisContainer();
}
private static void initDorisContainer() {
if (Objects.nonNull(dorisContainerService)) {
+ LOG.info("The doris container has been started and will be used directly.");
return;
}
dorisContainerService = new DorisContainerService();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/DorisContainerService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/DorisContainerService.java
index 0740b45cf..6f2e8802f 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/DorisContainerService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/DorisContainerService.java
@@ -96,11 +96,6 @@ public void startContainer() {
throw new DorisRuntimeException("Failed to start containers doris", ex);
}
LOG.info("Doris container started successfully.");
- try {
- Thread.sleep(20000);
- } catch (InterruptedException e) {
- throw new DorisRuntimeException(e);
- }
}
@Override
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
index 3e5ada554..708ad3a09 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
@@ -55,12 +55,13 @@ public abstract class AbstractE2EService extends AbstractAutoCITestBase {
@BeforeClass
public static void initE2EContainers() {
- LOG.info("Starting to init E2E containers.");
+ LOG.info("Trying to Start init E2E containers.");
initMySQLContainer();
}
private static void initMySQLContainer() {
if (Objects.nonNull(mysqlContainerService)) {
+ LOG.info("The MySQL container has been started and will be used directly.");
return;
}
mysqlContainerService = new MySQLContainerService();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisE2ECase.java
similarity index 98%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisE2ECase.java
index f9aed1250..3d6a3d2f2 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisE2ECase.java
@@ -33,8 +33,8 @@
import java.util.List;
import java.util.UUID;
-public class Doris2DorisService extends AbstractE2EService {
- private static final Logger LOG = LoggerFactory.getLogger(Doris2DorisService.class);
+public class Doris2DorisE2ECase extends AbstractE2EService {
+ private static final Logger LOG = LoggerFactory.getLogger(Doris2DorisE2ECase.class);
private static final String DATABASE_SOURCE = "test_doris2doris_source";
private static final String DATABASE_SINK = "test_doris2doris_sink";
private static final String TABLE = "test_tbl";
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisE2ECase.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisE2ECase.java
index 26e425156..fae741bcd 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisE2ECase.java
@@ -27,8 +27,8 @@
import java.util.Arrays;
import java.util.List;
-public class Mysql2DorisService extends AbstractE2EService {
- private static final Logger LOG = LoggerFactory.getLogger(Mysql2DorisService.class);
+public class Mysql2DorisE2ECase extends AbstractE2EService {
+ private static final Logger LOG = LoggerFactory.getLogger(Mysql2DorisE2ECase.class);
private static final String DATABASE = "test_e2e_mysql";
private static final String CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS " + DATABASE;
private static final String MYSQL_CONF = "--" + DatabaseSyncConfig.MYSQL_CONF;
From d5f7a4a11a5a2875f9688f6bbc413c9bd39b02e9 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 15 Aug 2024 17:07:29 +0800
Subject: [PATCH 11/48] fix
---
.../doris/flink/{autoci => }/DorisTestBase.java | 2 +-
.../itcase => catalog}/DorisCatalogITCase.java | 12 +++++++++---
.../{autoci/itcase => sink}/DorisSinkITCase.java | 5 ++---
.../itcase => sink/schema}/SchemaManagerITCase.java | 5 ++---
.../{autoci/itcase => source}/DorisSourceITCase.java | 5 ++---
.../DorisRowDataJdbcLookupFunctionITCase.java | 5 ++---
6 files changed, 18 insertions(+), 16 deletions(-)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci => }/DorisTestBase.java (99%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/itcase => catalog}/DorisCatalogITCase.java (98%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/itcase => sink}/DorisSinkITCase.java (99%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/itcase => sink/schema}/SchemaManagerITCase.java (98%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/itcase => source}/DorisSourceITCase.java (99%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/itcase => table}/DorisRowDataJdbcLookupFunctionITCase.java (97%)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/DorisTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/DorisTestBase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
index 0e214788b..5097a2119 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/DorisTestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci;
+package org.apache.doris.flink;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisCatalogITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
similarity index 98%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisCatalogITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
index 779e081f3..712b1881b 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisCatalogITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.itcase;
+package org.apache.doris.flink.catalog;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.EnvironmentSettings;
@@ -40,8 +40,7 @@
import org.apache.flink.util.CollectionUtil;
import com.google.common.collect.Lists;
-import org.apache.doris.flink.autoci.DorisTestBase;
-import org.apache.doris.flink.catalog.DorisCatalog;
+import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisConnectionOptions;
import org.junit.Assert;
import org.junit.Before;
@@ -184,6 +183,13 @@ public void setup()
catalog.createTable(new ObjectPath(TEST_DB, TEST_TABLE_SINK_GROUPBY), createTable(), true);
}
+ @Test
+ @Ignore
+ public void testQueryFenodes() {
+ String actual = catalog.queryFenodes();
+ assertEquals(getFenodes(), actual);
+ }
+
@Test
public void testListDatabases() {
List actual = catalog.listDatabases();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSinkITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSinkITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
index 0576e03ac..7ca11b867 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSinkITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.itcase;
+package org.apache.doris.flink.sink;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.RuntimeExecutionMode;
@@ -26,11 +26,10 @@
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.doris.flink.autoci.DorisTestBase;
+import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
-import org.apache.doris.flink.sink.DorisSink;
import org.apache.doris.flink.sink.DorisSink.Builder;
import org.apache.doris.flink.sink.batch.DorisBatchSink;
import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/SchemaManagerITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
similarity index 98%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/SchemaManagerITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
index 9466cd52a..3dde08d51 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/SchemaManagerITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.itcase;
+package org.apache.doris.flink.sink.schema;
-import org.apache.doris.flink.autoci.DorisTestBase;
+import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.catalog.doris.FieldSchema;
import org.apache.doris.flink.catalog.doris.TableSchema;
@@ -25,7 +25,6 @@
import org.apache.doris.flink.exception.IllegalArgumentException;
import org.apache.doris.flink.rest.models.Field;
import org.apache.doris.flink.rest.models.Schema;
-import org.apache.doris.flink.sink.schema.SchemaChangeManager;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSourceITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSourceITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
index 87a9b6a6a..a13e96f7f 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisSourceITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.itcase;
+package org.apache.doris.flink.source;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
@@ -25,12 +25,11 @@
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
-import org.apache.doris.flink.autoci.DorisTestBase;
+import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisStreamOptions;
import org.apache.doris.flink.datastream.DorisSourceFunction;
import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
-import org.apache.doris.flink.source.DorisSource;
import org.junit.Assert;
import org.junit.Test;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisRowDataJdbcLookupFunctionITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
similarity index 97%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisRowDataJdbcLookupFunctionITCase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
index 5f4467d6f..0ad1781ae 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/itcase/DorisRowDataJdbcLookupFunctionITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.itcase;
+package org.apache.doris.flink.table;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.data.GenericRowData;
@@ -26,10 +26,9 @@
import org.apache.flink.util.Collector;
import com.google.common.cache.Cache;
-import org.apache.doris.flink.autoci.DorisTestBase;
+import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.table.DorisRowDataJdbcLookupFunction;
import org.junit.Before;
import org.junit.Test;
From fe596b38c6b05e0a80d2d9f2848be7c1889d1b3d Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Thu, 22 Aug 2024 12:22:51 +0800
Subject: [PATCH 12/48] fix
---
.../flink/catalog/DorisCatalogITCase.java | 46 ++---
.../AbstractContainerTestBase.java} | 12 +-
.../e2e => container}/AbstractE2EService.java | 12 +-
.../container/AbstractITCaseService.java | 133 +++++++++++++++
.../ContainerUtils.java} | 18 +-
.../flink/{ => container}/DorisTestBase.java | 2 +-
.../e2e/CustomerSingleThreadExecutor.java | 2 +-
.../e2e/Doris2DorisE2ECase.java | 12 +-
.../e2e/Mysql2DorisE2ECase.java | 64 +++----
.../instance}/ContainerService.java | 2 +-
.../instance/DorisContainer.java} | 11 +-
.../instance/MySQLContainer.java} | 17 +-
.../doris/flink/sink/DorisSinkITCase.java | 158 ++++++++----------
.../sink/schema/SchemaManagerITCase.java | 65 ++++---
.../doris/flink/source/DorisSourceITCase.java | 100 +++++------
.../DorisRowDataJdbcLookupFunctionITCase.java | 72 ++++----
.../e2e/mysql2doris/testAutoAddTable.txt | 1 -
.../mysql2doris/testMySQL2DorisByDefault.txt | 3 +-
.../mysql2doris/testMySQL2DorisSQLParse.txt | 1 -
19 files changed, 410 insertions(+), 321 deletions(-)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/AbstractAutoCITestBase.java => container/AbstractContainerTestBase.java} (89%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/e2e => container}/AbstractE2EService.java (93%)
create mode 100644 flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/e2e/E2EContainerUtils.java => container/ContainerUtils.java} (86%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{ => container}/DorisTestBase.java (99%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci => container}/e2e/CustomerSingleThreadExecutor.java (99%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci => container}/e2e/Doris2DorisE2ECase.java (94%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci => container}/e2e/Mysql2DorisE2ECase.java (87%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/container => container/instance}/ContainerService.java (96%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/container/DorisContainerService.java => container/instance/DorisContainer.java} (96%)
rename flink-doris-connector/src/test/java/org/apache/doris/flink/{autoci/container/MySQLContainerService.java => container/instance/MySQLContainer.java} (85%)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
index 712b1881b..b3a3ce04f 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
@@ -40,8 +40,8 @@
import org.apache.flink.util.CollectionUtil;
import com.google.common.collect.Lists;
-import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisConnectionOptions;
+import org.apache.doris.flink.container.AbstractITCaseService;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
@@ -62,40 +62,14 @@
import static org.junit.Assert.assertTrue;
/** Class for unit tests to run on catalogs. */
-public class DorisCatalogITCase extends DorisTestBase {
+public class DorisCatalogITCase extends AbstractITCaseService {
private static final String TEST_CATALOG_NAME = "doris_catalog";
- private static final String TEST_FENODES = getFenodes();
- private static final String TEST_JDBCURL = getJdbcUrl();
- private static final String TEST_USERNAME = USERNAME;
- private static final String TEST_PWD = PASSWORD;
- // private static final String TEST_FENODES = "127.0.0.1:8030";
- // private static final String TEST_JDBCURL = "jdbc:mysql://127.0.0.1:9030";
- // private static final String TEST_USERNAME = "root";
- // private static final String TEST_PWD = "";
private static final String TEST_DB = "catalog_db";
private static final String TEST_TABLE = "t_all_types";
private static final String TEST_TABLE_SINK = "t_all_types_sink";
private static final String TEST_TABLE_SINK_GROUPBY = "t_all_types_sink_groupby";
- protected static final Schema TABLE_SCHEMA =
- Schema.newBuilder()
- .column("id", DataTypes.STRING())
- .column("c_boolean", DataTypes.BOOLEAN())
- .column("c_char", DataTypes.CHAR(1))
- .column("c_date", DataTypes.DATE())
- .column("c_datetime", DataTypes.TIMESTAMP(0))
- .column("c_decimal", DataTypes.DECIMAL(10, 2))
- .column("c_double", DataTypes.DOUBLE())
- .column("c_float", DataTypes.FLOAT())
- .column("c_int", DataTypes.INT())
- .column("c_bigint", DataTypes.BIGINT())
- .column("c_largeint", DataTypes.STRING())
- .column("c_smallint", DataTypes.SMALLINT())
- .column("c_string", DataTypes.STRING())
- .column("c_tinyint", DataTypes.TINYINT())
- .build();
-
- protected static final TableSchema TABLE_SCHEMA_1 =
+ private static final TableSchema TABLE_SCHEMA =
TableSchema.builder()
.field("id", new AtomicDataType(new VarCharType(false, 128)))
.field("c_boolean", DataTypes.BOOLEAN())
@@ -162,10 +136,10 @@ public void setup()
TableNotExistException, DatabaseNotExistException {
DorisConnectionOptions connectionOptions =
new DorisConnectionOptions.DorisConnectionOptionsBuilder()
- .withFenodes(TEST_FENODES)
- .withJdbcUrl(TEST_JDBCURL)
- .withUsername(TEST_USERNAME)
- .withPassword(TEST_PWD)
+ .withFenodes(getFenodes())
+ .withJdbcUrl(getDorisQueryUrl())
+ .withUsername(getDorisUsername())
+ .withPassword(getDorisPassword())
.build();
Map props = new HashMap<>();
@@ -272,7 +246,7 @@ public void testGetTable() throws TableNotExistException {
CatalogBaseTable table = catalog.getTable(new ObjectPath(TEST_DB, TEST_TABLE));
Schema actual = table.getUnresolvedSchema();
assertEquals(
- TABLE_SCHEMA_1.getFieldNames(),
+ TABLE_SCHEMA.getFieldNames(),
actual.getColumns().stream().map(Schema.UnresolvedColumn::getName).toArray());
}
@@ -308,7 +282,7 @@ public void testCreateTableDbNoExists()
public void testCreateTable() throws TableAlreadyExistException, DatabaseNotExistException {
CatalogTableImpl catalogTable =
new CatalogTableImpl(
- TABLE_SCHEMA_1,
+ TABLE_SCHEMA,
new HashMap() {
{
put("connector", "doris-1");
@@ -425,7 +399,7 @@ private static CatalogDatabase createDb() {
private static CatalogTable createTable() {
return new CatalogTableImpl(
- TABLE_SCHEMA_1,
+ TABLE_SCHEMA,
new HashMap() {
{
put("connector", "doris");
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
similarity index 89%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
index 059161a25..4d2cfac30 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/AbstractAutoCITestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci;
+package org.apache.doris.flink.container;
-import org.apache.doris.flink.autoci.container.ContainerService;
-import org.apache.doris.flink.autoci.container.DorisContainerService;
+import org.apache.doris.flink.container.instance.ContainerService;
+import org.apache.doris.flink.container.instance.DorisContainer;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -26,8 +26,8 @@
import java.sql.Connection;
import java.util.Objects;
-public abstract class AbstractAutoCITestBase {
- private static final Logger LOG = LoggerFactory.getLogger(AbstractAutoCITestBase.class);
+public abstract class AbstractContainerTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractContainerTestBase.class);
private static ContainerService dorisContainerService;
@BeforeClass
@@ -41,7 +41,7 @@ private static void initDorisContainer() {
LOG.info("The doris container has been started and will be used directly.");
return;
}
- dorisContainerService = new DorisContainerService();
+ dorisContainerService = new DorisContainer();
dorisContainerService.startContainer();
LOG.info("Doris container was started.");
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
similarity index 93%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
index 708ad3a09..532d153cd 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/AbstractE2EService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
@@ -15,15 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.e2e;
+package org.apache.doris.flink.container;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.doris.flink.autoci.AbstractAutoCITestBase;
-import org.apache.doris.flink.autoci.container.ContainerService;
-import org.apache.doris.flink.autoci.container.MySQLContainerService;
+import org.apache.doris.flink.container.e2e.CustomerSingleThreadExecutor;
+import org.apache.doris.flink.container.instance.ContainerService;
+import org.apache.doris.flink.container.instance.MySQLContainer;
import org.apache.doris.flink.exception.DorisRuntimeException;
import org.apache.doris.flink.tools.cdc.CdcTools;
import org.apache.doris.flink.tools.cdc.DatabaseSyncConfig;
@@ -37,7 +37,7 @@
import java.util.Map;
import java.util.Objects;
-public abstract class AbstractE2EService extends AbstractAutoCITestBase {
+public abstract class AbstractE2EService extends AbstractContainerTestBase {
private static final Logger LOG = LoggerFactory.getLogger(AbstractE2EService.class);
private static ContainerService mysqlContainerService;
private static final CustomerSingleThreadExecutor singleThreadExecutor =
@@ -64,7 +64,7 @@ private static void initMySQLContainer() {
LOG.info("The MySQL container has been started and will be used directly.");
return;
}
- mysqlContainerService = new MySQLContainerService();
+ mysqlContainerService = new MySQLContainer();
mysqlContainerService.startContainer();
LOG.info("Mysql container was started.");
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
new file mode 100644
index 000000000..1b5b3114b
--- /dev/null
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.container;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.runtime.highavailability.nonha.embedded.HaLeadershipControl;
+import org.apache.flink.runtime.minicluster.MiniCluster;
+import org.apache.flink.runtime.minicluster.RpcServiceSharing;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.test.util.MiniClusterWithClientResource;
+import org.apache.flink.util.function.SupplierWithException;
+
+import org.junit.Rule;
+
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+public abstract class AbstractITCaseService extends AbstractContainerTestBase {
+
+ protected static void waitForJobStatus(
+ JobClient client, List expectedStatus, Deadline deadline) throws Exception {
+ waitUntilCondition(
+ () -> {
+ JobStatus currentStatus = (JobStatus) client.getJobStatus().get();
+ if (expectedStatus.contains(currentStatus)) {
+ return true;
+ } else if (currentStatus.isTerminalState()) {
+ try {
+ client.getJobExecutionResult().get();
+ } catch (Exception var4) {
+ throw new IllegalStateException(
+ String.format(
+ "Job has entered %s state, but expecting %s",
+ currentStatus, expectedStatus),
+ var4);
+ }
+
+ throw new IllegalStateException(
+ String.format(
+ "Job has entered a terminal state %s, but expecting %s",
+ currentStatus, expectedStatus));
+ } else {
+ return false;
+ }
+ },
+ deadline,
+ 100L,
+ "Condition was not met in given timeout.");
+ }
+
+ protected static void waitUntilCondition(
+ SupplierWithException condition,
+ Deadline timeout,
+ long retryIntervalMillis,
+ String errorMsg)
+ throws Exception {
+ while (timeout.hasTimeLeft() && !(Boolean) condition.get()) {
+ long timeLeft = Math.max(0L, timeout.timeLeft().toMillis());
+ Thread.sleep(Math.min(retryIntervalMillis, timeLeft));
+ }
+
+ if (!timeout.hasTimeLeft()) {
+ throw new TimeoutException(errorMsg);
+ }
+ }
+
+ @Rule
+ public final MiniClusterWithClientResource miniClusterResource =
+ new MiniClusterWithClientResource(
+ new MiniClusterResourceConfiguration.Builder()
+ .setNumberTaskManagers(1)
+ .setNumberSlotsPerTaskManager(2)
+ .setRpcServiceSharing(RpcServiceSharing.DEDICATED)
+ .withHaLeadershipControl()
+ .build());
+
+ /** The type of failover. */
+ protected enum FailoverType {
+ TM,
+ JM,
+ NONE
+ }
+
+ protected static void triggerFailover(
+ FailoverType type, JobID jobId, MiniCluster miniCluster, Runnable afterFailAction)
+ throws Exception {
+ switch (type) {
+ case TM:
+ restartTaskManager(miniCluster, afterFailAction);
+ break;
+ case JM:
+ triggerJobManagerFailover(jobId, miniCluster, afterFailAction);
+ break;
+ case NONE:
+ break;
+ default:
+ throw new IllegalStateException("Unexpected value: " + type);
+ }
+ }
+
+ protected static void restartTaskManager(MiniCluster miniCluster, Runnable afterFailAction)
+ throws Exception {
+ miniCluster.terminateTaskManager(0).get();
+ afterFailAction.run();
+ miniCluster.startTaskManager();
+ }
+
+ protected static void triggerJobManagerFailover(
+ JobID jobId, MiniCluster miniCluster, Runnable afterFailAction) throws Exception {
+ final HaLeadershipControl haLeadershipControl = miniCluster.getHaLeadershipControl().get();
+ haLeadershipControl.revokeJobMasterLeadership(jobId).get();
+ afterFailAction.run();
+ haLeadershipControl.grantJobMasterLeadership(jobId).get();
+ }
+}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
similarity index 86%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
index 210f527b3..38520e8cc 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/E2EContainerUtils.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.e2e;
+package org.apache.doris.flink.container;
import org.apache.commons.lang3.StringUtils;
import org.apache.doris.flink.exception.DorisRuntimeException;
@@ -36,9 +36,9 @@
import java.util.Objects;
import java.util.stream.Collectors;
-public class E2EContainerUtils {
+public class ContainerUtils {
- protected static void executeSQLStatement(Connection connection, Logger logger, String... sql) {
+ public static void executeSQLStatement(Connection connection, Logger logger, String... sql) {
if (Objects.isNull(sql) || sql.length == 0) {
return;
}
@@ -54,9 +54,9 @@ protected static void executeSQLStatement(Connection connection, Logger logger,
}
}
- protected static String loadFileContent(String resourcePath) {
+ public static String loadFileContent(String resourcePath) {
try (InputStream stream =
- E2EContainerUtils.class.getClassLoader().getResourceAsStream(resourcePath)) {
+ ContainerUtils.class.getClassLoader().getResourceAsStream(resourcePath)) {
return new BufferedReader(new InputStreamReader(Objects.requireNonNull(stream)))
.lines()
.collect(Collectors.joining("\n"));
@@ -65,15 +65,15 @@ protected static String loadFileContent(String resourcePath) {
}
}
- protected static List parseFileArgs(String resourcePath) {
- String fileContent = E2EContainerUtils.loadFileContent(resourcePath);
+ public static List parseFileArgs(String resourcePath) {
+ String fileContent = ContainerUtils.loadFileContent(resourcePath);
String[] args = fileContent.split("\n");
List argList = new ArrayList<>();
for (String arg : args) {
String[] split = arg.trim().split("\\s+");
List stringList =
Arrays.stream(split)
- .map(E2EContainerUtils::removeQuotes)
+ .map(ContainerUtils::removeQuotes)
.collect(Collectors.toList());
argList.addAll(stringList);
}
@@ -93,7 +93,7 @@ private static String removeQuotes(String str) {
return str;
}
- protected static String[] parseFileContentSQL(String resourcePath) {
+ public static String[] parseFileContentSQL(String resourcePath) {
String fileContent = loadFileContent(resourcePath);
return Arrays.stream(fileContent.split(";")).map(String::trim).toArray(String[]::new);
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/DorisTestBase.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/DorisTestBase.java
index 5097a2119..fc5daa2d5 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisTestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/DorisTestBase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink;
+package org.apache.doris.flink.container;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/CustomerSingleThreadExecutor.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/CustomerSingleThreadExecutor.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/CustomerSingleThreadExecutor.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/CustomerSingleThreadExecutor.java
index 066d9a44b..1ae640175 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/CustomerSingleThreadExecutor.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/CustomerSingleThreadExecutor.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.e2e;
+package org.apache.doris.flink.container.e2e;
import org.apache.doris.flink.exception.DorisRuntimeException;
import org.slf4j.Logger;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
similarity index 94%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisE2ECase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
index 3d6a3d2f2..bd267df27 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Doris2DorisE2ECase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.e2e;
+package org.apache.doris.flink.container.e2e;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -24,6 +24,8 @@
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
+import org.apache.doris.flink.container.AbstractE2EService;
+import org.apache.doris.flink.container.ContainerUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -143,13 +145,13 @@ public void testDoris2Doris() throws Exception {
private void initializeDorisTable() {
String[] sourceInitSql =
- E2EContainerUtils.parseFileContentSQL(
+ ContainerUtils.parseFileContentSQL(
"autoci/e2e/doris2doris/initialize/test_doris2doris_source_test_tbl.sql");
- E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sourceInitSql);
+ ContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sourceInitSql);
String[] sinkInitSql =
- E2EContainerUtils.parseFileContentSQL(
+ ContainerUtils.parseFileContentSQL(
"autoci/e2e/doris2doris/initialize/test_doris2doris_sink_test_tbl.sql");
- E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sinkInitSql);
+ ContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, sinkInitSql);
LOG.info("Initialization of doris table successful.");
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
similarity index 87%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisE2ECase.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
index fae741bcd..7992f656d 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/e2e/Mysql2DorisE2ECase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
@@ -15,8 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.e2e;
+package org.apache.doris.flink.container.e2e;
+import org.apache.doris.flink.container.AbstractE2EService;
+import org.apache.doris.flink.container.ContainerUtils;
import org.apache.doris.flink.exception.DorisRuntimeException;
import org.apache.doris.flink.tools.cdc.DatabaseSyncConfig;
import org.junit.Before;
@@ -37,37 +39,37 @@ public class Mysql2DorisE2ECase extends AbstractE2EService {
public void initialize() {
// init mysql table
LOG.info("start to init mysql table.");
- E2EContainerUtils.executeSQLStatement(getMySQLQueryConnection(), LOG, CREATE_DATABASE);
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(getMySQLQueryConnection(), LOG, CREATE_DATABASE);
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
- E2EContainerUtils.parseFileContentSQL(
+ ContainerUtils.parseFileContentSQL(
"autoci/e2e/mysql2doris/initialize/mysql_tbl1.sql"));
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
- E2EContainerUtils.parseFileContentSQL(
+ ContainerUtils.parseFileContentSQL(
"autoci/e2e/mysql2doris/initialize/mysql_tbl2.sql"));
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
- E2EContainerUtils.parseFileContentSQL(
+ ContainerUtils.parseFileContentSQL(
"autoci/e2e/mysql2doris/initialize/mysql_tbl3.sql"));
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
- E2EContainerUtils.parseFileContentSQL(
+ ContainerUtils.parseFileContentSQL(
"autoci/e2e/mysql2doris/initialize/mysql_tbl4.sql"));
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
- E2EContainerUtils.parseFileContentSQL(
+ ContainerUtils.parseFileContentSQL(
"autoci/e2e/mysql2doris/initialize/mysql_tbl5.sql"));
// init doris table
LOG.info("start to init doris table.");
- E2EContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, CREATE_DATABASE);
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(getDorisQueryConnection(), LOG, CREATE_DATABASE);
+ ContainerUtils.executeSQLStatement(
getDorisQueryConnection(),
LOG,
"DROP TABLE IF EXISTS test_e2e_mysql.tbl1",
@@ -99,7 +101,7 @@ private List setMysql2DorisDefaultConfig(List argList) {
private void startMysql2DorisJob(String jobName, String resourcePath) {
LOG.info("start a mysql to doris job. jobName={}, resourcePath={}", jobName, resourcePath);
- List argList = E2EContainerUtils.parseFileArgs(resourcePath);
+ List argList = ContainerUtils.parseFileArgs(resourcePath);
String[] args = setMysql2DorisDefaultConfig(argList).toArray(new String[0]);
submitE2EJob(jobName, args);
verifyInitializeResult();
@@ -114,7 +116,7 @@ private void verifyInitializeResult() {
Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3", "doris_5,5");
String sql1 =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.tbl5) res order by 1";
- E2EContainerUtils.checkResult(getDorisQueryConnection(), expected, sql1, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, sql1, 2);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@@ -141,7 +143,7 @@ public void testAutoAddTable() throws InterruptedException {
// auto add table
LOG.info("starting to create auto_add table.");
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"CREATE TABLE test_e2e_mysql.auto_add ( \n"
@@ -153,12 +155,12 @@ public void testAutoAddTable() throws InterruptedException {
Thread.sleep(20000);
List autoAddResult = Arrays.asList("doris_4_1,4", "doris_4_2,4");
String autoAddSql = "select * from test_e2e_mysql.auto_add order by 1";
- E2EContainerUtils.checkResult(getDorisQueryConnection(), autoAddResult, autoAddSql, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), autoAddResult, autoAddSql, 2);
// incremental data
LOG.info("starting to increment data.");
addIncrementalData();
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"insert into test_e2e_mysql.auto_add values ('doris_4_3',43)",
@@ -176,12 +178,12 @@ public void testAutoAddTable() throws InterruptedException {
"doris_4_3,43");
String incrementDataSql =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.auto_add) res order by 1";
- E2EContainerUtils.checkResult(
+ ContainerUtils.checkResult(
getDorisQueryConnection(), incrementDataExpected, incrementDataSql, 2);
// schema change
LOG.info("starting to mock schema change.");
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"alter table test_e2e_mysql.auto_add add column c1 varchar(128)",
@@ -191,7 +193,7 @@ public void testAutoAddTable() throws InterruptedException {
List schemaChangeExpected =
Arrays.asList("doris_4_1,null", "doris_4_3,null", "doris_4_4,c1_val");
String schemaChangeSql = "select * from test_e2e_mysql.auto_add order by 1";
- E2EContainerUtils.checkResult(
+ ContainerUtils.checkResult(
getDorisQueryConnection(), schemaChangeExpected, schemaChangeSql, 2);
cancelCurrentE2EJob(jobName);
}
@@ -210,7 +212,7 @@ public void testMySQL2DorisSQLParse() throws Exception {
// mock create table
LOG.info("start to create table in mysql.");
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"CREATE TABLE test_e2e_mysql.add_tbl (\n"
@@ -223,7 +225,7 @@ public void testMySQL2DorisSQLParse() throws Exception {
Thread.sleep(20000);
List createTableExpected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3");
String createTableSql = "select * from test_e2e_mysql.add_tbl order by 1";
- E2EContainerUtils.checkResult(
+ ContainerUtils.checkResult(
getDorisQueryConnection(), createTableExpected, createTableSql, 2);
cancelCurrentE2EJob(jobName);
}
@@ -232,13 +234,13 @@ private void tbl1SchemaChange() {
// mock schema change
LOG.info("start to schema change in mysql.");
try {
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"alter table test_e2e_mysql.tbl1 add column c1 varchar(128)",
"alter table test_e2e_mysql.tbl1 drop column age");
Thread.sleep(20000);
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"insert into test_e2e_mysql.tbl1 values ('doris_1_1_1','c1_val')");
@@ -253,14 +255,14 @@ private void verifyTbl1SchemaChange() {
List schemaChangeExpected =
Arrays.asList("doris_1,null", "doris_1_1,null", "doris_1_1_1,c1_val");
String schemaChangeSql = "select * from test_e2e_mysql.tbl1 order by 1";
- E2EContainerUtils.checkResult(
+ ContainerUtils.checkResult(
getDorisQueryConnection(), schemaChangeExpected, schemaChangeSql, 2);
}
private void addIncrementalData() {
// add incremental data
try {
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"insert into test_e2e_mysql.tbl1 values ('doris_1_1',10)",
@@ -281,7 +283,7 @@ private void verifyIncrementalDataResult() {
"doris_1,18", "doris_1_1,10", "doris_2_1,11", "doris_3,3", "doris_3_1,12");
String sql2 =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 ) res order by 1";
- E2EContainerUtils.checkResult(getDorisQueryConnection(), expected2, sql2, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected2, sql2, 2);
}
@Test
@@ -300,7 +302,7 @@ public void testMySQL2DorisEnableDelete() throws Exception {
startMysql2DorisJob(jobName, "autoci/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt");
addIncrementalData();
- E2EContainerUtils.executeSQLStatement(
+ ContainerUtils.executeSQLStatement(
getMySQLQueryConnection(),
LOG,
"delete from test_e2e_mysql.tbl3 where name='doris_3'",
@@ -318,7 +320,7 @@ public void testMySQL2DorisEnableDelete() throws Exception {
"doris_5,5");
String sql =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.tbl5) res order by 1";
- E2EContainerUtils.checkResult(getDorisQueryConnection(), expected, sql, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, sql, 2);
cancelCurrentE2EJob(jobName);
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/ContainerService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
similarity index 96%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/ContainerService.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
index 923118655..31edec40d 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/ContainerService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.container;
+package org.apache.doris.flink.container.instance;
import org.apache.doris.flink.exception.DorisRuntimeException;
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/DorisContainerService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
similarity index 96%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/DorisContainerService.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
index 6f2e8802f..5b5a36510 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/DorisContainerService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.container;
+package org.apache.doris.flink.container.instance;
import com.google.common.collect.Lists;
import org.apache.doris.flink.exception.DorisRuntimeException;
@@ -45,8 +45,8 @@
import java.util.Map;
import java.util.concurrent.locks.LockSupport;
-public class DorisContainerService implements ContainerService {
- private static final Logger LOG = LoggerFactory.getLogger(DorisContainerService.class);
+public class DorisContainer implements ContainerService {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisContainer.class);
private static final String DEFAULT_DOCKER_IMAGE = "apache/doris:doris-all-in-one-2.1.0";
private static final String DORIS_DOCKER_IMAGE =
System.getProperty("image") == null
@@ -59,7 +59,7 @@ public class DorisContainerService implements ContainerService {
private static final String PASSWORD = "";
private final GenericContainer dorisContainer;
- public DorisContainerService() {
+ public DorisContainer() {
dorisContainer = createDorisContainer();
}
@@ -149,8 +149,7 @@ public void close() {
private void initializeJDBCDriver() throws MalformedURLException {
URLClassLoader urlClassLoader =
new URLClassLoader(
- new URL[] {new URL(DRIVER_JAR)},
- DorisContainerService.class.getClassLoader());
+ new URL[] {new URL(DRIVER_JAR)}, DorisContainer.class.getClassLoader());
LOG.info("Try to connect to Doris.");
Thread.currentThread().setContextClassLoader(urlClassLoader);
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/MySQLContainerService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
similarity index 85%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/MySQLContainerService.java
rename to flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
index 883f6a46a..d7ae0b57a 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/autoci/container/MySQLContainerService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
@@ -15,12 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.flink.autoci.container;
+package org.apache.doris.flink.container.instance;
import org.apache.doris.flink.exception.DorisRuntimeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.lifecycle.Startables;
import java.sql.Connection;
@@ -28,20 +27,22 @@
import java.sql.SQLException;
import java.util.stream.Stream;
-public class MySQLContainerService implements ContainerService {
- private static final Logger LOG = LoggerFactory.getLogger(MySQLContainerService.class);
+public class MySQLContainer implements ContainerService {
+ private static final Logger LOG = LoggerFactory.getLogger(MySQLContainer.class);
private static final String MYSQL_VERSION = "mysql:8.0";
private static final String USERNAME = "root";
private static final String PASSWORD = "123456";
- private final MySQLContainer mysqlcontainer;
+ private final org.testcontainers.containers.MySQLContainer mysqlcontainer;
- public MySQLContainerService() {
+ public MySQLContainer() {
mysqlcontainer = createContainer();
}
- private MySQLContainer createContainer() {
+ private org.testcontainers.containers.MySQLContainer createContainer() {
LOG.info("Will create mysql container.");
- return new MySQLContainer(MYSQL_VERSION).withUsername(USERNAME).withPassword(PASSWORD);
+ return new org.testcontainers.containers.MySQLContainer(MYSQL_VERSION)
+ .withUsername(USERNAME)
+ .withPassword(PASSWORD);
}
@Override
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
index 7ca11b867..b820ef030 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
@@ -26,19 +26,19 @@
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.container.AbstractITCaseService;
+import org.apache.doris.flink.container.ContainerUtils;
import org.apache.doris.flink.sink.DorisSink.Builder;
import org.apache.doris.flink.sink.batch.DorisBatchSink;
import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer;
import org.apache.doris.flink.utils.MockSource;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
@@ -52,7 +52,8 @@
import static org.apache.flink.api.common.JobStatus.RUNNING;
/** DorisSink ITCase with csv and arrow format. */
-public class DorisSinkITCase extends DorisTestBase {
+public class DorisSinkITCase extends AbstractITCaseService {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSinkITCase.class);
static final String DATABASE = "test_sink";
static final String TABLE_CSV = "tbl_csv";
static final String TABLE_JSON = "tbl_json";
@@ -71,12 +72,20 @@ public void testSinkCsvFormat() throws Exception {
properties.setProperty("column_separator", ",");
properties.setProperty("line_delimiter", "\n");
properties.setProperty("format", "csv");
- submitJob(TABLE_CSV, properties, new String[] {"doris,1"});
+ DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
+ executionBuilder.setLabelPrefix(UUID.randomUUID().toString()).setStreamLoadProp(properties);
+ DorisOptions.Builder dorisBuilder = DorisOptions.builder();
+ dorisBuilder
+ .setFenodes(getFenodes())
+ .setTableIdentifier(DATABASE + "." + TABLE_CSV)
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword());
+ submitJob(dorisBuilder.build(), executionBuilder.build(), new String[] {"doris,1"});
Thread.sleep(10000);
List expected = Arrays.asList("doris,1");
String query = String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_CSV);
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
@Test
@@ -94,9 +103,18 @@ public void testSinkJsonFormat() throws Exception {
row2.put("name", "doris2");
row2.put("age", 2);
+ DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
+ executionBuilder.setLabelPrefix(UUID.randomUUID().toString()).setStreamLoadProp(properties);
+ DorisOptions.Builder dorisBuilder = DorisOptions.builder();
+ dorisBuilder
+ .setFenodes(getFenodes())
+ .setTableIdentifier(DATABASE + "." + TABLE_JSON)
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword());
+
submitJob(
- TABLE_JSON,
- properties,
+ dorisBuilder.build(),
+ executionBuilder.build(),
new String[] {
new ObjectMapper().writeValueAsString(row1),
new ObjectMapper().writeValueAsString(row2)
@@ -105,28 +123,21 @@ public void testSinkJsonFormat() throws Exception {
Thread.sleep(10000);
List expected = Arrays.asList("doris1,1", "doris2,2");
String query = String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_JSON);
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
- public void submitJob(String table, Properties properties, String[] records) throws Exception {
+ private void submitJob(
+ DorisOptions dorisOptions, DorisExecutionOptions executionOptions, String[] records)
+ throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
Builder builder = DorisSink.builder();
final DorisReadOptions.Builder readOptionBuilder = DorisReadOptions.builder();
- DorisOptions.Builder dorisBuilder = DorisOptions.builder();
- dorisBuilder
- .setFenodes(getFenodes())
- .setTableIdentifier(DATABASE + "." + table)
- .setUsername(USERNAME)
- .setPassword(PASSWORD);
- DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
- executionBuilder.setLabelPrefix(UUID.randomUUID().toString()).setStreamLoadProp(properties);
-
builder.setDorisReadOptions(readOptionBuilder.build())
- .setDorisExecutionOptions(executionBuilder.build())
+ .setDorisExecutionOptions(executionOptions)
.setSerializer(new SimpleStringSerializer())
- .setDorisOptions(dorisBuilder.build());
+ .setDorisOptions(dorisOptions);
env.fromElements(records).sinkTo(builder.build());
env.execute();
@@ -169,8 +180,8 @@ public void testTableSinkJsonFormat() throws Exception {
getFenodes(),
getBenodes(),
DATABASE + "." + TABLE_JSON_TBL,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sinkDDL);
tEnv.executeSql("INSERT INTO doris_sink SELECT 'doris',1 union all SELECT 'flink',2");
@@ -178,7 +189,7 @@ public void testTableSinkJsonFormat() throws Exception {
List expected = Arrays.asList("doris,1", "flink,2");
String query =
String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_JSON_TBL);
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
@Test
@@ -215,8 +226,8 @@ public void testTableBatch() throws Exception {
+ ")",
getFenodes(),
DATABASE + "." + TABLE_CSV_BATCH_TBL,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sinkDDL);
tEnv.executeSql("INSERT INTO doris_sink SELECT 'doris',1 union all SELECT 'flink',2");
@@ -225,7 +236,7 @@ public void testTableBatch() throws Exception {
String query =
String.format(
"select name,age from %s.%s order by 1", DATABASE, TABLE_CSV_BATCH_TBL);
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
@Test
@@ -239,8 +250,8 @@ public void testDataStreamBatch() throws Exception {
dorisBuilder
.setFenodes(getFenodes())
.setTableIdentifier(DATABASE + "." + TABLE_CSV_BATCH_DS)
- .setUsername(USERNAME)
- .setPassword(PASSWORD);
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword());
Properties properties = new Properties();
properties.setProperty("column_separator", ",");
properties.setProperty("line_delimiter", "\n");
@@ -265,7 +276,7 @@ public void testDataStreamBatch() throws Exception {
String query =
String.format(
"select name,age from %s.%s order by 1", DATABASE, TABLE_CSV_BATCH_DS);
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
@Test
@@ -303,8 +314,8 @@ public void testTableGroupCommit() throws Exception {
+ ")",
getFenodes(),
DATABASE + "." + TABLE_GROUP_COMMIT,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sinkDDL);
tEnv.executeSql(
"INSERT INTO doris_group_commit_sink SELECT 'doris',1 union all SELECT 'group_commit',2 union all SELECT 'flink',3");
@@ -314,8 +325,7 @@ public void testTableGroupCommit() throws Exception {
String query =
String.format(
"select name,age from %s.%s order by 1", DATABASE, TABLE_GROUP_COMMIT);
- //
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
@Test
@@ -346,8 +356,8 @@ public void testTableGzFormat() throws Exception {
+ ")",
getFenodes(),
DATABASE + "." + TABLE_GZ_FORMAT,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sinkDDL);
tEnv.executeSql(
"INSERT INTO doris_gz_format_sink SELECT 'doris',1 union all SELECT 'flink',2");
@@ -356,13 +366,13 @@ public void testTableGzFormat() throws Exception {
List expected = Arrays.asList("doris,1", "flink,2");
String query =
String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_GZ_FORMAT);
- //
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
@Test
public void testJobManagerFailoverSink() throws Exception {
- initializeFailoverTable(TABLE_CSV_JM);
+ LOG.info("start to test JobManagerFailoverSink.");
+ initializeTable(TABLE_CSV_JM);
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(2);
env.enableCheckpointing(10000);
@@ -375,8 +385,8 @@ public void testJobManagerFailoverSink() throws Exception {
dorisBuilder
.setFenodes(getFenodes())
.setTableIdentifier(DATABASE + "." + TABLE_CSV_JM)
- .setUsername(USERNAME)
- .setPassword(PASSWORD);
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword());
DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
Properties properties = new Properties();
properties.setProperty("column_separator", ",");
@@ -414,12 +424,13 @@ public void testJobManagerFailoverSink() throws Exception {
Arrays.asList("1,0", "1,1", "2,0", "2,1", "3,0", "3,1", "4,0", "4,1", "5,0", "5,1");
String query =
String.format("select id,task_id from %s.%s order by 1,2", DATABASE, TABLE_CSV_JM);
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
@Test
public void testTaskManagerFailoverSink() throws Exception {
- initializeFailoverTable(TABLE_CSV_TM);
+ LOG.info("start to test TaskManagerFailoverSink.");
+ initializeTable(TABLE_CSV_TM);
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(2);
env.enableCheckpointing(10000);
@@ -432,8 +443,8 @@ public void testTaskManagerFailoverSink() throws Exception {
dorisBuilder
.setFenodes(getFenodes())
.setTableIdentifier(DATABASE + "." + TABLE_CSV_TM)
- .setUsername(USERNAME)
- .setPassword(PASSWORD);
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword());
DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
Properties properties = new Properties();
properties.setProperty("column_separator", ",");
@@ -468,7 +479,7 @@ public void testTaskManagerFailoverSink() throws Exception {
Arrays.asList("1,0", "1,1", "2,0", "2,1", "3,0", "3,1", "4,0", "4,1", "5,0", "5,1");
String query =
String.format("select id,task_id from %s.%s order by 1,2", DATABASE, TABLE_CSV_TM);
- checkResult(expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
}
private void sleepMs(long millis) {
@@ -478,43 +489,20 @@ private void sleepMs(long millis) {
}
}
- private void initializeTable(String table) throws Exception {
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256),\n"
- + "`age` int\n"
- + ") DISTRIBUTED BY HASH(`name`) BUCKETS 1\n"
- + "PROPERTIES (\n"
- + "\"replication_num\" = \"1\"\n"
- + ")\n",
- DATABASE, table));
- }
- }
-
- private void initializeFailoverTable(String table) throws Exception {
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`id` int,\n"
- + "`task_id` int\n"
- + ") DISTRIBUTED BY HASH(`id`) BUCKETS 1\n"
- + "PROPERTIES (\n"
- + "\"replication_num\" = \"1\"\n"
- + ")\n",
- DATABASE, table));
- }
+ private void initializeTable(String table) {
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE),
+ String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table),
+ String.format(
+ "CREATE TABLE %s.%s ( \n"
+ + "`name` varchar(256),\n"
+ + "`age` int\n"
+ + ") DISTRIBUTED BY HASH(`name`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ DATABASE, table));
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
index 3dde08d51..37ca3a2d2 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
@@ -17,31 +17,30 @@
package org.apache.doris.flink.sink.schema;
-import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.catalog.doris.FieldSchema;
import org.apache.doris.flink.catalog.doris.TableSchema;
import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.container.AbstractITCaseService;
+import org.apache.doris.flink.container.ContainerUtils;
import org.apache.doris.flink.exception.IllegalArgumentException;
import org.apache.doris.flink.rest.models.Field;
import org.apache.doris.flink.rest.models.Schema;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
-public class SchemaManagerITCase extends DorisTestBase {
-
+public class SchemaManagerITCase extends AbstractITCaseService {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaManagerITCase.class);
private static final String DATABASE = "test_sc_db";
private DorisOptions options;
private SchemaChangeManager schemaChangeManager;
@@ -52,34 +51,31 @@ public void setUp() throws Exception {
DorisOptions.builder()
.setFenodes(getFenodes())
.setTableIdentifier(DATABASE + ".add_column")
- .setUsername(USERNAME)
- .setPassword(PASSWORD)
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword())
.build();
schemaChangeManager = new SchemaChangeManager(options);
}
- private void initDorisSchemaChangeTable(String table) throws SQLException {
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`id` varchar(32),\n"
- + "`age` int\n"
- + ") DISTRIBUTED BY HASH(`id`) BUCKETS 1\n"
- + "PROPERTIES (\n"
- + "\"replication_num\" = \"1\"\n"
- + ")\n",
- DATABASE, table));
- }
+ private void initDorisSchemaChangeTable(String table) {
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE),
+ String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table),
+ String.format(
+ "CREATE TABLE %s.%s ( \n"
+ + "`id` varchar(32),\n"
+ + "`age` int\n"
+ + ") DISTRIBUTED BY HASH(`id`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ DATABASE, table));
}
@Test
- public void testAddColumn() throws SQLException, IOException, IllegalArgumentException {
+ public void testAddColumn() throws IOException, IllegalArgumentException {
String addColumnTbls = "add_column";
initDorisSchemaChangeTable(addColumnTbls);
FieldSchema field = new FieldSchema("c1", "int", "");
@@ -93,7 +89,7 @@ public void testAddColumn() throws SQLException, IOException, IllegalArgumentExc
@Test
public void testAddColumnWithChineseComment()
- throws SQLException, IOException, IllegalArgumentException, InterruptedException {
+ throws IOException, IllegalArgumentException, InterruptedException {
String addColumnTbls = "add_column";
initDorisSchemaChangeTable(addColumnTbls);
@@ -149,7 +145,7 @@ private String getColumnType(String table, String columnName) {
}
@Test
- public void testDropColumn() throws SQLException, IOException, IllegalArgumentException {
+ public void testDropColumn() throws IOException, IllegalArgumentException {
String dropColumnTbls = "drop_column";
initDorisSchemaChangeTable(dropColumnTbls);
schemaChangeManager.dropColumn(DATABASE, dropColumnTbls, "age");
@@ -161,7 +157,7 @@ public void testDropColumn() throws SQLException, IOException, IllegalArgumentEx
}
@Test
- public void testRenameColumn() throws SQLException, IOException, IllegalArgumentException {
+ public void testRenameColumn() throws IOException, IllegalArgumentException {
String renameColumnTbls = "rename_column";
initDorisSchemaChangeTable(renameColumnTbls);
schemaChangeManager.renameColumn(DATABASE, renameColumnTbls, "age", "age1");
@@ -173,8 +169,7 @@ public void testRenameColumn() throws SQLException, IOException, IllegalArgument
}
@Test
- public void testModifyColumnComment()
- throws SQLException, IOException, IllegalArgumentException {
+ public void testModifyColumnComment() throws IOException, IllegalArgumentException {
String modifyColumnCommentTbls = "modify_column_comment";
initDorisSchemaChangeTable(modifyColumnCommentTbls);
String columnName = "age";
@@ -188,7 +183,7 @@ public void testModifyColumnComment()
@Test
public void testOnlyModifyColumnType()
- throws SQLException, IOException, IllegalArgumentException, InterruptedException {
+ throws IOException, IllegalArgumentException, InterruptedException {
String modifyColumnTbls = "modify_column_type";
String columnName = "age";
String newColumnType = "bigint";
@@ -203,7 +198,7 @@ public void testOnlyModifyColumnType()
@Test
public void testModifyColumnTypeAndComment()
- throws SQLException, IOException, IllegalArgumentException, InterruptedException {
+ throws IOException, IllegalArgumentException, InterruptedException {
String modifyColumnTbls = "modify_column_type_and_comment";
initDorisSchemaChangeTable(modifyColumnTbls);
String columnName = "age";
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
index a13e96f7f..2bfb52dbd 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
@@ -25,33 +25,33 @@
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
-import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisStreamOptions;
+import org.apache.doris.flink.container.AbstractITCaseService;
+import org.apache.doris.flink.container.ContainerUtils;
import org.apache.doris.flink.datastream.DorisSourceFunction;
import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
/** DorisSource ITCase. */
-public class DorisSourceITCase extends DorisTestBase {
- static final String DATABASE = "test_source";
- static final String TABLE_READ = "tbl_read";
- static final String TABLE_READ_OLD_API = "tbl_read_old_api";
- static final String TABLE_READ_TBL = "tbl_read_tbl";
- static final String TABLE_READ_TBL_OLD_API = "tbl_read_tbl_old_api";
- static final String TABLE_READ_TBL_ALL_OPTIONS = "tbl_read_tbl_all_options";
- static final String TABLE_READ_TBL_PUSH_DOWN = "tbl_read_tbl_push_down";
- static final String TABLE_READ_TBL_PUSH_DOWN_WITH_UNION_ALL =
- "tbl_read_tbl_push_down_with_union_all";
+public class DorisSourceITCase extends AbstractITCaseService {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSourceITCase.class);
+ private static final String DATABASE = "test_source";
+ private static final String TABLE_READ = "tbl_read";
+ private static final String TABLE_READ_OLD_API = "tbl_read_old_api";
+ private static final String TABLE_READ_TBL = "tbl_read_tbl";
+ private static final String TABLE_READ_TBL_OLD_API = "tbl_read_tbl_old_api";
+ private static final String TABLE_READ_TBL_ALL_OPTIONS = "tbl_read_tbl_all_options";
+ private static final String TABLE_READ_TBL_PUSH_DOWN = "tbl_read_tbl_push_down";
+ private static final String TABLE_READ_TBL_PUSH_DOWN_WITH_UNION_ALL = "tbl_read_tbl_push_down_with_union_all";
@Test
public void testSource() throws Exception {
@@ -63,8 +63,8 @@ public void testSource() throws Exception {
dorisBuilder
.setFenodes(getFenodes())
.setTableIdentifier(DATABASE + "." + TABLE_READ)
- .setUsername(USERNAME)
- .setPassword(PASSWORD);
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword());
DorisSource> source =
DorisSource.>builder()
@@ -89,8 +89,8 @@ public void testOldSourceApi() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
Properties properties = new Properties();
properties.put("fenodes", getFenodes());
- properties.put("username", USERNAME);
- properties.put("password", PASSWORD);
+ properties.put("username", getDorisUsername());
+ properties.put("password", getDorisPassword());
properties.put("table.identifier", DATABASE + "." + TABLE_READ_OLD_API);
DorisStreamOptions options = new DorisStreamOptions(properties);
@@ -128,7 +128,10 @@ public void testTableSource() throws Exception {
+ " 'username' = '%s',"
+ " 'password' = '%s'"
+ ")",
- getFenodes(), DATABASE + "." + TABLE_READ_TBL, USERNAME, PASSWORD);
+ getFenodes(),
+ DATABASE + "." + TABLE_READ_TBL,
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sourceDDL);
TableResult tableResult = tEnv.executeSql("SELECT * FROM doris_source");
@@ -174,7 +177,10 @@ public void testTableSourceOldApi() throws Exception {
+ " 'username' = '%s',"
+ " 'password' = '%s'"
+ ")",
- getFenodes(), DATABASE + "." + TABLE_READ_TBL_OLD_API, USERNAME, PASSWORD);
+ getFenodes(),
+ DATABASE + "." + TABLE_READ_TBL_OLD_API,
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sourceDDL);
TableResult tableResult = tEnv.executeSql("SELECT * FROM doris_source");
@@ -219,8 +225,8 @@ public void testTableSourceAllOptions() throws Exception {
+ ")",
getFenodes(),
DATABASE + "." + TABLE_READ_TBL_ALL_OPTIONS,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sourceDDL);
TableResult tableResult = tEnv.executeSql("SELECT * FROM doris_source");
@@ -255,8 +261,8 @@ public void testTableSourceFilterAndProjectionPushDown() throws Exception {
+ ")",
getFenodes(),
DATABASE + "." + TABLE_READ_TBL_PUSH_DOWN,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sourceDDL);
TableResult tableResult = tEnv.executeSql("SELECT age FROM doris_source where age = '18'");
@@ -291,8 +297,8 @@ public void testTableSourceFilterWithUnionAll() throws Exception {
+ ")",
getFenodes(),
DATABASE + "." + TABLE_READ_TBL_PUSH_DOWN_WITH_UNION_ALL,
- USERNAME,
- PASSWORD);
+ getDorisUsername(),
+ getDorisPassword());
tEnv.executeSql(sourceDDL);
TableResult tableResult =
tEnv.executeSql(
@@ -310,29 +316,23 @@ public void testTableSourceFilterWithUnionAll() throws Exception {
Assert.assertArrayEquals(expected, actual.toArray());
}
- private void initializeTable(String table) throws Exception {
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE));
- statement.execute(String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table));
- statement.execute(
- String.format(
- "CREATE TABLE %s.%s ( \n"
- + "`name` varchar(256),\n"
- + "`age` int\n"
- + ") DISTRIBUTED BY HASH(`name`) BUCKETS 1\n"
- + "PROPERTIES (\n"
- + "\"replication_num\" = \"1\"\n"
- + ")\n",
- DATABASE, table));
- statement.execute(
- String.format("insert into %s.%s values ('doris',18)", DATABASE, table));
- statement.execute(
- String.format("insert into %s.%s values ('flink',10)", DATABASE, table));
- statement.execute(
- String.format("insert into %s.%s values ('apache',12)", DATABASE, table));
- }
+ private void initializeTable(String table) {
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE),
+ String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table),
+ String.format(
+ "CREATE TABLE %s.%s ( \n"
+ + "`name` varchar(256),\n"
+ + "`age` int\n"
+ + ") DISTRIBUTED BY HASH(`name`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ DATABASE, table),
+ String.format("insert into %s.%s values ('doris',18)", DATABASE, table),
+ String.format("insert into %s.%s values ('flink',10)", DATABASE, table),
+ String.format("insert into %s.%s values ('apache',12)", DATABASE, table));
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
index 0ad1781ae..7f9021f81 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
@@ -26,15 +26,15 @@
import org.apache.flink.util.Collector;
import com.google.common.cache.Cache;
-import org.apache.doris.flink.DorisTestBase;
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.container.AbstractITCaseService;
+import org.apache.doris.flink.container.ContainerUtils;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -43,45 +43,43 @@
import static org.junit.Assert.assertEquals;
-public class DorisRowDataJdbcLookupFunctionITCase extends DorisTestBase {
+public class DorisRowDataJdbcLookupFunctionITCase extends AbstractITCaseService {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DorisRowDataJdbcLookupFunctionITCase.class);
private static final String LOOKUP_TABLE = "test.t_lookup_table";
- private static String[] fieldNames = new String[] {"id1", "id2", "c_string", "c_double"};
- private static DataType[] fieldDataTypes =
+ private static final String[] fieldNames = new String[] {"id1", "id2", "c_string", "c_double"};
+ private static final DataType[] fieldDataTypes =
new DataType[] {
DataTypes.INT(), DataTypes.STRING(), DataTypes.STRING(), DataTypes.DOUBLE()
};
- private static String[] lookupKeys = new String[] {"id1", "id2"};
- private static int[] keyIndexs = new int[] {0, 1};
+ private static final String[] lookupKeys = new String[] {"id1", "id2"};
+ private static final int[] keyIndexs = new int[] {0, 1};
@Before
public void setUp() throws Exception {
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", "test"));
- statement.execute(String.format("DROP TABLE IF EXISTS %s", LOOKUP_TABLE));
- statement.execute(
- String.format(
- "CREATE TABLE %s ( \n"
- + "`id1` int,\n"
- + "`id2` varchar(128),\n"
- + "`c_string` string,\n"
- + "`c_double` double\n"
- + ") DISTRIBUTED BY HASH(`id1`) BUCKETS 1\n"
- + "PROPERTIES (\n"
- + "\"replication_num\" = \"1\"\n"
- + ")\n",
- LOOKUP_TABLE));
- statement.execute(
- String.format(
- "insert into %s values (1,'A','zhangsanA',1.12),"
- + "(1,'A','zhangsanA-1',11.12),"
- + "(2,'B','zhangsanB',2.12),(4,'D','zhangsanD',4.12)",
- LOOKUP_TABLE));
- }
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", "test"),
+ String.format("DROP TABLE IF EXISTS %s", LOOKUP_TABLE),
+ String.format(
+ "CREATE TABLE %s ( \n"
+ + "`id1` int,\n"
+ + "`id2` varchar(128),\n"
+ + "`c_string` string,\n"
+ + "`c_double` double\n"
+ + ") DISTRIBUTED BY HASH(`id1`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ LOOKUP_TABLE),
+ String.format(
+ "insert into %s values (1,'A','zhangsanA',1.12),"
+ + "(1,'A','zhangsanA-1',11.12),"
+ + "(2,'B','zhangsanB',2.12),(4,'D','zhangsanD',4.12)",
+ LOOKUP_TABLE));
}
@Test
@@ -167,9 +165,9 @@ private DorisRowDataJdbcLookupFunction buildRowDataJdbcLookupFunction(
DorisOptions.builder()
.setFenodes(getFenodes())
.setTableIdentifier(LOOKUP_TABLE)
- .setJdbcUrl(getJdbcUrl())
- .setUsername(USERNAME)
- .setPassword(PASSWORD)
+ .setJdbcUrl(getDorisQueryUrl())
+ .setUsername(getDorisUsername())
+ .setPassword(getDorisPassword())
.build();
DorisRowDataJdbcLookupFunction lookupFunction =
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testAutoAddTable.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testAutoAddTable.txt
index e5c03a968..88ec4541a 100644
--- a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testAutoAddTable.txt
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testAutoAddTable.txt
@@ -1,6 +1,5 @@
mysql-sync-database
--including-tables "tbl.*|auto_add"
--table-conf replication_num=1
- --sink-conf sink.check-interval=5000
--single-sink true
--ignore-default-value true
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt
index 922538143..6f69a75b1 100644
--- a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisByDefault.txt
@@ -1,4 +1,3 @@
mysql-sync-database
--including-tables "tbl1|tbl2|tbl3|tbl5"
- --table-conf replication_num=1
- --sink-conf sink.check-interval=5000
\ No newline at end of file
+ --table-conf replication_num=1
\ No newline at end of file
diff --git a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
index 1190d10de..d863ecfae 100644
--- a/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
+++ b/flink-doris-connector/src/test/resources/autoci/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
@@ -1,6 +1,5 @@
mysql-sync-database
--including-tables "tbl.*|add_tbl"
--table-conf replication_num=1
- --sink-conf sink.check-interval=5000
--schema-change-mode sql_parser
--single-sink true
\ No newline at end of file
From ad98652960db79ec1e876875d805cf4001cc7afc Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Mon, 26 Aug 2024 10:56:28 +0800
Subject: [PATCH 13/48] fix
---
.../apache/doris/flink/container/AbstractITCaseService.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
index 1b5b3114b..5ff32af53 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
@@ -29,11 +29,14 @@
import org.apache.flink.util.function.SupplierWithException;
import org.junit.Rule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.TimeoutException;
public abstract class AbstractITCaseService extends AbstractContainerTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractITCaseService.class);
protected static void waitForJobStatus(
JobClient client, List expectedStatus, Deadline deadline) throws Exception {
@@ -102,6 +105,7 @@ protected enum FailoverType {
protected static void triggerFailover(
FailoverType type, JobID jobId, MiniCluster miniCluster, Runnable afterFailAction)
throws Exception {
+ LOG.info("Will job trigger failover. type={}, jobId={}", type, jobId);
switch (type) {
case TM:
restartTaskManager(miniCluster, afterFailAction);
From d7b29d456fe7fc7860c070f3511a9f8b32bf8a1e Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Mon, 26 Aug 2024 11:12:20 +0800
Subject: [PATCH 14/48] fix
---
.../apache/doris/flink/container/AbstractITCaseService.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
index 5ff32af53..956b8be65 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
@@ -122,16 +122,20 @@ protected static void triggerFailover(
protected static void restartTaskManager(MiniCluster miniCluster, Runnable afterFailAction)
throws Exception {
+ LOG.info("flink cluster will terminate task manager.");
miniCluster.terminateTaskManager(0).get();
afterFailAction.run();
+ LOG.info("flink cluster will start task manager.");
miniCluster.startTaskManager();
}
protected static void triggerJobManagerFailover(
JobID jobId, MiniCluster miniCluster, Runnable afterFailAction) throws Exception {
+ LOG.info("flink cluster will revoke job master leadership. jobId={}", jobId);
final HaLeadershipControl haLeadershipControl = miniCluster.getHaLeadershipControl().get();
haLeadershipControl.revokeJobMasterLeadership(jobId).get();
afterFailAction.run();
+ LOG.info("flink cluster will grant job master leadership. jobId={}", jobId);
haLeadershipControl.grantJobMasterLeadership(jobId).get();
}
}
From e51a4361d21870e4438b3c1f370c0df0fa6c6ada Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Mon, 26 Aug 2024 11:28:08 +0800
Subject: [PATCH 15/48] fix
---
.../java/org/apache/doris/flink/sink/DorisSinkITCase.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
index b820ef030..d8ad0abd9 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
@@ -415,11 +415,13 @@ public void testJobManagerFailoverSink() throws Exception {
triggerFailover(
FailoverType.JM, jobID, miniClusterResource.getMiniCluster(), () -> sleepMs(100));
+ LOG.info("Waiting the JobManagerFailoverSink job to be finished. jobId={}", jobID);
waitForJobStatus(
jobClient,
Collections.singletonList(FINISHED),
Deadline.fromNow(Duration.ofSeconds(120)));
+ LOG.info("Will check job manager failover sink result.");
List expected =
Arrays.asList("1,0", "1,1", "2,0", "2,1", "3,0", "3,1", "4,0", "4,1", "5,0", "5,1");
String query =
@@ -470,11 +472,13 @@ public void testTaskManagerFailoverSink() throws Exception {
triggerFailover(
FailoverType.TM, jobID, miniClusterResource.getMiniCluster(), () -> sleepMs(100));
+ LOG.info("Waiting the TaskManagerFailoverSink job to be finished. jobId={}", jobID);
waitForJobStatus(
jobClient,
Collections.singletonList(FINISHED),
Deadline.fromNow(Duration.ofSeconds(120)));
+ LOG.info("Will check task manager failover sink result.");
List expected =
Arrays.asList("1,0", "1,1", "2,0", "2,1", "3,0", "3,1", "4,0", "4,1", "5,0", "5,1");
String query =
From 257431a4973742a33db5bb461d105d11576c1698 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Mon, 26 Aug 2024 11:49:25 +0800
Subject: [PATCH 16/48] fix
---
.../doris/flink/container/ContainerUtils.java | 11 ++++++++++-
.../container/e2e/Mysql2DorisE2ECase.java | 16 ++++++++--------
.../doris/flink/sink/DorisSinkITCase.java | 18 +++++++++---------
3 files changed, 27 insertions(+), 18 deletions(-)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
index 38520e8cc..2cfacddb5 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
@@ -99,7 +99,11 @@ public static String[] parseFileContentSQL(String resourcePath) {
}
public static void checkResult(
- Connection connection, List expected, String query, int columnSize) {
+ Connection connection,
+ Logger logger,
+ List expected,
+ String query,
+ int columnSize) {
List actual = new ArrayList<>();
try (Statement statement = connection.createStatement()) {
ResultSet sinkResultSet = statement.executeQuery(query);
@@ -116,6 +120,11 @@ public static void checkResult(
actual.add(StringUtils.join(row, ","));
}
} catch (SQLException e) {
+ logger.info(
+ "Failed to check query result. expected={}, actual={}",
+ String.join(",", expected),
+ String.join(",", actual),
+ e);
throw new DorisRuntimeException(e);
}
Assert.assertArrayEquals(expected.toArray(), actual.toArray());
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
index 7992f656d..d3cc720f3 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
@@ -116,7 +116,7 @@ private void verifyInitializeResult() {
Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3", "doris_5,5");
String sql1 =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.tbl5) res order by 1";
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, sql1, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, sql1, 2);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@@ -155,7 +155,7 @@ public void testAutoAddTable() throws InterruptedException {
Thread.sleep(20000);
List autoAddResult = Arrays.asList("doris_4_1,4", "doris_4_2,4");
String autoAddSql = "select * from test_e2e_mysql.auto_add order by 1";
- ContainerUtils.checkResult(getDorisQueryConnection(), autoAddResult, autoAddSql, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, autoAddResult, autoAddSql, 2);
// incremental data
LOG.info("starting to increment data.");
@@ -179,7 +179,7 @@ public void testAutoAddTable() throws InterruptedException {
String incrementDataSql =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.auto_add) res order by 1";
ContainerUtils.checkResult(
- getDorisQueryConnection(), incrementDataExpected, incrementDataSql, 2);
+ getDorisQueryConnection(), LOG, incrementDataExpected, incrementDataSql, 2);
// schema change
LOG.info("starting to mock schema change.");
@@ -194,7 +194,7 @@ public void testAutoAddTable() throws InterruptedException {
Arrays.asList("doris_4_1,null", "doris_4_3,null", "doris_4_4,c1_val");
String schemaChangeSql = "select * from test_e2e_mysql.auto_add order by 1";
ContainerUtils.checkResult(
- getDorisQueryConnection(), schemaChangeExpected, schemaChangeSql, 2);
+ getDorisQueryConnection(), LOG, schemaChangeExpected, schemaChangeSql, 2);
cancelCurrentE2EJob(jobName);
}
@@ -226,7 +226,7 @@ public void testMySQL2DorisSQLParse() throws Exception {
List createTableExpected = Arrays.asList("doris_1,1", "doris_2,2", "doris_3,3");
String createTableSql = "select * from test_e2e_mysql.add_tbl order by 1";
ContainerUtils.checkResult(
- getDorisQueryConnection(), createTableExpected, createTableSql, 2);
+ getDorisQueryConnection(), LOG, createTableExpected, createTableSql, 2);
cancelCurrentE2EJob(jobName);
}
@@ -256,7 +256,7 @@ private void verifyTbl1SchemaChange() {
Arrays.asList("doris_1,null", "doris_1_1,null", "doris_1_1_1,c1_val");
String schemaChangeSql = "select * from test_e2e_mysql.tbl1 order by 1";
ContainerUtils.checkResult(
- getDorisQueryConnection(), schemaChangeExpected, schemaChangeSql, 2);
+ getDorisQueryConnection(), LOG, schemaChangeExpected, schemaChangeSql, 2);
}
private void addIncrementalData() {
@@ -283,7 +283,7 @@ private void verifyIncrementalDataResult() {
"doris_1,18", "doris_1_1,10", "doris_2_1,11", "doris_3,3", "doris_3_1,12");
String sql2 =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 ) res order by 1";
- ContainerUtils.checkResult(getDorisQueryConnection(), expected2, sql2, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected2, sql2, 2);
}
@Test
@@ -320,7 +320,7 @@ public void testMySQL2DorisEnableDelete() throws Exception {
"doris_5,5");
String sql =
"select * from ( select * from test_e2e_mysql.tbl1 union all select * from test_e2e_mysql.tbl2 union all select * from test_e2e_mysql.tbl3 union all select * from test_e2e_mysql.tbl5) res order by 1";
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, sql, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, sql, 2);
cancelCurrentE2EJob(jobName);
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
index d8ad0abd9..32256bcee 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
@@ -85,7 +85,7 @@ public void testSinkCsvFormat() throws Exception {
Thread.sleep(10000);
List expected = Arrays.asList("doris,1");
String query = String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_CSV);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
@Test
@@ -123,7 +123,7 @@ public void testSinkJsonFormat() throws Exception {
Thread.sleep(10000);
List expected = Arrays.asList("doris1,1", "doris2,2");
String query = String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_JSON);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
private void submitJob(
@@ -189,7 +189,7 @@ public void testTableSinkJsonFormat() throws Exception {
List expected = Arrays.asList("doris,1", "flink,2");
String query =
String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_JSON_TBL);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
@Test
@@ -236,7 +236,7 @@ public void testTableBatch() throws Exception {
String query =
String.format(
"select name,age from %s.%s order by 1", DATABASE, TABLE_CSV_BATCH_TBL);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
@Test
@@ -276,7 +276,7 @@ public void testDataStreamBatch() throws Exception {
String query =
String.format(
"select name,age from %s.%s order by 1", DATABASE, TABLE_CSV_BATCH_DS);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
@Test
@@ -325,7 +325,7 @@ public void testTableGroupCommit() throws Exception {
String query =
String.format(
"select name,age from %s.%s order by 1", DATABASE, TABLE_GROUP_COMMIT);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
@Test
@@ -366,7 +366,7 @@ public void testTableGzFormat() throws Exception {
List expected = Arrays.asList("doris,1", "flink,2");
String query =
String.format("select name,age from %s.%s order by 1", DATABASE, TABLE_GZ_FORMAT);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
@Test
@@ -426,7 +426,7 @@ public void testJobManagerFailoverSink() throws Exception {
Arrays.asList("1,0", "1,1", "2,0", "2,1", "3,0", "3,1", "4,0", "4,1", "5,0", "5,1");
String query =
String.format("select id,task_id from %s.%s order by 1,2", DATABASE, TABLE_CSV_JM);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
@Test
@@ -483,7 +483,7 @@ public void testTaskManagerFailoverSink() throws Exception {
Arrays.asList("1,0", "1,1", "2,0", "2,1", "3,0", "3,1", "4,0", "4,1", "5,0", "5,1");
String query =
String.format("select id,task_id from %s.%s order by 1,2", DATABASE, TABLE_CSV_TM);
- ContainerUtils.checkResult(getDorisQueryConnection(), expected, query, 2);
+ ContainerUtils.checkResult(getDorisQueryConnection(), LOG, expected, query, 2);
}
private void sleepMs(long millis) {
From 3a03df29084cfcd3f939d9ab74569358640bacb1 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Mon, 26 Aug 2024 12:04:59 +0800
Subject: [PATCH 17/48] fix
---
.../doris/flink/sink/DorisSinkITCase.java | 21 +++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
index 32256bcee..3d2a83282 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
@@ -372,7 +372,7 @@ public void testTableGzFormat() throws Exception {
@Test
public void testJobManagerFailoverSink() throws Exception {
LOG.info("start to test JobManagerFailoverSink.");
- initializeTable(TABLE_CSV_JM);
+ initializeFailoverTable(TABLE_CSV_JM);
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(2);
env.enableCheckpointing(10000);
@@ -432,7 +432,7 @@ public void testJobManagerFailoverSink() throws Exception {
@Test
public void testTaskManagerFailoverSink() throws Exception {
LOG.info("start to test TaskManagerFailoverSink.");
- initializeTable(TABLE_CSV_TM);
+ initializeFailoverTable(TABLE_CSV_TM);
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(2);
env.enableCheckpointing(10000);
@@ -509,4 +509,21 @@ private void initializeTable(String table) {
+ ")\n",
DATABASE, table));
}
+
+ private void initializeFailoverTable(String table) {
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE),
+ String.format("DROP TABLE IF EXISTS %s.%s", DATABASE, table),
+ String.format(
+ "CREATE TABLE %s.%s ( \n"
+ + "`id` int,\n"
+ + "`task_id` int\n"
+ + ") DISTRIBUTED BY HASH(`id`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ DATABASE, table));
+ }
}
From edbda2a65a4811b451b450c175c96d45335d51b3 Mon Sep 17 00:00:00 2001
From: DongLiang-0 <1747644936@qq.com>
Date: Mon, 26 Aug 2024 14:46:31 +0800
Subject: [PATCH 18/48] fix
---
.../container/AbstractContainerTestBase.java | 4 +-
.../flink/container/AbstractE2EService.java | 4 +-
.../doris/flink/container/DorisTestBase.java | 326 ------------------
.../container/instance/ContainerService.java | 2 +
.../container/instance/DorisContainer.java | 5 +
.../container/instance/MySQLContainer.java | 5 +
6 files changed, 16 insertions(+), 330 deletions(-)
delete mode 100644 flink-doris-connector/src/test/java/org/apache/doris/flink/container/DorisTestBase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
index 4d2cfac30..e7c7852bd 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
@@ -37,8 +37,8 @@ public static void initContainers() {
}
private static void initDorisContainer() {
- if (Objects.nonNull(dorisContainerService)) {
- LOG.info("The doris container has been started and will be used directly.");
+ if (Objects.nonNull(dorisContainerService) && dorisContainerService.isAlive()) {
+ LOG.info("The doris container has been started and is alive status.");
return;
}
dorisContainerService = new DorisContainer();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
index 532d153cd..627b4fc11 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
+++ b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
@@ -60,8 +60,8 @@ public static void initE2EContainers() {
}
private static void initMySQLContainer() {
- if (Objects.nonNull(mysqlContainerService)) {
- LOG.info("The MySQL container has been started and will be used directly.");
+ if (Objects.nonNull(mysqlContainerService) && mysqlContainerService.isAlive()) {
+ LOG.info("The MySQL container has been started and is alive status.");
return;
}
mysqlContainerService = new MySQLContainer();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/DorisTestBase.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/container/DorisTestBase.java
deleted file mode 100644
index fc5daa2d5..000000000
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/DorisTestBase.java
+++ /dev/null
@@ -1,326 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.flink.container;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.api.common.time.Deadline;
-import org.apache.flink.core.execution.JobClient;
-import org.apache.flink.runtime.highavailability.nonha.embedded.HaLeadershipControl;
-import org.apache.flink.runtime.minicluster.MiniCluster;
-import org.apache.flink.runtime.minicluster.RpcServiceSharing;
-import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
-import org.apache.flink.test.util.MiniClusterWithClientResource;
-import org.apache.flink.util.function.SupplierWithException;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.Network;
-import org.testcontainers.containers.output.Slf4jLogConsumer;
-import org.testcontainers.utility.DockerLoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.InetAddress;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.locks.LockSupport;
-
-public abstract class DorisTestBase {
- protected static final Logger LOG = LoggerFactory.getLogger(DorisTestBase.class);
- private static final String DEFAULT_DOCKER_IMAGE = "apache/doris:doris-all-in-one-2.1.0";
- protected static final String DORIS_DOCKER_IMAGE =
- System.getProperty("image") == null
- ? DEFAULT_DOCKER_IMAGE
- : System.getProperty("image");
- private static final String DRIVER_JAR =
- "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar";
- protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";
- protected static final String URL = "jdbc:mysql://%s:9030";
- protected static final String USERNAME = "root";
- protected static final String PASSWORD = "";
- protected static final GenericContainer DORIS_CONTAINER = createDorisContainer();
-
- protected static String getFenodes() {
- return DORIS_CONTAINER.getHost() + ":8030";
- }
-
- protected static String getBenodes() {
- return DORIS_CONTAINER.getHost() + ":8040";
- }
-
- protected static String getJdbcUrl() {
- return String.format(URL, DORIS_CONTAINER.getHost());
- }
-
- protected static String getHost() {
- return DORIS_CONTAINER.getHost();
- }
-
- static {
- startContainers();
- }
-
- public static void startContainers() {
- try {
- LOG.info("Starting doris containers...");
- // singleton doris container
- DORIS_CONTAINER.start();
- initializeJdbcConnection();
- } catch (Exception ex) {
- LOG.error("Failed to start containers doris, ", ex);
- }
- LOG.info("Containers doris are started.");
- }
-
- public static GenericContainer createDorisContainer() {
- LOG.info("Create doris containers...");
- GenericContainer container =
- new GenericContainer<>(DORIS_DOCKER_IMAGE)
- .withNetwork(Network.newNetwork())
- .withNetworkAliases("DorisContainer")
- .withPrivilegedMode(true)
- .withLogConsumer(
- new Slf4jLogConsumer(
- DockerLoggerFactory.getLogger(DORIS_DOCKER_IMAGE)))
- .withExposedPorts(8030, 9030, 8040, 9060);
-
- container.setPortBindings(
- Lists.newArrayList(
- String.format("%s:%s", "8030", "8030"),
- String.format("%s:%s", "9030", "9030"),
- String.format("%s:%s", "9060", "9060"),
- String.format("%s:%s", "8040", "8040")));
-
- return container;
- }
-
- protected static void initializeJdbcConnection() throws Exception {
- URLClassLoader urlClassLoader =
- new URLClassLoader(
- new URL[] {new URL(DRIVER_JAR)}, DorisTestBase.class.getClassLoader());
- LOG.info("Try to connect to Doris...");
- Thread.currentThread().setContextClassLoader(urlClassLoader);
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- ResultSet resultSet;
- do {
- LOG.info("Wait for the Backend to start successfully...");
- resultSet = statement.executeQuery("show backends");
- } while (!isBeReady(resultSet, Duration.ofSeconds(1L)));
- }
- LOG.info("Connected to Doris successfully...");
- printClusterStatus();
- }
-
- private static boolean isBeReady(ResultSet rs, Duration duration) throws SQLException {
- LockSupport.parkNanos(duration.toNanos());
- if (rs.next()) {
- String isAlive = rs.getString("Alive").trim();
- String totalCap = rs.getString("TotalCapacity").trim();
- return "true".equalsIgnoreCase(isAlive) && !"0.000".equalsIgnoreCase(totalCap);
- }
- return false;
- }
-
- protected static void printClusterStatus() throws Exception {
- LOG.info("Current machine IP: {}", InetAddress.getLocalHost());
- echo("sh", "-c", "cat /proc/cpuinfo | grep 'cpu cores' | uniq");
- echo("sh", "-c", "free -h");
- try (Connection connection =
- DriverManager.getConnection(
- String.format(URL, DORIS_CONTAINER.getHost()), USERNAME, PASSWORD);
- Statement statement = connection.createStatement()) {
- ResultSet showFrontends = statement.executeQuery("show frontends");
- LOG.info("Frontends status: {}", convertList(showFrontends));
- ResultSet showBackends = statement.executeQuery("show backends");
- LOG.info("Backends status: {}", convertList(showBackends));
- }
- }
-
- static void echo(String... cmd) {
- try {
- Process p = Runtime.getRuntime().exec(cmd);
- InputStream is = p.getInputStream();
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
- String line;
- while ((line = reader.readLine()) != null) {
- System.out.println(line);
- }
- p.waitFor();
- is.close();
- reader.close();
- p.destroy();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- private static List