From 061b4df18a3944195996aebbada99a439a28e8b0 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 15 Mar 2023 06:28:14 -0500 Subject: [PATCH 01/45] Added Debezium DB2 connector for Pulsar --- pulsar-io/debezium/db2/pom.xml | 65 +++++++++++++++++++ .../io/debezium/db2/DebeziumDB2Source.java | 36 ++++++++++ .../pulsar/io/debezium/db2/package-info.java | 19 ++++++ .../META-INF/services/pulsar-io.yaml | 22 +++++++ .../resources/debezium-db2-source-config.yaml | 37 +++++++++++ pulsar-io/debezium/pom.xml | 1 + 6 files changed, 180 insertions(+) create mode 100644 pulsar-io/debezium/db2/pom.xml create mode 100644 pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/DebeziumDB2Source.java create mode 100644 pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java create mode 100644 pulsar-io/debezium/db2/src/main/resources/META-INF/services/pulsar-io.yaml create mode 100644 pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml diff --git a/pulsar-io/debezium/db2/pom.xml b/pulsar-io/debezium/db2/pom.xml new file mode 100644 index 0000000000000..081e83fe31ec7 --- /dev/null +++ b/pulsar-io/debezium/db2/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + org.apache.pulsar + pulsar-io-debezium + 3.0.0-SNAPSHOT + + + pulsar-io-debezium-db2 + Pulsar IO :: Debezium :: DB2 + + + + + ${project.groupId} + pulsar-io-core + ${project.version} + provided + + + + ${project.groupId} + pulsar-io-debezium-core + ${project.version} + + + + io.debezium + debezium-connector-db2 + ${debezium.version} + + + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + + diff --git a/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/DebeziumDB2Source.java b/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/DebeziumDB2Source.java new file mode 100644 index 0000000000000..d59d1d456a0d5 --- /dev/null +++ b/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/DebeziumDB2Source.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.debezium.db2; + +import java.util.Map; +import org.apache.kafka.connect.runtime.TaskConfig; +import org.apache.pulsar.io.debezium.DebeziumSource; + + +/** + * A pulsar source that runs debezium oracle source. + */ +public class DebeziumDB2Source extends DebeziumSource { + private static final String DEFAULT_TASK = "io.debezium.connector.db2.Db2ConnectorTask"; + + @Override + public void setDbConnectorTask(Map config) throws Exception { + throwExceptionIfConfigNotMatch(config, TaskConfig.TASK_CLASS_CONFIG, DEFAULT_TASK); + } +} diff --git a/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java b/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java new file mode 100644 index 0000000000000..7efd1fea05d77 --- /dev/null +++ b/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.debezium.oracle; \ No newline at end of file diff --git a/pulsar-io/debezium/db2/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/debezium/db2/src/main/resources/META-INF/services/pulsar-io.yaml new file mode 100644 index 0000000000000..9aeba132fea99 --- /dev/null +++ b/pulsar-io/debezium/db2/src/main/resources/META-INF/services/pulsar-io.yaml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: debezium-db2 +description: Debezium DB2 Source +sourceClass: org.apache.pulsar.io.debezium.db2.DebeziumDB2Source diff --git a/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml new file mode 100644 index 0000000000000..fa14e9a55b403 --- /dev/null +++ b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +tenant: "public" +namespace: "default" +name: "debezium-db2-source" +topicName: "debezium-db2-topic" +archive: "connectors/pulsar-io-debezium-db2-2.9.0-SNAPSHOT.nar" + +parallelism: 1 + +configs: + ## config for db2 XE, docker image: https://github.com/MaksymBilenko/docker-db2-12c + ## docker run -itd --name mydb2 --privileged=true -p 50000:50000 -e LICENSE=accept -e DB2INST1_PASSWORD=admin -e DBNAME=testdb -v :/database ibmcom/db2 + database.hostname: "localhost" + database.port: "50000" + database.user: "db2inst1" + database.password: "admin" + database.dbname: "mydb2" + + database.history.pulsar.service.url: "pulsar://127.0.0.1:6650" diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 050da1af45330..d70d325497c81 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -78,6 +78,7 @@ mongodb oracle mssql + db2 From 9972036765f572ba509f33c57a65169ac950140f Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 17 Mar 2023 15:27:09 -0500 Subject: [PATCH 02/45] Initial commit for DB2 Debezium connector testing --- .../ci-integration-pulsar-io-db2.yaml | 116 ++++++++++++ build/run_integration_group.sh | 4 + distribution/io/src/assemble/io.xml | 1 + .../latest-version-image/Dockerfile | 13 +- .../containers/DebeziumDB2DbContainer.java | 72 ++++++++ .../debezium/DebeziumDB2DbSourceTester.java | 169 ++++++++++++++++++ 6 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-integration-pulsar-io-db2.yaml create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java diff --git a/.github/workflows/ci-integration-pulsar-io-db2.yaml b/.github/workflows/ci-integration-pulsar-io-db2.yaml new file mode 100644 index 0000000000000..b7dfd98e8da52 --- /dev/null +++ b/.github/workflows/ci-integration-pulsar-io-db2.yaml @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Integration - Pulsar-IO DB2 Source +on: + pull_request: + branches: + - master + push: + branches: + - branch-* + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + +jobs: + + pulsar-io: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + + - name: Check changed files + id: check_changes + run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" + + - name: Cache local Maven repository + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + uses: actions/cache@v2 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + - name: Set up JDK 11 + uses: actions/setup-java@v2 + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + with: + distribution: 'adopt' + java-version: 11 + + - name: clean disk + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: | + sudo swapoff -a + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + - name: run install by skip tests + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: mvn -q -B -ntp clean install -DskipTests + + - name: build pulsar image + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true + + - name: build pulsar-all image + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true + + - name: build artifacts and docker image + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests + + - name: run integration tests + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + run: ./build/run_integration_group.sh PULSAR_IO_DB2 + + - name: Upload container logs + uses: actions/upload-artifact@v2 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: container-logs + path: tests/integration/target/container-logs + + - name: Upload surefire-reports + uses: actions/upload-artifact@v2 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: surefire-reports + path: tests/integration/target/surefire-reports \ No newline at end of file diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh index bc1255d8d68aa..87d0bbeafb2e5 100755 --- a/build/run_integration_group.sh +++ b/build/run_integration_group.sh @@ -215,6 +215,10 @@ test_group_pulsar_io_ora() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-io-ora-source.xml -DintegrationTests -Dgroups=source -DtestRetryCount=0 } +test_group_pulsar_io_db2() { + mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-io-db2-source.xml -DintegrationTests -Dgroups=source -DtestRetryCount=0 +} + list_test_groups() { declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]' } diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 5b652170fdbb5..c215e3b323684 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -73,6 +73,7 @@ ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar + ${basedir}/../../pulsar-io/debezium/db2/target/pulsar-io-debezium-db2-${project.version}.nar ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index f0093fa1eafc5..42927ec28801d 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -129,5 +129,16 @@ RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remo RUN jar uf connectors/pulsar-io-debezium-oracle-*.nar META-INF/bundled-dependencies/ojdbc8-19.3.0.0.jar META-INF/bundled-dependencies/ucp-19.3.0.0.jar META-INF/bundled-dependencies/oraclepki-19.3.0.0.jar META-INF/bundled-dependencies/osdt_cert-19.3.0.0.jar META-INF/bundled-dependencies/osdt_core-19.3.0.0.jar META-INF/bundled-dependencies/simplefan-19.3.0.0.jar META-INF/bundled-dependencies/orai18n-19.3.0.0.jar META-INF/bundled-dependencies/xdb-19.3.0.0.jar META-INF/bundled-dependencies/xmlparserv2-19.3.0.0.jar - +# Download IBM DB2 JDBC driver for DB2 Debezium Connector tests: +RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/ibm/db2/jcc/11.5.8.0/jcc-11.5.8.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/ibm/db2/jcc/db2jcc/db2jcc4/db2jcc-db2jcc4.jar +RUN mkdir -p /asncdctools/src +RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc.c +RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/dbsetup.sh +RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc_UDF.sql +RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdcaddremove.sql +RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdctables.sql +RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/cdcsetup.sh + +RUN jar uf connectors/pulsar-io-debezium-db2-*.nar META-INF/bundled-dependencies/jcc-11.5.8.0.jar META-INF/bundled-dependencies/db2jcc-db2jcc4.jar CMD bash diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java new file mode 100644 index 0000000000000..45991e9bfe07b --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.containers; + + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; + +public class DebeziumDB2DbContainer extends ChaosContainer { + + public static final String NAME = "debezium-db2"; + static final Integer[] PORTS = { 50000 }; + + // + private static final String IMAGE_NAME = "devingbost/debezium-db2:0.0.1"; + + public DebeziumDB2DbContainer(String clusterName) { + super(clusterName, IMAGE_NAME); + } + + @Override + public String getContainerName() { + return clusterName; + } + + @Override + protected void configure() { + super.configure(); + this.withNetworkAliases(NAME) + .withExposedPorts(PORTS) + .withEnv("LICENSE", "accept") + .withEnv("DB2INSTANCE", "db2inst1") + .withEnv("DB2INST1_PASSWORD", "admin") + .withEnv("DBNAME", "mydb2") + .withEnv("BLU", "false") + .withEnv("ENABLE_ORACLE_COMPATIBILITY", "false") + .withEnv("UPDATEAVAIL", "NO") + .withEnv("TO_CREATE_SAMPLEDB", "false") + .withEnv("REPODB", "false") + .withEnv("IS_OSXFS", "false") + .withEnv("PERSISTENT_HOME", "true") + .withEnv("HADR_ENABLED", "false") + // .withEnv("ETCD_ENDPOINT", "") + // .withEnv("ETCD_USERNAME", "") + // .withEnv("ETCD_PASSWORD", "") + .withPrivilegedMode(true) + .withStartupTimeout(Duration.of(300, ChronoUnit.SECONDS)) + .withCreateContainerCmdModifier(createContainerCmd -> { + createContainerCmd.withHostName(NAME); + createContainerCmd.withName(getContainerName()); + }) + .waitingFor(new HostPortWaitStrategy()); + } + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java new file mode 100644 index 0000000000000..917c1a160b7b8 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io.sources.debezium; + +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.tests.integration.containers.DebeziumDB2DbContainer; +import org.apache.pulsar.tests.integration.containers.DebeziumOracleDbContainer; +import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.testng.util.Strings; + +/** + * A tester for testing Debezium OracleDb source. + */ +@Slf4j +public class DebeziumDB2DbSourceTester extends SourceTester { + + private static final String NAME = "debezium-db2"; + private static final long SLEEP_AFTER_COMMAND_MS = 30_000; + + private final String pulsarServiceUrl; + + @Getter + private DebeziumDB2DbContainer debeziumDB2DbContainer; + + private final PulsarCluster pulsarCluster; + + public DebeziumDB2DbSourceTester(PulsarCluster cluster) { + super(NAME); + this.pulsarCluster = cluster; + this.numEntriesToInsert = 1; + this.numEntriesExpectAfterStart = 0; + + pulsarServiceUrl = "pulsar://pulsar-proxy:" + PulsarContainer.BROKER_PORT; + + sourceConfig.put("database.hostname", DebeziumDB2DbContainer.NAME); + sourceConfig.put("database.port", "50000"); + sourceConfig.put("database.user", "db2inst1"); + sourceConfig.put("database.password", "admin"); + //sourceConfig.put("database.server.name", "XE"); + sourceConfig.put("database.dbname", "mydb2"); + sourceConfig.put("snapshot.mode", "schema_only"); //?? + + sourceConfig.put("schema.include.list", "inv"); //?? + sourceConfig.put("database.history.pulsar.service.url", pulsarServiceUrl); + sourceConfig.put("topic.namespace", "debezium/db2"); + } + + @Override + public void setServiceContainer(DebeziumDB2DbContainer container) { + log.info("start debezium db2 server container."); + Preconditions.checkState(debeziumDB2DbContainer == null); + debeziumDB2DbContainer = container; + pulsarCluster.startService(DebeziumOracleDbContainer.NAME, debeziumDB2DbContainer); + } + + @SneakyThrows + @Override + public void prepareSource() { + + waitForDB2SetupStatus("KAFKA"); + Thread.sleep(SLEEP_AFTER_COMMAND_MS); + + runSqlCmd("CREATE TABLE DB2INST1.STORES(store_id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL,store_name VARCHAR(150) NOT NULL,state_id INT NOT NULL,zip_code VARCHAR(6),PRIMARY KEY (store_id));"); + runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111');"); + runSqlCmd("call ASNCDC.ADDTABLE('DB2INST1','STORES')"); + }; + + private void waitForDB2SetupStatus(String status) throws Exception { + for (int i = 0; i < 1000; i++) { + ContainerExecResult response = runSqlCmd("SELECT target_server from ASNCDC.IBMSNAP_PRUNCNTL;"); + if ((response.getStderr() != null && response.getStderr().contains(status)) + || (response.getStdout() != null && response.getStdout().contains(status))) { + return; + } + Thread.sleep(1000); + } + throw new IllegalStateException("DB2 did not initialize properly"); + } + + private ContainerExecResult runSqlCmd(String cmd) throws Exception { + log.info("Executing \"{}\"", cmd); + ContainerExecResult response = this.debeziumDB2DbContainer + .execCmdAsUser("root", + "/bin/bash", "-c", + "/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin'; " + + "/opt/ibm/db2/V11.5/bin/db2 \"" + cmd + "\"" + ); + if (Strings.isNullOrEmpty(response.getStderr())) { + log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); + } else { + log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); + } + return response; + } + + @Override + public void prepareInsertEvent() throws Exception { + runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore2', 2, '22222');"); + runSqlCmd("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2';"); + } + + @Override + public void prepareDeleteEvent() throws Exception { + runSqlCmd("DELETE FROM DB2INST1.STORES WHERE store_name='mystore2';"); + runSqlCmd("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2';"); + } + + @Override + public void prepareUpdateEvent() throws Exception { + runSqlCmd("UPDATE DB2INST1.STORES SET zip_code='33333' WHERE store_name='mystore2';"); + runSqlCmd("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2';"); + } + + @Override + public Map produceSourceMessages(int numMessages) { + log.info("debezium db2 server already contains preconfigured data."); + return null; + } + + @Override + public int initialDelayForMsgReceive() { + // LogMiner takes a lot of time to get messages out + return 30; + } + + @Override + public String keyContains() { + return "mydb2.DB2INST1.STORES.Key"; + } + + @Override + public String valueContains() { + return "mydb2.DB2INST1.STORES.Value"; + } + + @Override + public void close() { + if (pulsarCluster != null) { + if (debeziumDB2DbContainer != null) { + PulsarCluster.stopService(DebeziumOracleDbContainer.NAME, debeziumDB2DbContainer); + debeziumDB2DbContainer = null; + } + } + } + +} From 4ea52168bc0e8c6531ae8e276d6373483f07c81e Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 17 Mar 2023 15:32:13 -0500 Subject: [PATCH 03/45] Updated db2 source config --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 917c1a160b7b8..f078bec43e23c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -54,16 +54,18 @@ public DebeziumDB2DbSourceTester(PulsarCluster cluster) { this.numEntriesExpectAfterStart = 0; pulsarServiceUrl = "pulsar://pulsar-proxy:" + PulsarContainer.BROKER_PORT; - + sourceConfig.put("connector.class", "io.debezium.connector.db2.Db2Connector"); sourceConfig.put("database.hostname", DebeziumDB2DbContainer.NAME); sourceConfig.put("database.port", "50000"); sourceConfig.put("database.user", "db2inst1"); sourceConfig.put("database.password", "admin"); - //sourceConfig.put("database.server.name", "XE"); sourceConfig.put("database.dbname", "mydb2"); - sourceConfig.put("snapshot.mode", "schema_only"); //?? + sourceConfig.put("topic.prefix", "stores"); + sourceConfig.put("table.include.list", "DB2INST1.STORES"); + + //sourceConfig.put("snapshot.mode", "schema_only"); //?? - sourceConfig.put("schema.include.list", "inv"); //?? + //sourceConfig.put("schema.include.list", "inv"); //?? sourceConfig.put("database.history.pulsar.service.url", pulsarServiceUrl); sourceConfig.put("topic.namespace", "debezium/db2"); } From f38a5c9c5debae752442f4dbc3441cad7da51db3 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 17 Mar 2023 16:05:49 -0500 Subject: [PATCH 04/45] Removed CI file --- .../ci-integration-pulsar-io-db2.yaml | 116 ------------------ 1 file changed, 116 deletions(-) delete mode 100644 .github/workflows/ci-integration-pulsar-io-db2.yaml diff --git a/.github/workflows/ci-integration-pulsar-io-db2.yaml b/.github/workflows/ci-integration-pulsar-io-db2.yaml deleted file mode 100644 index b7dfd98e8da52..0000000000000 --- a/.github/workflows/ci-integration-pulsar-io-db2.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Integration - Pulsar-IO DB2 Source -on: - pull_request: - branches: - - master - push: - branches: - - branch-* - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - pulsar-io: - name: - runs-on: ubuntu-latest - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'adopt' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - name: run install by skip tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -q -B -ntp clean install -DskipTests - - - name: build pulsar image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true - - - name: build pulsar-all image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true - - - name: build artifacts and docker image - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests - - - name: run integration tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: ./build/run_integration_group.sh PULSAR_IO_DB2 - - - name: Upload container logs - uses: actions/upload-artifact@v2 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: container-logs - path: tests/integration/target/container-logs - - - name: Upload surefire-reports - uses: actions/upload-artifact@v2 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: surefire-reports - path: tests/integration/target/surefire-reports \ No newline at end of file From 5f281bb939ac920dd3b2bf5a1686e016abdb354c Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 22 Mar 2023 16:22:47 -0500 Subject: [PATCH 05/45] Updating docs pom --- pulsar-io/docs/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 305c7f1473077..2693104c46934 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -97,6 +97,11 @@ pulsar-io-debezium-mssql ${project.version} + + ${project.groupId} + pulsar-io-debezium-db2 + ${project.version} + ${project.groupId} pulsar-io-dynamodb From d3c5bbd45dca80b8902de8b1ce36de0d6e6af433 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 24 Mar 2023 06:05:43 -0500 Subject: [PATCH 06/45] Added missing hooks to execute DB2 tests and updated some test code. --- .../resources/debezium-db2-source-config.yaml | 5 ++- .../containers/DebeziumDB2DbContainer.java | 3 -- .../debezium/DebeziumDB2DbSourceTester.java | 8 +--- .../debezium/PulsarDebeziumSourcesTest.java | 41 +++++++++++++++++++ 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml index fa14e9a55b403..1f82abb86f912 100644 --- a/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml +++ b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml @@ -26,7 +26,10 @@ archive: "connectors/pulsar-io-debezium-db2-2.9.0-SNAPSHOT.nar" parallelism: 1 configs: - ## config for db2 XE, docker image: https://github.com/MaksymBilenko/docker-db2-12c + ## config for db2, docker image: ibmcom/db2, which was moved to icr.io/db2_community/db2 + ## As per note at https://hub.docker.com/r/ibmcom/db2 + ## docs are being moved to: https://www.ibm.com/docs/en/db2/11.5?topic=deployments-db2-community-edition-docker + ## For testing, you can use this image that was configured with the required libraries: devingbost/debezium-db2:0.0.1 ## docker run -itd --name mydb2 --privileged=true -p 50000:50000 -e LICENSE=accept -e DB2INST1_PASSWORD=admin -e DBNAME=testdb -v :/database ibmcom/db2 database.hostname: "localhost" database.port: "50000" diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 45991e9bfe07b..123f924fdb5db 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -57,9 +57,6 @@ protected void configure() { .withEnv("IS_OSXFS", "false") .withEnv("PERSISTENT_HOME", "true") .withEnv("HADR_ENABLED", "false") - // .withEnv("ETCD_ENDPOINT", "") - // .withEnv("ETCD_USERNAME", "") - // .withEnv("ETCD_PASSWORD", "") .withPrivilegedMode(true) .withStartupTimeout(Duration.of(300, ChronoUnit.SECONDS)) .withCreateContainerCmdModifier(createContainerCmd -> { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index f078bec43e23c..57411cde14ece 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -63,9 +63,6 @@ public DebeziumDB2DbSourceTester(PulsarCluster cluster) { sourceConfig.put("topic.prefix", "stores"); sourceConfig.put("table.include.list", "DB2INST1.STORES"); - //sourceConfig.put("snapshot.mode", "schema_only"); //?? - - //sourceConfig.put("schema.include.list", "inv"); //?? sourceConfig.put("database.history.pulsar.service.url", pulsarServiceUrl); sourceConfig.put("topic.namespace", "debezium/db2"); } @@ -75,7 +72,7 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { log.info("start debezium db2 server container."); Preconditions.checkState(debeziumDB2DbContainer == null); debeziumDB2DbContainer = container; - pulsarCluster.startService(DebeziumOracleDbContainer.NAME, debeziumDB2DbContainer); + pulsarCluster.startService(DebeziumDB2DbContainer.NAME, debeziumDB2DbContainer); } @SneakyThrows @@ -144,7 +141,6 @@ public Map produceSourceMessages(int numMessages) { @Override public int initialDelayForMsgReceive() { - // LogMiner takes a lot of time to get messages out return 30; } @@ -162,7 +158,7 @@ public String valueContains() { public void close() { if (pulsarCluster != null) { if (debeziumDB2DbContainer != null) { - PulsarCluster.stopService(DebeziumOracleDbContainer.NAME, debeziumDB2DbContainer); + PulsarCluster.stopService(DebeziumDB2DbContainer.NAME, debeziumDB2DbContainer); debeziumDB2DbContainer = null; } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java index 5c57c904fc77f..b31cff5503add 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java @@ -28,6 +28,7 @@ import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.tests.integration.containers.DebeziumDB2DbContainer; import org.apache.pulsar.tests.integration.containers.DebeziumMongoDbContainer; import org.apache.pulsar.tests.integration.containers.DebeziumMsSqlContainer; import org.apache.pulsar.tests.integration.containers.DebeziumMySQLContainer; @@ -78,6 +79,10 @@ public void testDebeziumMongoDbSource() throws Exception{ public void testDebeziumMsSqlSource() throws Exception{ testDebeziumMsSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); } + @Test(groups = "source") + public void testDebeziumDb2Source() throws Exception{ + testDebeziumDb2Connect("org.apache.kafka.connect.json.JsonConverter", true); + } private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope, boolean testWithClientBuilder) throws Exception { @@ -242,6 +247,41 @@ private void testDebeziumMsSqlConnect(String converterClassName, boolean jsonWit runner.testSource(sourceTester); } + private void testDebeziumDb2Connect(String converterClassName, boolean jsonWithEnvelope) throws Exception { + + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); + final String consumeTopicName = "debezium/db2/mydb2.DB2INST1.STORES"; + final String sourceName = "test-source-debezium-db2-" + functionRuntimeType + "-" + randomName(8); + + final int numMessages = 1; + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); + + admin.topics().createNonPartitionedTopic(consumeTopicName); + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + DebeziumDB2DbSourceTester sourceTester = new DebeziumDB2DbSourceTester(pulsarCluster); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); + + DebeziumDB2DbContainer db2Container = new DebeziumDB2DbContainer(pulsarCluster.getClusterName()); + sourceTester.setServiceContainer(db2Container); + + PulsarIODebeziumSourceRunner runner = new PulsarIODebeziumSourceRunner(pulsarCluster, functionRuntimeType.toString(), + converterClassName, tenant, namespace, sourceName, outputTopicName, numMessages, jsonWithEnvelope, + consumeTopicName, client); + + runner.testSource(sourceTester); + } protected void initNamespace(PulsarAdmin admin) { log.info("[initNamespace] start."); try { @@ -253,6 +293,7 @@ protected void initNamespace(PulsarAdmin admin) { "debezium/mongodb", "debezium/postgresql", "debezium/mssql", + "debezium/db2" }; Policies policies = new Policies(); policies.retention_policies = new RetentionPolicies(-1, 50); From 0fa7509a515a4e70e307d0ffd99ae94bcb7d76d0 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 28 Mar 2023 15:14:23 -0500 Subject: [PATCH 07/45] Updated source config for doc purposes --- .../resources/debezium-db2-source-config.yaml | 37 ++++++++++++++----- .../debezium/DebeziumDB2DbSourceTester.java | 3 +- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml index 1f82abb86f912..85923f2c988ff 100644 --- a/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml +++ b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml @@ -17,24 +17,43 @@ # under the License. # +## config for db2, docker image: ibmcom/db2, which was moved to icr.io/db2_community/db2 +## As per note at https://hub.docker.com/r/ibmcom/db2 +## docs are being moved to: https://www.ibm.com/docs/en/db2/11.5?topic=deployments-db2-community-edition-docker +## For testing, you can use this image that was configured with the required libraries: devingbost/debezium-db2:0.0.1 +## docker run -itd --name mydb2 --privileged=true -p 50000:50000 -e LICENSE=accept -e DB2INST1_PASSWORD=admin -e DBNAME=testdb -v :/database ibmcom/db2 + tenant: "public" namespace: "default" name: "debezium-db2-source" -topicName: "debezium-db2-topic" -archive: "connectors/pulsar-io-debezium-db2-2.9.0-SNAPSHOT.nar" - +inputs: [ "db2-connect-topic" ] +topicName: "db2-connect-topic" +archive: "connectors/pulsar-io-debezium-db2-2.11.0-SNAPSHOT.nar" parallelism: 1 - configs: - ## config for db2, docker image: ibmcom/db2, which was moved to icr.io/db2_community/db2 - ## As per note at https://hub.docker.com/r/ibmcom/db2 - ## docs are being moved to: https://www.ibm.com/docs/en/db2/11.5?topic=deployments-db2-community-edition-docker - ## For testing, you can use this image that was configured with the required libraries: devingbost/debezium-db2:0.0.1 - ## docker run -itd --name mydb2 --privileged=true -p 50000:50000 -e LICENSE=accept -e DB2INST1_PASSWORD=admin -e DBNAME=testdb -v :/database ibmcom/db2 database.hostname: "localhost" database.port: "50000" database.user: "db2inst1" database.password: "admin" database.dbname: "mydb2" + database.server.name: "db2inst1" + topic.prefix: "stores" # Example for hypothetical stores topics + table.exclude.list: "DB2INST1.EXCLUDEDSTORES" # Example for hypothetical stores table + database.history: "org.apache.pulsar.io.debezium.PulsarDatabaseHistory" + database.history.pulsar.topic: "debezium-db2-source-history-topic" database.history.pulsar.service.url: "pulsar://127.0.0.1:6650" + ## PULSAR_SERVICE_URL_CONFIG + pulsar.service.url: "pulsar://127.0.0.1:6650" + topic.namespace: "public/default" + # CONVERTERS + key.converter: "org.apache.kafka.connect.json.JsonConverter" + value.converter: "org.apache.kafka.connect.json.JsonConverter" + typeClassName: "org.apache.pulsar.common.schema.KeyValue" + + task.class: "io.debezium.connector.db2.Db2ConnectorTask" + + offset.storage.topic: "offset-topic" + snapshot.mode: "initial" + database.tcpKeepAlive: "true" + decimal.handling.mode: "double" diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 57411cde14ece..ef721675fb3e9 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -24,7 +24,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumDB2DbContainer; -import org.apache.pulsar.tests.integration.containers.DebeziumOracleDbContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.io.sources.SourceTester; @@ -32,7 +31,7 @@ import org.testng.util.Strings; /** - * A tester for testing Debezium OracleDb source. + * A tester for testing Debezium DB2 source. */ @Slf4j public class DebeziumDB2DbSourceTester extends SourceTester { From 92dc034cbd4af193924d026bb2d65468b3524966 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 29 Mar 2023 10:11:07 -0500 Subject: [PATCH 08/45] Update pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java Changed oracle to db2 Co-authored-by: Lari Hotari --- .../java/org/apache/pulsar/io/debezium/db2/package-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java b/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java index 7efd1fea05d77..89a015103457c 100644 --- a/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java +++ b/pulsar-io/debezium/db2/src/main/java/org/apache/pulsar/io/debezium/db2/package-info.java @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.io.debezium.oracle; \ No newline at end of file +package org.apache.pulsar.io.debezium.db2; \ No newline at end of file From 3e97cc112eaaa0dad3ab36f7143e8c57da768c5f Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Thu, 30 Mar 2023 16:30:39 -0500 Subject: [PATCH 09/45] Separated db2 connection with subsequent commands --- .../containers/DebeziumDB2DbContainer.java | 26 +++++++++---------- .../debezium/DebeziumDB2DbSourceTester.java | 17 ++++++++---- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 123f924fdb5db..e568d25bc201d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -45,19 +45,19 @@ protected void configure() { super.configure(); this.withNetworkAliases(NAME) .withExposedPorts(PORTS) - .withEnv("LICENSE", "accept") - .withEnv("DB2INSTANCE", "db2inst1") - .withEnv("DB2INST1_PASSWORD", "admin") - .withEnv("DBNAME", "mydb2") - .withEnv("BLU", "false") - .withEnv("ENABLE_ORACLE_COMPATIBILITY", "false") - .withEnv("UPDATEAVAIL", "NO") - .withEnv("TO_CREATE_SAMPLEDB", "false") - .withEnv("REPODB", "false") - .withEnv("IS_OSXFS", "false") - .withEnv("PERSISTENT_HOME", "true") - .withEnv("HADR_ENABLED", "false") - .withPrivilegedMode(true) + .withEnv("LICENSE", "accept") + .withEnv("DB2INSTANCE", "db2inst1") + .withEnv("DB2INST1_PASSWORD", "admin") + .withEnv("DBNAME", "mydb2") + .withEnv("BLU", "false") + .withEnv("ENABLE_ORACLE_COMPATIBILITY", "false") + .withEnv("UPDATEAVAIL", "NO") + .withEnv("TO_CREATE_SAMPLEDB", "false") + .withEnv("REPODB", "false") + .withEnv("IS_OSXFS", "false") + .withEnv("PERSISTENT_HOME", "true") + .withEnv("HADR_ENABLED", "false") + .withPrivilegedMode(true) .withStartupTimeout(Duration.of(300, ChronoUnit.SECONDS)) .withCreateContainerCmdModifier(createContainerCmd -> { createContainerCmd.withHostName(NAME); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index ef721675fb3e9..dc5b31264fe8d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -77,9 +77,17 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { @SneakyThrows @Override public void prepareSource() { - - waitForDB2SetupStatus("KAFKA"); - Thread.sleep(SLEEP_AFTER_COMMAND_MS); + String cmd = "/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin';"; + ContainerExecResult response = this.debeziumDB2DbContainer + .execCmdAsUser("root", + "/bin/bash", "-c", + cmd + ); + if (Strings.isNullOrEmpty(response.getStderr())) { + log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); + } else { + log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); + } runSqlCmd("CREATE TABLE DB2INST1.STORES(store_id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL,store_name VARCHAR(150) NOT NULL,state_id INT NOT NULL,zip_code VARCHAR(6),PRIMARY KEY (store_id));"); runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111');"); @@ -103,8 +111,7 @@ private ContainerExecResult runSqlCmd(String cmd) throws Exception { ContainerExecResult response = this.debeziumDB2DbContainer .execCmdAsUser("root", "/bin/bash", "-c", - "/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin'; " - + "/opt/ibm/db2/V11.5/bin/db2 \"" + cmd + "\"" + "/opt/ibm/db2/V11.5/bin/db2 -x \"" + cmd + "\"" ); if (Strings.isNullOrEmpty(response.getStderr())) { log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); From 817d7dc741736f6a0483a447f8ab45da4eb0467c Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Thu, 30 Mar 2023 17:28:23 -0500 Subject: [PATCH 10/45] Added more startup commands --- .../debezium/DebeziumDB2DbSourceTester.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index dc5b31264fe8d..23dfe018f9b51 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -77,7 +77,16 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { @SneakyThrows @Override public void prepareSource() { - String cmd = "/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin';"; + runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin';"); + runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 bind db2schema.bnd blocking all grant public sqlerror continue"); + + runSqlCmd("VALUES ASNCDC.ASNCDCSERVICES('start','asncdc')"); + + runSqlCmd("CREATE TABLE DB2INST1.STORES(store_id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL,store_name VARCHAR(150) NOT NULL,state_id INT NOT NULL,zip_code VARCHAR(6),PRIMARY KEY (store_id));"); + runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111');"); + runSqlCmd("CALL ASNCDC.ADDTABLE('DB2INST1','STORES')"); + }; + private ContainerExecResult runDb2Cmd(String cmd) throws Exception { ContainerExecResult response = this.debeziumDB2DbContainer .execCmdAsUser("root", "/bin/bash", "-c", @@ -88,22 +97,7 @@ public void prepareSource() { } else { log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); } - - runSqlCmd("CREATE TABLE DB2INST1.STORES(store_id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL,store_name VARCHAR(150) NOT NULL,state_id INT NOT NULL,zip_code VARCHAR(6),PRIMARY KEY (store_id));"); - runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111');"); - runSqlCmd("call ASNCDC.ADDTABLE('DB2INST1','STORES')"); - }; - - private void waitForDB2SetupStatus(String status) throws Exception { - for (int i = 0; i < 1000; i++) { - ContainerExecResult response = runSqlCmd("SELECT target_server from ASNCDC.IBMSNAP_PRUNCNTL;"); - if ((response.getStderr() != null && response.getStderr().contains(status)) - || (response.getStdout() != null && response.getStdout().contains(status))) { - return; - } - Thread.sleep(1000); - } - throw new IllegalStateException("DB2 did not initialize properly"); + return response; } private ContainerExecResult runSqlCmd(String cmd) throws Exception { From 58a4ca0ae9be5a03f1fdd6b1fe125b962ce275d2 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Thu, 30 Mar 2023 17:46:43 -0500 Subject: [PATCH 11/45] Updated image for Debezium test container to use the new DB2 image --- .../debezium-db2-image/Dockerfile | 21 +++++ .../docker-images/debezium-db2-image/pom.xml | 80 +++++++++++++++++++ .../latest-version-image/Dockerfile | 34 ++++---- tests/docker-images/pom.xml | 1 + .../containers/DebeziumDB2DbContainer.java | 2 +- 5 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 tests/docker-images/debezium-db2-image/Dockerfile create mode 100644 tests/docker-images/debezium-db2-image/pom.xml diff --git a/tests/docker-images/debezium-db2-image/Dockerfile b/tests/docker-images/debezium-db2-image/Dockerfile new file mode 100644 index 0000000000000..3932107c9ff80 --- /dev/null +++ b/tests/docker-images/debezium-db2-image/Dockerfile @@ -0,0 +1,21 @@ +FROM ibmcom/db2:11.5.0.0a + +MAINTAINER Devin Bost +# Originally from https://github.com/debezium/debezium-connector-db2/blob/bee92787b1e02bb3e0813a6d8b7f62d34ad6d9e2/src/test/docker/db2-cdc-docker/Dockerfile + +RUN mkdir -p /asncdctools/src + +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc.c" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/dbsetup.sh" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc_UDF.sql" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdcaddremove.sql" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdctables.sql" + +RUN chmod -R 777 /asncdctools + +RUN mkdir /var/custom +RUN chmod -R 777 /var/custom + +RUN cd /var/custom && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/cdcsetup.sh" + +RUN chmod 777 /var/custom/cdcsetup.sh \ No newline at end of file diff --git a/tests/docker-images/debezium-db2-image/pom.xml b/tests/docker-images/debezium-db2-image/pom.xml new file mode 100644 index 0000000000000..6d7588f11d06c --- /dev/null +++ b/tests/docker-images/debezium-db2-image/pom.xml @@ -0,0 +1,80 @@ + + + + org.apache.pulsar.tests + docker-images + 3.0.0-SNAPSHOT + + 4.0.0 + debezium-db2-test-image + Apache Pulsar :: Tests :: Docker Images :: Debezium DB2 Test Image + pom + + + + docker + + target/pulsar-server-distribution-bin.tar.gz + ${env.UBUNTU_MIRROR} + ${env.UBUNTU_SECURITY_MIRROR} + + + + integrationTests + + + + + + io.fabric8 + docker-maven-plugin + + + default + package + + build + + + + + ${docker.organization}/debezium-db2-test-image + + ${project.basedir} + + latest + ${project.version} + + true + + + + + + + + + + + + diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 42927ec28801d..43c2ab190c8a2 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -117,28 +117,28 @@ COPY --from=pulsar-all /pulsar/connectors/pulsar-io-kinesis-*.nar /pulsar/connec # download Oracle JDBC driver for Oracle Debezium Connector tests RUN mkdir -p META-INF/bundled-dependencies -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ojdbc8/19.3.0.0/ojdbc8-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ucp/19.3.0.0/ucp-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/oraclepki/19.3.0.0/oraclepki-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_cert/19.3.0.0/osdt_cert-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_core/19.3.0.0/osdt_core-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/simplefan/19.3.0.0/simplefan-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/orai18n/19.3.0.0/orai18n-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xdb/19.3.0.0/xdb-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xmlparserv2/19.3.0.0/xmlparserv2-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ojdbc8/19.3.0.0/ojdbc8-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ucp/19.3.0.0/ucp-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/oraclepki/19.3.0.0/oraclepki-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_cert/19.3.0.0/osdt_cert-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_core/19.3.0.0/osdt_core-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/simplefan/19.3.0.0/simplefan-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/orai18n/19.3.0.0/orai18n-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xdb/19.3.0.0/xdb-19.3.0.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xmlparserv2/19.3.0.0/xmlparserv2-19.3.0.0.jar" RUN jar uf connectors/pulsar-io-debezium-oracle-*.nar META-INF/bundled-dependencies/ojdbc8-19.3.0.0.jar META-INF/bundled-dependencies/ucp-19.3.0.0.jar META-INF/bundled-dependencies/oraclepki-19.3.0.0.jar META-INF/bundled-dependencies/osdt_cert-19.3.0.0.jar META-INF/bundled-dependencies/osdt_core-19.3.0.0.jar META-INF/bundled-dependencies/simplefan-19.3.0.0.jar META-INF/bundled-dependencies/orai18n-19.3.0.0.jar META-INF/bundled-dependencies/xdb-19.3.0.0.jar META-INF/bundled-dependencies/xmlparserv2-19.3.0.0.jar # Download IBM DB2 JDBC driver for DB2 Debezium Connector tests: -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/ibm/db2/jcc/11.5.8.0/jcc-11.5.8.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/ibm/db2/jcc/db2jcc/db2jcc4/db2jcc-db2jcc4.jar +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/ibm/db2/jcc/11.5.8.0/jcc-11.5.8.0.jar" +RUN cd META-INF/bundled-dependencies && curl -sSLO "https://search.maven.org/remotecontent?filepath=com/ibm/db2/jcc/db2jcc/db2jcc4/db2jcc-db2jcc4.jar" RUN mkdir -p /asncdctools/src -RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc.c -RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/dbsetup.sh -RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc_UDF.sql -RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdcaddremove.sql -RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdctables.sql -RUN cd /asncdctools/src && curl -sSLO https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/cdcsetup.sh +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc.c" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/dbsetup.sh" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdc_UDF.sql" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdcaddremove.sql" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/asncdctables.sql" +RUN cd /asncdctools/src && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/cdcsetup.sh" RUN jar uf connectors/pulsar-io-debezium-db2-*.nar META-INF/bundled-dependencies/jcc-11.5.8.0.jar META-INF/bundled-dependencies/db2jcc-db2jcc4.jar CMD bash diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index ade69e0778b04..c05f7d7ed47b6 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -51,6 +51,7 @@ java-test-plugins latest-version-image java-test-image + debezium-db2-test-image diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index e568d25bc201d..28162dc68eb52 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -29,7 +29,7 @@ public class DebeziumDB2DbContainer extends ChaosContainer Date: Thu, 30 Mar 2023 19:59:49 -0500 Subject: [PATCH 12/45] Fixed image reference --- .../{debezium-db2-image => debezium-db2-test-image}/Dockerfile | 0 .../{debezium-db2-image => debezium-db2-test-image}/pom.xml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/docker-images/{debezium-db2-image => debezium-db2-test-image}/Dockerfile (100%) rename tests/docker-images/{debezium-db2-image => debezium-db2-test-image}/pom.xml (100%) diff --git a/tests/docker-images/debezium-db2-image/Dockerfile b/tests/docker-images/debezium-db2-test-image/Dockerfile similarity index 100% rename from tests/docker-images/debezium-db2-image/Dockerfile rename to tests/docker-images/debezium-db2-test-image/Dockerfile diff --git a/tests/docker-images/debezium-db2-image/pom.xml b/tests/docker-images/debezium-db2-test-image/pom.xml similarity index 100% rename from tests/docker-images/debezium-db2-image/pom.xml rename to tests/docker-images/debezium-db2-test-image/pom.xml From e6b55991ca5d729b0d2f445eee114048fd2d1473 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 04:04:20 -0500 Subject: [PATCH 13/45] Fix license header --- .../debezium-db2-test-image/Dockerfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/docker-images/debezium-db2-test-image/Dockerfile b/tests/docker-images/debezium-db2-test-image/Dockerfile index 3932107c9ff80..aa68206f6e797 100644 --- a/tests/docker-images/debezium-db2-test-image/Dockerfile +++ b/tests/docker-images/debezium-db2-test-image/Dockerfile @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# FROM ibmcom/db2:11.5.0.0a MAINTAINER Devin Bost From 262d7c616d6029561d0ae39ae005b19c26afcacc Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 04:31:04 -0500 Subject: [PATCH 14/45] Fixed format of license header --- tests/docker-images/java-test-image/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 9e852050edf2e..5cfaa86a7abdd 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. # - FROM ubuntu:20.04 RUN groupadd -g 10001 pulsar From d29db07e6b9fd763a0cc6329908e1738eb292607 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 04:41:47 -0500 Subject: [PATCH 15/45] Fixed license format by running mvn initialize license:format --- tests/docker-images/debezium-db2-test-image/Dockerfile | 1 + tests/docker-images/java-test-image/Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/docker-images/debezium-db2-test-image/Dockerfile b/tests/docker-images/debezium-db2-test-image/Dockerfile index aa68206f6e797..d4b9d4599dd20 100644 --- a/tests/docker-images/debezium-db2-test-image/Dockerfile +++ b/tests/docker-images/debezium-db2-test-image/Dockerfile @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. # + FROM ibmcom/db2:11.5.0.0a MAINTAINER Devin Bost diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 5cfaa86a7abdd..9e852050edf2e 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. # + FROM ubuntu:20.04 RUN groupadd -g 10001 pulsar From e493cf8922b2f3a00d528a55e38fe4661e156dfc Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 06:24:22 -0500 Subject: [PATCH 16/45] Added more license details and updated CI to build docker image for DB2 --- .github/workflows/pulsar-ci.yaml | 89 ++++++++++++++++++- .../debezium-db2-test-image/Dockerfile | 15 ++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 721a1d2eafc72..79c734c4186ea 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -459,12 +459,85 @@ jobs: continue-on-error: true with: action: wait + pulsar-debezium-db2-test-image: + name: Build Pulsar debezium-db2-test-image docker image + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: ['preconditions', 'build-and-license-check'] + if: ${{ needs.preconditions.outputs.docs_only != 'true'}} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Pick ubuntu mirror for the docker image build + run: | + # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror + + - name: Build debezium-db2-test-image docker image + run: | + # build docker image + mvn -B -am -pl tests/docker-images/debezium-db2-test-image install -Pcore-modules,-main,integrationTests,docker \ + -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \ + -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true + + - name: save docker image apachepulsar/debezium-db2-test-image:latest to Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/debezium-db2-test-image:latest pulsar-debezium-db2-test-image + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait integration-tests: name: CI - Integration - ${{ matrix.name }} runs-on: ubuntu-20.04 timeout-minutes: ${{ matrix.timeout || 60 }} - needs: ['preconditions', 'pulsar-java-test-image'] + needs: ['preconditions', 'pulsar-java-test-image', 'pulsar-debezium-db2-test-image'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI - Integration - ${{ matrix.name }} @@ -556,6 +629,10 @@ jobs: run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image + - name: Load docker image apachepulsar/debezium-db2-test-image:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-debezium-db2-test-image + - name: Run setup commands if: ${{ matrix.setup }} run: | @@ -672,6 +749,10 @@ jobs: run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image + - name: Load docker image apachepulsar/debezium-db2-test-image:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-debezium-db2-test-image + - name: Restore coverage files from build artifacts and create Jacoco reports run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files @@ -705,7 +786,7 @@ jobs: action: wait delete-integration-test-docker-image-artifact: - name: "Delete integration test docker image artifact" + name: "Delete integration test docker image artifacts" runs-on: ubuntu-20.04 timeout-minutes: 10 needs: [ @@ -733,6 +814,10 @@ jobs: run: | gh-actions-artifact-client.js delete pulsar-java-test-image.zst + - name: Delete db2 docker image from GitHub Actions Artifacts + run: | + gh-actions-artifact-client.js delete pulsar-debezium-db2-test-image.zst + pulsar-test-latest-version-image: name: Build Pulsar docker image runs-on: ubuntu-20.04 diff --git a/tests/docker-images/debezium-db2-test-image/Dockerfile b/tests/docker-images/debezium-db2-test-image/Dockerfile index d4b9d4599dd20..ff83cc99eb3d6 100644 --- a/tests/docker-images/debezium-db2-test-image/Dockerfile +++ b/tests/docker-images/debezium-db2-test-image/Dockerfile @@ -19,6 +19,21 @@ FROM ibmcom/db2:11.5.0.0a +# As of 3/31/2023: +# "If the Program is designated as "Non-Production", the Program can only be deployed as part of the Licensee's internal development and test environment +# for internal non-production activities, including but not limited to testing, performance +# tuning, fault diagnosis, internal benchmarking, staging, quality assurance activity +# and/or developing internally used additions or extensions to the Program using published +# application programming interfaces. Licensee is not authorized to use any part of the +# Program for any other purposes without acquiring the appropriate production entitlements." +# (https://www.ibm.com/support/customer/csol/terms/?id=L-GMET-8KJN5X&lc=en#detail-document) +# Additionally: +# "Client license files are not required for connections to Db2, Cloudscape, +# or IBM Informix® databases from the IBM Data Server Driver for JDBC and SQLJ version +# 3.50 or later. Client license files are not required for direct connections to Db2 for +# z/OS if Db2 Connect Unlimited Edition for System z® server license activation is performed." +# (https://www.ibm.com/docs/en/db2/11.5?topic=apis-installing-data-server-driver-jdbc-sqlj) + MAINTAINER Devin Bost # Originally from https://github.com/debezium/debezium-connector-db2/blob/bee92787b1e02bb3e0813a6d8b7f62d34ad6d9e2/src/test/docker/db2-cdc-docker/Dockerfile From fa9188ef27aef6fbd56155fa1b942259f964df24 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 08:12:38 -0500 Subject: [PATCH 17/45] Added DB2 image to Pulsar IO step in CI --- .github/workflows/pulsar-ci.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 79c734c4186ea..160188e1807f3 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -930,7 +930,7 @@ jobs: name: CI - System - ${{ matrix.name }} runs-on: ubuntu-20.04 timeout-minutes: 60 - needs: ['preconditions', 'pulsar-test-latest-version-image'] + needs: ['preconditions', 'pulsar-test-latest-version-image', 'pulsar-debezium-db2-test-image'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI - System - ${{ matrix.name }} @@ -1014,6 +1014,10 @@ jobs: run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + - name: Load docker image apachepulsar/debezium-db2-test-image:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-debezium-db2-test-image + - name: Run setup commands if: ${{ matrix.setup }} run: | From 4f052722232e3665a977e6a73613ad5a100ac4b7 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 10:08:32 -0500 Subject: [PATCH 18/45] Added check to ensure that commands aren't run on DB2 until DB finishes setup --- .../debezium-db2-test-image/Dockerfile | 4 +++- .../sources/debezium/DebeziumDB2DbSourceTester.java | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/docker-images/debezium-db2-test-image/Dockerfile b/tests/docker-images/debezium-db2-test-image/Dockerfile index ff83cc99eb3d6..723ea04fe1f08 100644 --- a/tests/docker-images/debezium-db2-test-image/Dockerfile +++ b/tests/docker-images/debezium-db2-test-image/Dockerfile @@ -52,4 +52,6 @@ RUN chmod -R 777 /var/custom RUN cd /var/custom && curl -sSLO "https://raw.githubusercontent.com/debezium/debezium-connector-db2/main/src/test/docker/db2-cdc-docker/cdcsetup.sh" -RUN chmod 777 /var/custom/cdcsetup.sh \ No newline at end of file +RUN chmod 777 /var/custom/cdcsetup.sh + +RUN chmod 554 /opt/ibm/db2/V11.5/adm/db2start \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 23dfe018f9b51..243044528a354 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -77,6 +77,8 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { @SneakyThrows @Override public void prepareSource() { + Thread.sleep(300 * 1000); // Startup takes at least 300 seconds. + waitForDB2Startup("active"); runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin';"); runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 bind db2schema.bnd blocking all grant public sqlerror continue"); @@ -86,6 +88,17 @@ public void prepareSource() { runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111');"); runSqlCmd("CALL ASNCDC.ADDTABLE('DB2INST1','STORES')"); }; + private void waitForDB2Startup(String status) throws Exception { + for (int i = 0; i < 1000; i++) { + ContainerExecResult response = runDb2Cmd("/opt/ibm/db2/V11.5/adm/db2start"); + if ((response.getStderr() != null && response.getStderr().contains(status)) + || (response.getStdout() != null && response.getStdout().contains(status))) { + return; + } + Thread.sleep(1000); + } + throw new IllegalStateException("DB2 did not startup properly"); + } private ContainerExecResult runDb2Cmd(String cmd) throws Exception { ContainerExecResult response = this.debeziumDB2DbContainer .execCmdAsUser("root", From b04c23e222507fa9f90451fb1bf1a22e48f9262c Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 10:22:29 -0500 Subject: [PATCH 19/45] Added wait checks to ensure DB2 has initialized before connecting --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 243044528a354..9a4e8ba835e3a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -78,8 +78,8 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { @Override public void prepareSource() { Thread.sleep(300 * 1000); // Startup takes at least 300 seconds. - waitForDB2Startup("active"); - runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin';"); + waitForDB2Startup("/opt/ibm/db2/V11.5/adm/db2start","active"); + waitForDB2Startup("/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin'","authorization"); runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 bind db2schema.bnd blocking all grant public sqlerror continue"); runSqlCmd("VALUES ASNCDC.ASNCDCSERVICES('start','asncdc')"); @@ -88,9 +88,9 @@ public void prepareSource() { runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111');"); runSqlCmd("CALL ASNCDC.ADDTABLE('DB2INST1','STORES')"); }; - private void waitForDB2Startup(String status) throws Exception { + private void waitForDB2Startup(String cmd, String status) throws Exception { for (int i = 0; i < 1000; i++) { - ContainerExecResult response = runDb2Cmd("/opt/ibm/db2/V11.5/adm/db2start"); + ContainerExecResult response = runDb2Cmd(cmd); if ((response.getStderr() != null && response.getStderr().contains(status)) || (response.getStdout() != null && response.getStdout().contains(status))) { return; From 5ab21637f112ecab9f178d36214dfd49fac9ef39 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 11:59:26 -0500 Subject: [PATCH 20/45] Added more logging to troubleshoot container startup --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 9a4e8ba835e3a..17c36577b876e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -93,6 +93,7 @@ private void waitForDB2Startup(String cmd, String status) throws Exception { ContainerExecResult response = runDb2Cmd(cmd); if ((response.getStderr() != null && response.getStderr().contains(status)) || (response.getStdout() != null && response.getStdout().contains(status))) { + log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); return; } Thread.sleep(1000); From c6812ef1de231ebce368f283dc23c63fada28010 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 12:00:23 -0500 Subject: [PATCH 21/45] Added more logging to troubleshoot container startup if there are errors --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 17c36577b876e..a49b79e724593 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -93,7 +93,11 @@ private void waitForDB2Startup(String cmd, String status) throws Exception { ContainerExecResult response = runDb2Cmd(cmd); if ((response.getStderr() != null && response.getStderr().contains(status)) || (response.getStdout() != null && response.getStdout().contains(status))) { - log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); + if (Strings.isNullOrEmpty(response.getStderr())) { + log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); + } else { + log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); + } return; } Thread.sleep(1000); From 77557d182f8da25a427fc4756a59aa3882e2a461 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 13:40:37 -0500 Subject: [PATCH 22/45] Add more checks since something is timing out in the test --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index a49b79e724593..c535997c33d2b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -77,9 +77,11 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { @SneakyThrows @Override public void prepareSource() { - Thread.sleep(300 * 1000); // Startup takes at least 300 seconds. + log.info("Starting DB2"); waitForDB2Startup("/opt/ibm/db2/V11.5/adm/db2start","active"); + log.info("DB2 Started. Establishing connection."); waitForDB2Startup("/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin'","authorization"); + log.info("DB2 Connection established. Preparing CDC."); runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 bind db2schema.bnd blocking all grant public sqlerror continue"); runSqlCmd("VALUES ASNCDC.ASNCDCSERVICES('start','asncdc')"); From dd7515e2992bd7200a594c138ae2da9bdb9d36e7 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 13:42:34 -0500 Subject: [PATCH 23/45] Increasing timeout since DB2 takes a while to start --- .../tests/integration/containers/DebeziumDB2DbContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 28162dc68eb52..b4d33f0eb0e7c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -58,7 +58,7 @@ protected void configure() { .withEnv("PERSISTENT_HOME", "true") .withEnv("HADR_ENABLED", "false") .withPrivilegedMode(true) - .withStartupTimeout(Duration.of(300, ChronoUnit.SECONDS)) + .withStartupTimeout(Duration.of(600, ChronoUnit.SECONDS)) .withCreateContainerCmdModifier(createContainerCmd -> { createContainerCmd.withHostName(NAME); createContainerCmd.withName(getContainerName()); From bf0d3e1a839a375a8c4b426dff2df725e266f1d2 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 13:43:22 -0500 Subject: [PATCH 24/45] Increasing timeout since DB2 takes a while to start. --- .../tests/integration/containers/DebeziumDB2DbContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index b4d33f0eb0e7c..9a27b8a070c27 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -58,7 +58,7 @@ protected void configure() { .withEnv("PERSISTENT_HOME", "true") .withEnv("HADR_ENABLED", "false") .withPrivilegedMode(true) - .withStartupTimeout(Duration.of(600, ChronoUnit.SECONDS)) + .withStartupTimeout(Duration.of(900, ChronoUnit.SECONDS)) .withCreateContainerCmdModifier(createContainerCmd -> { createContainerCmd.withHostName(NAME); createContainerCmd.withName(getContainerName()); From 414738c6a17d60fc417625a093e240b6ac6493f4 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 16:00:38 -0500 Subject: [PATCH 25/45] Increasing wait time between subsequent commands --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index c535997c33d2b..74b84378cca5d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -102,7 +102,7 @@ private void waitForDB2Startup(String cmd, String status) throws Exception { } return; } - Thread.sleep(1000); + Thread.sleep(20000); } throw new IllegalStateException("DB2 did not startup properly"); } From a9673dec922beb559079e8f39e3def983f60cff2 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 31 Mar 2023 16:17:45 -0500 Subject: [PATCH 26/45] Catching exception and printing more debug info --- .../debezium/DebeziumDB2DbSourceTester.java | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 74b84378cca5d..735584c66e839 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -19,12 +19,16 @@ package org.apache.pulsar.tests.integration.io.sources.debezium; import com.google.common.base.Preconditions; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.Map; +import java.util.concurrent.ExecutionException; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumDB2DbContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.docker.ContainerExecException; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; @@ -92,15 +96,25 @@ public void prepareSource() { }; private void waitForDB2Startup(String cmd, String status) throws Exception { for (int i = 0; i < 1000; i++) { - ContainerExecResult response = runDb2Cmd(cmd); - if ((response.getStderr() != null && response.getStderr().contains(status)) - || (response.getStdout() != null && response.getStdout().contains(status))) { - if (Strings.isNullOrEmpty(response.getStderr())) { - log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); - } else { - log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); + try { + ContainerExecResult response = runDb2Cmd(cmd); + if ((response.getStderr() != null && response.getStderr().contains(status)) + || (response.getStdout() != null && response.getStdout().contains(status))) { + if (Strings.isNullOrEmpty(response.getStderr())) { + log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); + } else { + log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); + } + return; + } + } + catch (ExecutionException e) { + if (e.getCause() instanceof ContainerExecException) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String exceptionAsString = sw.toString(); + log.info("Command failed. Message is {}. StackTrace is {}", e.getMessage(), exceptionAsString); } - return; } Thread.sleep(20000); } From 49c342fa3df75204cf0a2f3e9d5064ba58ad60ff Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Mon, 3 Apr 2023 17:47:39 -0500 Subject: [PATCH 27/45] Refactored DB2 testing to use JDBC driver --- tests/integration/pom.xml | 12 +++ .../containers/DebeziumDB2DbContainer.java | 85 +++++++++++++++-- .../debezium/DebeziumDB2DbSourceTester.java | 91 +++---------------- 3 files changed, 106 insertions(+), 82 deletions(-) diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index d9b817ca2e2b5..574870076fbd7 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -101,6 +101,18 @@ ${project.version} test + + + com.ibm.db2 + jcc + 11.5.8.0 + test + org.testcontainers diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 9a27b8a070c27..624da267b9791 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -19,9 +19,12 @@ package org.apache.pulsar.tests.integration.containers; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.time.Duration; import java.time.temporal.ChronoUnit; -import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; public class DebeziumDB2DbContainer extends ChaosContainer { @@ -30,6 +33,19 @@ public class DebeziumDB2DbContainer extends ChaosContainer { createContainerCmd.withHostName(NAME); createContainerCmd.withName(getContainerName()); - }) - .waitingFor(new HostPortWaitStrategy()); + }); + } + public String getDriverClassName() { + return "com.ibm.db2.jcc.DB2Driver"; + } + public String getUsername(){ + return USERNAME; + } + public String getPassword(){ + return PASS; + } + + public String getJdbcUrl() { + return "jdbc:db2://" + this.getHost() + ":" + this.getMappedPort(50000) + "/" + DBNAME; + } + + public String createTableStatement(){ + return String.format("CREATE TABLE %s.STORES(store_id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL,store_name VARCHAR(150) NOT NULL,state_id INT NOT NULL,zip_code VARCHAR(6),PRIMARY KEY (store_id))", + getUsername().toUpperCase()); + } + public String insertStatement(){ + return String.format("INSERT INTO %s.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111')", + getUsername().toUpperCase()); + } + public String selectStatement(){ + return String.format("SELECT * FROM %s.STORES", + getUsername().toUpperCase()); + } + private Connection connection; + public Connection getConnection() throws SQLException, ClassNotFoundException { + if(connection == null){ + Class.forName("com.ibm.db2.jcc.DB2Driver"); + var conn = DriverManager.getConnection(this.getJdbcUrl(), getUsername(), getPassword()); + this.connection = conn; + } + return connection; + } + public String enableCdcStatement(){ + return "VALUES ASNCDC.ASNCDCSERVICES('start','asncdc')"; + } + public String addCdcTableStatement(){ + return String.format("CALL ASNCDC.ADDTABLE('%s','STORES')", getUsername()); + } + public PreparedStatement executePreparedStatement(String sqlQuery) throws SQLException, ClassNotFoundException { + var myConnection = this.getConnection(); + return myConnection.prepareStatement(sqlQuery); } + public String getStoreNameResult(PreparedStatement statement) throws SQLException { + var result = statement.execute(); + if(result == true) + { + var resultSet = statement.getResultSet(); + while(resultSet.next()) + { + var storeName = resultSet.getString("store_name"); + return storeName; + } + } + return null; + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 735584c66e839..a01917ca3d0b9 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -19,20 +19,14 @@ package org.apache.pulsar.tests.integration.io.sources.debezium; import com.google.common.base.Preconditions; -import java.io.PrintWriter; -import java.io.StringWriter; import java.util.Map; -import java.util.concurrent.ExecutionException; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumDB2DbContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; -import org.apache.pulsar.tests.integration.docker.ContainerExecException; -import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; -import org.testng.util.Strings; /** * A tester for testing Debezium DB2 source. @@ -82,89 +76,34 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { @Override public void prepareSource() { log.info("Starting DB2"); - waitForDB2Startup("/opt/ibm/db2/V11.5/adm/db2start","active"); - log.info("DB2 Started. Establishing connection."); - waitForDB2Startup("/opt/ibm/db2/V11.5/bin/db2 connect to mydb2 user 'db2inst1' using 'admin'","authorization"); - log.info("DB2 Connection established. Preparing CDC."); - runDb2Cmd("/opt/ibm/db2/V11.5/bin/db2 bind db2schema.bnd blocking all grant public sqlerror continue"); - - runSqlCmd("VALUES ASNCDC.ASNCDCSERVICES('start','asncdc')"); - - runSqlCmd("CREATE TABLE DB2INST1.STORES(store_id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL,store_name VARCHAR(150) NOT NULL,state_id INT NOT NULL,zip_code VARCHAR(6),PRIMARY KEY (store_id));"); - runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore', 12, '11111');"); - runSqlCmd("CALL ASNCDC.ADDTABLE('DB2INST1','STORES')"); - }; - private void waitForDB2Startup(String cmd, String status) throws Exception { - for (int i = 0; i < 1000; i++) { - try { - ContainerExecResult response = runDb2Cmd(cmd); - if ((response.getStderr() != null && response.getStderr().contains(status)) - || (response.getStdout() != null && response.getStdout().contains(status))) { - if (Strings.isNullOrEmpty(response.getStderr())) { - log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); - } else { - log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); - } - return; - } - } - catch (ExecutionException e) { - if (e.getCause() instanceof ContainerExecException) { - StringWriter sw = new StringWriter(); - e.printStackTrace(new PrintWriter(sw)); - String exceptionAsString = sw.toString(); - log.info("Command failed. Message is {}. StackTrace is {}", e.getMessage(), exceptionAsString); - } - } - Thread.sleep(20000); - } - throw new IllegalStateException("DB2 did not startup properly"); - } - private ContainerExecResult runDb2Cmd(String cmd) throws Exception { - ContainerExecResult response = this.debeziumDB2DbContainer - .execCmdAsUser("root", - "/bin/bash", "-c", - cmd - ); - if (Strings.isNullOrEmpty(response.getStderr())) { - log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); - } else { - log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); - } - return response; - } - private ContainerExecResult runSqlCmd(String cmd) throws Exception { - log.info("Executing \"{}\"", cmd); - ContainerExecResult response = this.debeziumDB2DbContainer - .execCmdAsUser("root", - "/bin/bash", "-c", - "/opt/ibm/db2/V11.5/bin/db2 -x \"" + cmd + "\"" - ); - if (Strings.isNullOrEmpty(response.getStderr())) { - log.info("Result of \"{}\":\n{}", cmd, response.getStdout()); - } else { - log.warn("Result of \"{}\":\n{}\n{}", cmd, response.getStdout(), response.getStderr()); - } - return response; + debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.enableCdcStatement()); + debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.createTableStatement()); + debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()); + + debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.insertStatement()); + + var stmt = debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.selectStatement()); + var result = debeziumDB2DbContainer.getStoreNameResult(stmt); + } @Override public void prepareInsertEvent() throws Exception { - runSqlCmd("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore2', 2, '22222');"); - runSqlCmd("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2';"); + debeziumDB2DbContainer.executePreparedStatement("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore2', 2, '22222')"); + debeziumDB2DbContainer.executePreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override public void prepareDeleteEvent() throws Exception { - runSqlCmd("DELETE FROM DB2INST1.STORES WHERE store_name='mystore2';"); - runSqlCmd("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2';"); + debeziumDB2DbContainer.executePreparedStatement("DELETE FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.executePreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override public void prepareUpdateEvent() throws Exception { - runSqlCmd("UPDATE DB2INST1.STORES SET zip_code='33333' WHERE store_name='mystore2';"); - runSqlCmd("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2';"); + debeziumDB2DbContainer.executePreparedStatement("UPDATE DB2INST1.STORES SET zip_code='33333' WHERE store_name='mystore2'"); + debeziumDB2DbContainer.executePreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override From 4cdd64b5bf29bab4b4275c5e90ffcc8679642831 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 04:45:33 -0500 Subject: [PATCH 28/45] Added JDBC logging for DB2 --- .../tests/integration/containers/DebeziumDB2DbContainer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 624da267b9791..d5960dfb76439 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -25,7 +25,9 @@ import java.sql.SQLException; import java.time.Duration; import java.time.temporal.ChronoUnit; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class DebeziumDB2DbContainer extends ChaosContainer { public static final String NAME = "debezium-db2"; @@ -108,6 +110,7 @@ public String selectStatement(){ } private Connection connection; public Connection getConnection() throws SQLException, ClassNotFoundException { + log.info(String.format("JDBC Connection URL is: %s", this.getJdbcUrl())); if(connection == null){ Class.forName("com.ibm.db2.jcc.DB2Driver"); var conn = DriverManager.getConnection(this.getJdbcUrl(), getUsername(), getPassword()); From 5ffe1e4aadb4c93bed3ec0351a3246b4a2bf56ba Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 04:50:58 -0500 Subject: [PATCH 29/45] Added test wait strategy based on testcontainers DB2 container --- .../tests/integration/containers/DebeziumDB2DbContainer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index d5960dfb76439..ef4e2d81a9930 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -26,6 +26,7 @@ import java.time.Duration; import java.time.temporal.ChronoUnit; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; @Slf4j public class DebeziumDB2DbContainer extends ChaosContainer { @@ -80,7 +81,8 @@ protected void configure() { .withCreateContainerCmdModifier(createContainerCmd -> { createContainerCmd.withHostName(NAME); createContainerCmd.withName(getContainerName()); - }); + }).waitingFor((new LogMessageWaitStrategy()).withRegEx(".*Setup has completed\\..*") + .withStartupTimeout(Duration.of(10L, ChronoUnit.MINUTES)); } public String getDriverClassName() { return "com.ibm.db2.jcc.DB2Driver"; From ab96ad1a3af99ecfb9af5dfa2318ab4e2f6fad69 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 04:51:35 -0500 Subject: [PATCH 30/45] fixed typo --- .../tests/integration/containers/DebeziumDB2DbContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index ef4e2d81a9930..9a46ddd7c016c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -82,7 +82,7 @@ protected void configure() { createContainerCmd.withHostName(NAME); createContainerCmd.withName(getContainerName()); }).waitingFor((new LogMessageWaitStrategy()).withRegEx(".*Setup has completed\\..*") - .withStartupTimeout(Duration.of(10L, ChronoUnit.MINUTES)); + .withStartupTimeout(Duration.of(10L, ChronoUnit.MINUTES))); } public String getDriverClassName() { return "com.ibm.db2.jcc.DB2Driver"; From 189ef39c87666a3bc97bb866c56b1130d8cd7601 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 06:19:43 -0500 Subject: [PATCH 31/45] Fixed issue with DB2 calls in test --- .../containers/DebeziumDB2DbContainer.java | 5 +-- .../debezium/DebeziumDB2DbSourceTester.java | 31 ++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 9a46ddd7c016c..e05dcfd3325b4 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -126,12 +126,13 @@ public String enableCdcStatement(){ public String addCdcTableStatement(){ return String.format("CALL ASNCDC.ADDTABLE('%s','STORES')", getUsername()); } - public PreparedStatement executePreparedStatement(String sqlQuery) throws SQLException, ClassNotFoundException { + public PreparedStatement getPreparedStatement(String sqlQuery) throws SQLException, ClassNotFoundException { var myConnection = this.getConnection(); return myConnection.prepareStatement(sqlQuery); } - public String getStoreNameResult(PreparedStatement statement) throws SQLException { + public String getStoreNameResult() throws SQLException, ClassNotFoundException { + var statement = this.getPreparedStatement(this.selectStatement()); var result = statement.execute(); if(result == true) { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index a01917ca3d0b9..d82ed1d1e32cf 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -77,33 +77,34 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { public void prepareSource() { log.info("Starting DB2"); - debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.enableCdcStatement()); - debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.createTableStatement()); - debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()); - - debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.insertStatement()); - - var stmt = debeziumDB2DbContainer.executePreparedStatement(debeziumDB2DbContainer.selectStatement()); - var result = debeziumDB2DbContainer.getStoreNameResult(stmt); - + debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.enableCdcStatement()) + .execute(); + debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.createTableStatement()) + .execute(); + debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) + .execute(); + debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.insertStatement()) + .execute(); + var result = debeziumDB2DbContainer.getStoreNameResult(); + log.info("debeziumDB2DbContainer.getStoreNameResult() returned: %s", result); } @Override public void prepareInsertEvent() throws Exception { - debeziumDB2DbContainer.executePreparedStatement("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore2', 2, '22222')"); - debeziumDB2DbContainer.executePreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore2', 2, '22222')"); + debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override public void prepareDeleteEvent() throws Exception { - debeziumDB2DbContainer.executePreparedStatement("DELETE FROM DB2INST1.STORES WHERE store_name='mystore2'"); - debeziumDB2DbContainer.executePreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("DELETE FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override public void prepareUpdateEvent() throws Exception { - debeziumDB2DbContainer.executePreparedStatement("UPDATE DB2INST1.STORES SET zip_code='33333' WHERE store_name='mystore2'"); - debeziumDB2DbContainer.executePreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("UPDATE DB2INST1.STORES SET zip_code='33333' WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override From e4002619fd16dd0835c07b344181a104ebb3b06c Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 08:36:30 -0500 Subject: [PATCH 32/45] Removed CDC logic from test --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index d82ed1d1e32cf..c6578f5e30979 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -77,12 +77,12 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { public void prepareSource() { log.info("Starting DB2"); - debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.enableCdcStatement()) - .execute(); + //debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.enableCdcStatement()) + // .execute(); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.createTableStatement()) .execute(); - debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) - .execute(); + //debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) + // .execute(); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.insertStatement()) .execute(); var result = debeziumDB2DbContainer.getStoreNameResult(); From 2b2de0de5b451975548ebcc893a4bc6ddb3487b8 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 10:15:27 -0500 Subject: [PATCH 33/45] Added step to setup CDC --- .../debezium/DebeziumDB2DbSourceTester.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index c6578f5e30979..9a1deedf65a13 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -25,8 +25,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumDB2DbContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.testng.util.Strings; /** * A tester for testing Debezium DB2 source. @@ -77,12 +79,23 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { public void prepareSource() { log.info("Starting DB2"); - //debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.enableCdcStatement()) - // .execute(); + ContainerExecResult response = this.debeziumDB2DbContainer + .execCmdAsUser("root", + "/bin/bash", "-c", + "/var/custom/cdcsetup.sh" + ); + if (Strings.isNullOrEmpty(response.getStderr())) { + log.info("Result of \"{}\":\n{}", "/var/custom/cdcsetup.sh", response.getStdout()); + } else { + log.warn("Result of \"{}\":\n{}\n{}", "/var/custom/cdcsetup.sh", response.getStdout(), response.getStderr()); + } + + debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.enableCdcStatement()) + .execute(); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.createTableStatement()) .execute(); - //debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) - // .execute(); + debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) + .execute(); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.insertStatement()) .execute(); var result = debeziumDB2DbContainer.getStoreNameResult(); From 970fd53719d91368e2dacb9a638109e8d9d6a391 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 11:50:49 -0500 Subject: [PATCH 34/45] Refactored cdcsetup.sh execution to be synchronous to prevent race on DB --- .../debezium/DebeziumDB2DbSourceTester.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 9a1deedf65a13..047e51e32802c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -77,27 +77,28 @@ public void setServiceContainer(DebeziumDB2DbContainer container) { @SneakyThrows @Override public void prepareSource() { - log.info("Starting DB2"); - - ContainerExecResult response = this.debeziumDB2DbContainer - .execCmdAsUser("root", - "/bin/bash", "-c", - "/var/custom/cdcsetup.sh" - ); - if (Strings.isNullOrEmpty(response.getStderr())) { - log.info("Result of \"{}\":\n{}", "/var/custom/cdcsetup.sh", response.getStdout()); - } else { - log.warn("Result of \"{}\":\n{}\n{}", "/var/custom/cdcsetup.sh", response.getStdout(), response.getStderr()); - } - + log.info("Starting DB2. Running cdcsetup.sh next"); + var lsResult = debeziumDB2DbContainer.execInContainer("/bin/bash", "-c", "/var/custom/cdcsetup.sh"); + String stdout = lsResult.getStdout(); + log.info(String.format("Output of running cdcsetup.sh is: %s", stdout)); + int exitCode = lsResult.getExitCode(); + log.info(String.format("Running : %s", debeziumDB2DbContainer.enableCdcStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.enableCdcStatement()) .execute(); + + log.info(String.format("Running : %s", debeziumDB2DbContainer.createTableStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.createTableStatement()) .execute(); + + log.info(String.format("Running : %s", debeziumDB2DbContainer.addCdcTableStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) .execute(); + + log.info(String.format("Running : %s", debeziumDB2DbContainer.insertStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.insertStatement()) .execute(); + + log.info(String.format("Running : %s", debeziumDB2DbContainer.selectStatement())); var result = debeziumDB2DbContainer.getStoreNameResult(); log.info("debeziumDB2DbContainer.getStoreNameResult() returned: %s", result); } From f7bcb0d37a4edfe09eb4b4ddadd7842e7ad52932 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 13:22:23 -0500 Subject: [PATCH 35/45] Fixed style checks --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 047e51e32802c..1478f450ca040 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -25,10 +25,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumDB2DbContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; -import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; -import org.testng.util.Strings; /** * A tester for testing Debezium DB2 source. From 992aa52be9f6c4330d0c069060351cc546a7c6a9 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Tue, 4 Apr 2023 15:04:07 -0500 Subject: [PATCH 36/45] Fixed case sensitive values --- .../integration/containers/DebeziumDB2DbContainer.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index e05dcfd3325b4..91d85bab1ac01 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -47,7 +47,7 @@ public class DebeziumDB2DbContainer extends ChaosContainer Date: Tue, 4 Apr 2023 16:22:46 -0500 Subject: [PATCH 37/45] Fixed case --- .../tests/integration/containers/DebeziumDB2DbContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 91d85bab1ac01..53ecf80c75fbe 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -47,7 +47,7 @@ public class DebeziumDB2DbContainer extends ChaosContainer Date: Wed, 5 Apr 2023 05:38:48 -0500 Subject: [PATCH 38/45] Added more properties to DB2 source to try to fix connector restarts --- .../resources/debezium-db2-source-config.yaml | 2 +- .../containers/DebeziumDB2DbContainer.java | 2 +- .../debezium/DebeziumDB2DbSourceTester.java | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml index 85923f2c988ff..59842cd851b78 100644 --- a/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml +++ b/pulsar-io/debezium/db2/src/main/resources/debezium-db2-source-config.yaml @@ -20,7 +20,7 @@ ## config for db2, docker image: ibmcom/db2, which was moved to icr.io/db2_community/db2 ## As per note at https://hub.docker.com/r/ibmcom/db2 ## docs are being moved to: https://www.ibm.com/docs/en/db2/11.5?topic=deployments-db2-community-edition-docker -## For testing, you can use this image that was configured with the required libraries: devingbost/debezium-db2:0.0.1 +## For testing, you can use this image that was configured with the required libraries: apachepulsar/debezium-db2-test-image:latest ## docker run -itd --name mydb2 --privileged=true -p 50000:50000 -e LICENSE=accept -e DB2INST1_PASSWORD=admin -e DBNAME=testdb -v :/database ibmcom/db2 tenant: "public" diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java index 53ecf80c75fbe..d31a3e081c729 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumDB2DbContainer.java @@ -114,7 +114,7 @@ public String selectStatement(){ public Connection getConnection() throws SQLException, ClassNotFoundException { log.info(String.format("JDBC Connection URL is: %s", this.getJdbcUrl())); if(connection == null){ - Class.forName("com.ibm.db2.jcc.DB2Driver"); + Class.forName(this.getDriverClassName()); var conn = DriverManager.getConnection(this.getJdbcUrl(), getUsername(), getPassword()); this.connection = conn; } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 1478f450ca040..dd4cb198ea749 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -51,17 +51,29 @@ public DebeziumDB2DbSourceTester(PulsarCluster cluster) { this.numEntriesExpectAfterStart = 0; pulsarServiceUrl = "pulsar://pulsar-proxy:" + PulsarContainer.BROKER_PORT; - sourceConfig.put("connector.class", "io.debezium.connector.db2.Db2Connector"); + //sourceConfig.put("connector.class", "io.debezium.connector.db2.Db2Connector"); sourceConfig.put("database.hostname", DebeziumDB2DbContainer.NAME); sourceConfig.put("database.port", "50000"); sourceConfig.put("database.user", "db2inst1"); sourceConfig.put("database.password", "admin"); sourceConfig.put("database.dbname", "mydb2"); + sourceConfig.put("database.server.name", "db2inst1"); sourceConfig.put("topic.prefix", "stores"); sourceConfig.put("table.include.list", "DB2INST1.STORES"); - + sourceConfig.put("database.history","org.apache.pulsar.io.debezium.PulsarDatabaseHistory"); + sourceConfig.put("database.history.pulsar.topic", "debezium-db2-source-history-topic"); sourceConfig.put("database.history.pulsar.service.url", pulsarServiceUrl); sourceConfig.put("topic.namespace", "debezium/db2"); + sourceConfig.put("key.converter", "org.apache.kafka.connect.json.JsonConverter"); + sourceConfig.put("value.converter", "org.apache.kafka.connect.json.JsonConverter"); + sourceConfig.put("typeClassName", "org.apache.pulsar.common.schema.KeyValue"); + sourceConfig.put("task.class", "io.debezium.connector.db2.Db2ConnectorTask"); + + sourceConfig.put("offset.storage.topic", "offset-topic"); + sourceConfig.put("snapshot.mode", "initial"); + sourceConfig.put("database.tcpKeepAlive", "true"); + sourceConfig.put("decimal.handling.mode", "double"); + } @Override From faf2eabcac77a53b2470482d72deacfb66411fab Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 5 Apr 2023 07:28:17 -0500 Subject: [PATCH 39/45] Removing additional insert to avoid duplicate write from failing test --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index dd4cb198ea749..7db34ec7b0464 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -103,14 +103,14 @@ public void prepareSource() { log.info(String.format("Running : %s", debeziumDB2DbContainer.addCdcTableStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) .execute(); - +/* log.info(String.format("Running : %s", debeziumDB2DbContainer.insertStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.insertStatement()) .execute(); log.info(String.format("Running : %s", debeziumDB2DbContainer.selectStatement())); var result = debeziumDB2DbContainer.getStoreNameResult(); - log.info("debeziumDB2DbContainer.getStoreNameResult() returned: %s", result); + log.info("debeziumDB2DbContainer.getStoreNameResult() returned: %s", result);*/ } @Override From 16170a89147971167dc3e19527e55b3e6d3aff92 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 5 Apr 2023 10:05:16 -0500 Subject: [PATCH 40/45] Incremented number of expected messages. --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 4 ++-- .../io/sources/debezium/PulsarDebeziumSourcesTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 7db34ec7b0464..dd4cb198ea749 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -103,14 +103,14 @@ public void prepareSource() { log.info(String.format("Running : %s", debeziumDB2DbContainer.addCdcTableStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.addCdcTableStatement()) .execute(); -/* + log.info(String.format("Running : %s", debeziumDB2DbContainer.insertStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.insertStatement()) .execute(); log.info(String.format("Running : %s", debeziumDB2DbContainer.selectStatement())); var result = debeziumDB2DbContainer.getStoreNameResult(); - log.info("debeziumDB2DbContainer.getStoreNameResult() returned: %s", result);*/ + log.info("debeziumDB2DbContainer.getStoreNameResult() returned: %s", result); } @Override diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java index b31cff5503add..0475f86055692 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java @@ -255,7 +255,7 @@ private void testDebeziumDb2Connect(String converterClassName, boolean jsonWithE final String consumeTopicName = "debezium/db2/mydb2.DB2INST1.STORES"; final String sourceName = "test-source-debezium-db2-" + functionRuntimeType + "-" + randomName(8); - final int numMessages = 1; + final int numMessages = 2; @Cleanup PulsarClient client = PulsarClient.builder() From 972c3493e86cfc2a435a58ddf724aed8ce55c60c Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 5 Apr 2023 14:06:53 -0500 Subject: [PATCH 41/45] Added db2 bind command --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index dd4cb198ea749..8776414e3a600 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -92,6 +92,13 @@ public void prepareSource() { String stdout = lsResult.getStdout(); log.info(String.format("Output of running cdcsetup.sh is: %s", stdout)); int exitCode = lsResult.getExitCode(); + + log.info("Running schema bind next"); + var lsResult2 = debeziumDB2DbContainer.execInContainer("/bin/bash", "-c", "/opt/ibm/db2/V11.5/bin/db2", "bind", "db2schema.bnd", "blocking", "all", "grant", "public", "sqlerror", "continue"); + String stdout2 = lsResult2.getStdout(); + log.info(String.format("Output of running schema bind is: %s", stdout2)); + int exitCode2 = lsResult2.getExitCode(); + log.info(String.format("Running : %s", debeziumDB2DbContainer.enableCdcStatement())); debeziumDB2DbContainer.getPreparedStatement(debeziumDB2DbContainer.enableCdcStatement()) .execute(); From b437fd504e81af0181130e8c5189ac71cc3270c7 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 5 Apr 2023 14:08:40 -0500 Subject: [PATCH 42/45] Fixed bind command --- .../io/sources/debezium/DebeziumDB2DbSourceTester.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 8776414e3a600..3ac8c46506703 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -94,7 +94,7 @@ public void prepareSource() { int exitCode = lsResult.getExitCode(); log.info("Running schema bind next"); - var lsResult2 = debeziumDB2DbContainer.execInContainer("/bin/bash", "-c", "/opt/ibm/db2/V11.5/bin/db2", "bind", "db2schema.bnd", "blocking", "all", "grant", "public", "sqlerror", "continue"); + var lsResult2 = debeziumDB2DbContainer.execInContainer("/bin/bash", "-c", "/opt/ibm/db2/V11.5/bin/db2 bind db2schema.bnd blocking all grant public sqlerror continue"); String stdout2 = lsResult2.getStdout(); log.info(String.format("Output of running schema bind is: %s", stdout2)); int exitCode2 = lsResult2.getExitCode(); From 2dbd84a700daabf150f0ca01c5baf4253f462758 Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Wed, 5 Apr 2023 16:09:26 -0500 Subject: [PATCH 43/45] Added more logging --- .../integration/io/sources/SourceTester.java | 2 +- .../debezium/DebeziumDB2DbSourceTester.java | 16 ++++++++++------ .../debezium/PulsarIODebeziumSourceRunner.java | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/SourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/SourceTester.java index 37e241a873b6c..7f2d7a5916a30 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/SourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/SourceTester.java @@ -129,7 +129,7 @@ public void validateSourceResultJson(Consumer> consumer consumer.acknowledge(msg); msg = consumer.receive(1, TimeUnit.SECONDS); } - + log.info(String.format("Running Assert.assertEquals(recordsNumber, number) for (%d, %d)", recordsNumber, number)); Assert.assertEquals(recordsNumber, number); log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java index 3ac8c46506703..047abb18dc2e1 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumDB2DbSourceTester.java @@ -118,24 +118,28 @@ public void prepareSource() { log.info(String.format("Running : %s", debeziumDB2DbContainer.selectStatement())); var result = debeziumDB2DbContainer.getStoreNameResult(); log.info("debeziumDB2DbContainer.getStoreNameResult() returned: %s", result); + // Could add an assertion here that result == "store_name" } @Override public void prepareInsertEvent() throws Exception { - debeziumDB2DbContainer.getPreparedStatement("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore2', 2, '22222')"); - debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("INSERT INTO DB2INST1.STORES(store_name, state_id, zip_code) VALUES ('mystore2', 2, '22222')") + .execute(); + //debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override public void prepareDeleteEvent() throws Exception { - debeziumDB2DbContainer.getPreparedStatement("DELETE FROM DB2INST1.STORES WHERE store_name='mystore2'"); - debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("DELETE FROM DB2INST1.STORES WHERE store_name='mystore2'") + .execute(); + //debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override public void prepareUpdateEvent() throws Exception { - debeziumDB2DbContainer.getPreparedStatement("UPDATE DB2INST1.STORES SET zip_code='33333' WHERE store_name='mystore2'"); - debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); + debeziumDB2DbContainer.getPreparedStatement("UPDATE DB2INST1.STORES SET zip_code='33333' WHERE store_name='mystore2'") + .execute(); + //debeziumDB2DbContainer.getPreparedStatement("SELECT * FROM DB2INST1.STORES WHERE store_name='mystore2'"); } @Override diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java index 762dd34e17c91..d34e70df44618 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java @@ -94,8 +94,9 @@ public void testSource(SourceTester sourceTester .subscriptionType(SubscriptionType.Exclusive) .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); - log.info("[debezium mysql test] create consumer finish. converterName: {}", converterClassName); + log.info("[debezium test] create consumer finish. converterName: {}", converterClassName); + log.info("sourceTester.getNumEntriesExpectAfterStart() is {}", sourceTester.getNumEntriesExpectAfterStart()); // validate the source result sourceTester.validateSourceResult(consumer, sourceTester.getNumEntriesExpectAfterStart(), null, converterClassName); From ce6e3953e45f063aab441699595856c117f7b98a Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 14 Apr 2023 10:27:25 -0500 Subject: [PATCH 44/45] Added retention --- .../io/sources/debezium/PulsarDebeziumSourcesTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java index 0475f86055692..bb422f5c3c986 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java @@ -269,6 +269,8 @@ private void testDebeziumDb2Connect(String converterClassName, boolean jsonWithE admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); + admin.namespaces().setRetention("debezium/db2", new RetentionPolicies(0, 1)); + admin.namespaces().setRetention("public/default", new RetentionPolicies(0, 1)); @Cleanup DebeziumDB2DbSourceTester sourceTester = new DebeziumDB2DbSourceTester(pulsarCluster); sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); From 45f2fde9799e57515ba90faf7f4b9e185024ae1a Mon Sep 17 00:00:00 2001 From: Devin Bost Date: Fri, 14 Apr 2023 12:40:02 -0500 Subject: [PATCH 45/45] Fixed retention limit --- .../io/sources/debezium/PulsarDebeziumSourcesTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java index bb422f5c3c986..5724b14a2db4d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java @@ -269,8 +269,8 @@ private void testDebeziumDb2Connect(String converterClassName, boolean jsonWithE admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); - admin.namespaces().setRetention("debezium/db2", new RetentionPolicies(0, 1)); - admin.namespaces().setRetention("public/default", new RetentionPolicies(0, 1)); + admin.namespaces().setRetention("debezium/db2", new RetentionPolicies(-1, 1)); + admin.namespaces().setRetention("public/default", new RetentionPolicies(-1, 1)); @Cleanup DebeziumDB2DbSourceTester sourceTester = new DebeziumDB2DbSourceTester(pulsarCluster); sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope);