Skip to content

Commit

Permalink
upgrade Flink to 1.19 (#55)
Browse files Browse the repository at this point in the history
* make windows top N test result deterministic

* upgrade Flink to 1.19
  • Loading branch information
davetroiano authored Jun 28, 2024
1 parent 7f78c3b commit d7c1a7e
Show file tree
Hide file tree
Showing 26 changed files with 549 additions and 167 deletions.
18 changes: 9 additions & 9 deletions aggregating-count/flinksql/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ dependencies {
testImplementation 'junit:junit:4.13.2'
testImplementation 'org.testcontainers:testcontainers:1.19.3'
testImplementation 'org.testcontainers:kafka:1.19.3'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.0.2-1.18'
testImplementation 'org.apache.flink:flink-connector-base:1.18.0'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.18.0'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0:tests'
testImplementation 'org.apache.flink:flink-statebackend-rocksdb:1.18.0'
testImplementation 'commons-codec:commons-codec:1.17.0'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.2.0-1.19'
testImplementation 'org.apache.flink:flink-connector-base:1.19.1'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.19.1'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1:tests'
}
18 changes: 9 additions & 9 deletions aggregating-minmax/flinksql/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ dependencies {
testImplementation 'junit:junit:4.13.2'
testImplementation 'org.testcontainers:testcontainers:1.19.3'
testImplementation 'org.testcontainers:kafka:1.19.3'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.0.2-1.18'
testImplementation 'org.apache.flink:flink-connector-base:1.18.0'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.18.0'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0:tests'
testImplementation 'org.apache.flink:flink-statebackend-rocksdb:1.18.0'
testImplementation 'commons-codec:commons-codec:1.17.0'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.2.0-1.19'
testImplementation 'org.apache.flink:flink-connector-base:1.19.1'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.19.1'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1:tests'
}
18 changes: 9 additions & 9 deletions common/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,15 @@ dependencies {
testImplementation 'org.hamcrest:hamcrest:2.2'
testImplementation 'org.testcontainers:testcontainers:1.19.3'
testImplementation 'org.testcontainers:kafka:1.19.3'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.0.2-1.18'
testImplementation 'org.apache.flink:flink-connector-base:1.18.0'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.18.0'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0:tests'
testImplementation 'org.apache.flink:flink-statebackend-rocksdb:1.18.0'
testImplementation 'commons-codec:commons-codec:1.17.0'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.2.0-1.19'
testImplementation 'org.apache.flink:flink-connector-base:1.19.1'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.19.1'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1:tests'
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.2'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@

import com.google.common.io.Resources;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.contrib.streaming.state.EmbeddedRocksDBStateBackend;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.RestartStrategyOptions;
import org.apache.flink.configuration.StateBackendOptions;
import org.apache.flink.runtime.client.JobCancellationException;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.EnvironmentSettings;
Expand Down Expand Up @@ -42,10 +43,11 @@ public class AbstractFlinkKafkaTest {
@BeforeClass
public static void setup() {
// create Flink table environment that test subclasses will use to execute SQL statements
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
Configuration config = new Configuration();
config.set(RestartStrategyOptions.RESTART_STRATEGY, "none");
config.set(StateBackendOptions.STATE_BACKEND, "hashmap");
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
env.setParallelism(1);
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env.setStateBackend(new EmbeddedRocksDBStateBackend());
streamTableEnv = StreamTableEnvironment.create(env, EnvironmentSettings.newInstance().inStreamingMode().build());


Expand Down
18 changes: 9 additions & 9 deletions cumulating-windows/flinksql/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ dependencies {
testImplementation 'junit:junit:4.13.2'
testImplementation 'org.testcontainers:testcontainers:1.19.3'
testImplementation 'org.testcontainers:kafka:1.19.3'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.0.2-1.18'
testImplementation 'org.apache.flink:flink-connector-base:1.18.0'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.18.0'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0:tests'
testImplementation 'org.apache.flink:flink-statebackend-rocksdb:1.18.0'
testImplementation 'commons-codec:commons-codec:1.17.0'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.2.0-1.19'
testImplementation 'org.apache.flink:flink-connector-base:1.19.1'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.19.1'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1:tests'
}
18 changes: 9 additions & 9 deletions deduplication-windowed/flinksql/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ dependencies {
testImplementation 'junit:junit:4.13.2'
testImplementation 'org.testcontainers:testcontainers:1.19.3'
testImplementation 'org.testcontainers:kafka:1.19.3'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.0.2-1.18'
testImplementation 'org.apache.flink:flink-connector-base:1.18.0'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.18.0'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0:tests'
testImplementation 'org.apache.flink:flink-statebackend-rocksdb:1.18.0'
testImplementation 'commons-codec:commons-codec:1.17.0'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.2.0-1.19'
testImplementation 'org.apache.flink:flink-connector-base:1.19.1'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.19.1'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1:tests'
}
18 changes: 9 additions & 9 deletions deduplication/flinksql/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ dependencies {
testImplementation 'junit:junit:4.13.2'
testImplementation 'org.testcontainers:testcontainers:1.19.3'
testImplementation 'org.testcontainers:kafka:1.19.3'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.0.2-1.18'
testImplementation 'org.apache.flink:flink-connector-base:1.18.0'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils:1.18.0'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.18.0'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.18.0:tests'
testImplementation 'org.apache.flink:flink-statebackend-rocksdb:1.18.0'
testImplementation 'commons-codec:commons-codec:1.17.0'
testImplementation 'org.apache.flink:flink-sql-connector-kafka:3.2.0-1.19'
testImplementation 'org.apache.flink:flink-connector-base:1.19.1'
testImplementation 'org.apache.flink:flink-sql-avro-confluent-registry:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils:1.19.1'
testImplementation 'org.apache.flink:flink-test-utils-junit:1.19.1'
testImplementation 'org.apache.flink:flink-table-api-java-bridge:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1'
testImplementation 'org.apache.flink:flink-table-planner_2.12:1.19.1:tests'
}
6 changes: 3 additions & 3 deletions docker/docker-compose-flinksql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ services:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker:9092
flink-sql-client:
image: cnfldemos/flink-sql-client-kafka:1.16.0-scala_2.12-java11
image: cnfldemos/flink-sql-client-kafka:1.19.1-scala_2.12-java17
hostname: flink-sql-client
container_name: flink-sql-client
depends_on:
Expand All @@ -44,7 +44,7 @@ services:
volumes:
- ./settings/:/settings
flink-jobmanager:
image: cnfldemos/flink-kafka:1.16.0-scala_2.12-java11
image: cnfldemos/flink-kafka:1.19.1-scala_2.12-java17
hostname: flink-jobmanager
container_name: flink-jobmanager
ports:
Expand All @@ -56,7 +56,7 @@ services:
jobmanager.rpc.address: flink-jobmanager
rest.bind-port: 9081
flink-taskmanager:
image: cnfldemos/flink-kafka:1.16.0-scala_2.12-java11
image: cnfldemos/flink-kafka:1.19.1-scala_2.12-java17
hostname: flink-taskmanager
container_name: flink-taskmanager
depends_on:
Expand Down
9 changes: 9 additions & 0 deletions docker/flink/Dockerfile.flink-kafka
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM flink:1.19.1-scala_2.12-java11

# Download connector libraries
RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/3.2.0-1.19/flink-sql-connector-kafka-3.2.0-1.19.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.19.1/flink-json-1.19.1.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.19.1/flink-sql-avro-confluent-registry-1.19.1.jar;

# Copy configuration
COPY ./conf/* /opt/flink/conf/
22 changes: 22 additions & 0 deletions docker/flink/Dockerfile.flink-sql-client-kafka
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
FROM flink:1.19.1-scala_2.12-java11

# Create CLI lib folder
#COPY ./bin/* /opt/sql-client/
RUN mkdir -p /opt/sql-client/lib

# Download connector libraries
RUN wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/3.2.0-1.19/flink-sql-connector-kafka-3.2.0-1.19.jar; \
wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.19.1/flink-json-1.19.1.jar; \
wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.19.1/flink-sql-avro-confluent-registry-1.19.1.jar;

# Also copy to Flink lib so that, e.g., other catalog types can be used if desired
RUN cp /opt/sql-client/lib/* /opt/flink/lib/

# Copy configuration
COPY ./conf/* /opt/flink/conf/

WORKDIR /opt/sql-client
ENV SQL_CLIENT_HOME=/opt/sql-client

COPY ./docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]
15 changes: 15 additions & 0 deletions docker/flink/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
FLINK_VERSION ?= 1.19.1-scala_2.12-java17
BUILD_PLATFORM ?= linux/amd64
PUSH_PREFIX ?= cnfldemos

build-flink-kafka:
@docker buildx build --load --platform $(BUILD_PLATFORM) -t flink-kafka:$(FLINK_VERSION) -f Dockerfile.flink-kafka .

build-flink-sql-client-kafka:
@docker buildx build --load --platform $(BUILD_PLATFORM) -t flink-sql-client-kafka:$(FLINK_VERSION) -f Dockerfile.flink-sql-client-kafka .

push-flink-kafka:
@docker buildx build --push --platform linux/amd64,linux/arm64 -t $(PUSH_PREFIX)/flink-kafka:$(FLINK_VERSION) -f Dockerfile.flink-kafka .

push-flink-sql-client-kafka:
@docker buildx build --push --platform linux/amd64,linux/arm64 -t $(PUSH_PREFIX)/flink-sql-client-kafka:$(FLINK_VERSION) -f Dockerfile.flink-sql-client-kafka .
Loading

0 comments on commit d7c1a7e

Please sign in to comment.