-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update sarama library to support last kafka versions (#151)
- Bump version of sarama client to support kafka up to v3.0 - Add dummy consumer and producer for integration test - Add consumer offset integration test
- Loading branch information
1 parent
cca5a80
commit 9c58ed0
Showing
18 changed files
with
1,352 additions
and
66 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| FROM maven | ||
| COPY src src | ||
| COPY pom.xml pom.xml | ||
|
|
||
| RUN mvn clean compile package | ||
|
|
||
| WORKDIR / | ||
| ENTRYPOINT ["java","-Dcom.sun.management.jmxremote","-Djava.rmi.server.hostname=localhost","-Dcom.sun.management.jmxremote.port=1088","-Dcom.sun.management.jmxremote.rmi.port=1088","-Dcom.sun.management.jmxremote.local.only=false","-Dcom.sun.management.jmxremote.authenticate=false","-Dcom.sun.management.jmxremote.ssl=false","-jar","./target/kafka-dummy-1.0-jar-with-dependencies.jar"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <project xmlns="http://maven.apache.org/POM/4.0.0" | ||
| xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
| xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
| <modelVersion>4.0.0</modelVersion> | ||
| <groupId>1</groupId> | ||
| <artifactId>kafka-dummy</artifactId> | ||
| <version>1.0</version> | ||
| <dependencies> | ||
| <dependency> | ||
| <groupId>org.apache.kafka</groupId> | ||
| <artifactId>kafka-clients</artifactId> | ||
| <version>0.10.0.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.kafka</groupId> | ||
| <artifactId>kafka-streams</artifactId> | ||
| <version>0.10.0.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.flink</groupId> | ||
| <artifactId>flink-connector-kafka-0.10_2.11</artifactId> | ||
| <version>1.4.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.flink</groupId> | ||
| <artifactId>flink-streaming-java_2.11</artifactId> | ||
| <version>1.4.0</version> | ||
| </dependency> | ||
| </dependencies> | ||
| <build> | ||
| <plugins> | ||
| <plugin> | ||
| <groupId>org.apache.maven.plugins</groupId> | ||
| <artifactId>maven-assembly-plugin</artifactId> | ||
| <executions> | ||
| <execution> | ||
| <phase>package</phase> | ||
| <goals> | ||
| <goal>single</goal> | ||
| </goals> | ||
| <configuration> | ||
| <archive> | ||
| <manifest> | ||
| <mainClass> | ||
| kafkaDummy.Factory | ||
| </mainClass> | ||
| </manifest> | ||
| </archive> | ||
| <descriptorRefs> | ||
| <descriptorRef>jar-with-dependencies</descriptorRef> | ||
| </descriptorRefs> | ||
| </configuration> | ||
| </execution> | ||
| </executions> | ||
| </plugin> | ||
| <plugin> | ||
| <groupId>org.apache.maven.plugins</groupId> | ||
| <artifactId>maven-compiler-plugin</artifactId> | ||
| <configuration> | ||
| <source>1.8</source> | ||
| <target>1.8</target> | ||
| </configuration> | ||
| </plugin> | ||
| </plugins> | ||
| </build> | ||
| </project> |
13 changes: 13 additions & 0 deletions
13
tests/integration/consumer-producer/src/main/java/Factory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package kafkaDummy; | ||
|
|
||
| public class Factory { | ||
| public static void main(String[] args) { | ||
| if (args[0].equals("producer")) { | ||
| SimpleProducer.main(args); | ||
| } else if (args[0].equals("consumer")) { | ||
| SimpleConsumer.main(args); | ||
| } else { | ||
| System.out.println("First argument should be consumer/producer" + args[0]); | ||
| } | ||
| } | ||
| } |
43 changes: 43 additions & 0 deletions
43
tests/integration/consumer-producer/src/main/java/SimpleConsumer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package kafkaDummy; | ||
|
|
||
| import java.util.Properties; | ||
| import java.util.Arrays; | ||
| import org.apache.kafka.clients.consumer.KafkaConsumer; | ||
| import org.apache.kafka.clients.consumer.ConsumerRecords; | ||
| import org.apache.kafka.clients.consumer.ConsumerRecord; | ||
|
|
||
| public class SimpleConsumer { | ||
| public static void main(String[] args) { | ||
| if(args.length < 4){ | ||
| System.out.println("Usage: consumer <bootstrapBroker> <topic> <groupname>"); | ||
| return; | ||
| } | ||
|
|
||
| String bootstrapBroker = args[1].toString(); | ||
| String topic = args[2].toString(); | ||
| String group = args[3].toString(); | ||
|
|
||
| Properties props = new Properties(); | ||
| props.put("bootstrap.servers", bootstrapBroker); | ||
| props.put("group.id", group); | ||
| props.put("enable.auto.commit", "true"); | ||
| props.put("auto.commit.interval.ms", "1000"); | ||
| props.put("session.timeout.ms", "30000"); | ||
| props.put("key.deserializer", | ||
| "org.apache.kafka.common.serialization.StringDeserializer"); | ||
| props.put("value.deserializer", | ||
| "org.apache.kafka.common.serialization.StringDeserializer"); | ||
|
|
||
| KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(props); | ||
|
|
||
| consumer.subscribe(Arrays.asList(topic)); | ||
| System.out.println("Subscribed to topic " + topic); | ||
|
|
||
| while (true) { | ||
| ConsumerRecords<String, String> records = consumer.poll(10); | ||
| for (ConsumerRecord<String, String> record : records) { | ||
| System.out.printf("offset = %d, key = %s, value = %s\n", record.offset(), record.key(), record.value()); | ||
| } | ||
| } | ||
| } | ||
| } |
57 changes: 57 additions & 0 deletions
57
tests/integration/consumer-producer/src/main/java/SimpleProducer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package kafkaDummy; | ||
|
|
||
| import java.util.Properties; | ||
| import org.apache.kafka.clients.producer.Producer; | ||
| import org.apache.kafka.clients.producer.KafkaProducer; | ||
| import org.apache.kafka.clients.producer.ProducerRecord; | ||
|
|
||
| public class SimpleProducer { | ||
| public static void main(String[] args) { | ||
| if(args.length < 3){ | ||
| System.out.println("Usage: producer <bootstrapBroker> <topic>"); | ||
| return; | ||
| } | ||
|
|
||
| String bootstrapBroker = args[1].toString(); | ||
| String topicName = args[2].toString(); | ||
|
|
||
| Properties props = new Properties(); | ||
| props.put("bootstrap.servers", bootstrapBroker); | ||
| props.put("acks", "all"); | ||
| props.put("retries", 0); | ||
| props.put("batch.size", 16384); | ||
| props.put("linger.ms", 1); | ||
| props.put("buffer.memory", 33554432); | ||
| props.put("key.serializer", | ||
| "org.apache.kafka.common.serialization.StringSerializer"); | ||
| props.put("value.serializer", | ||
| "org.apache.kafka.common.serialization.StringSerializer"); | ||
|
|
||
| Producer<String, String> producer = new KafkaProducer | ||
| <String, String>(props); | ||
|
|
||
| int msg = 0; | ||
| while(true) { | ||
| producer.send(new ProducerRecord<String, String>(topicName, | ||
| Integer.toString(msg), Integer.toString(msg))); | ||
| System.out.println("Message sent successfully"); | ||
| msg++; | ||
| wait(2000); | ||
| } | ||
|
|
||
| // never reached | ||
| // producer.close(); | ||
| } | ||
|
|
||
| public static void wait(int ms) | ||
| { | ||
| try | ||
| { | ||
| Thread.sleep(ms); | ||
| } | ||
| catch(InterruptedException ex) | ||
| { | ||
| Thread.currentThread().interrupt(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.