diff --git a/.circleci/config.yml b/.circleci/config.yml
index 596ddb808..33ac7bfbe 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -37,7 +37,7 @@ jobs:
command: sudo apt-get update && sudo apt-get -y install libnuma1
- run:
name: Maven Test
- command: mvn test
+ command: mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
- store_artifacts:
path: test.log
@@ -58,7 +58,11 @@ workflows:
mysql: "5.7"
requires: [ "build" ]
- test:
- name: "test-8.0"
- mysql: "8.0"
+ name: "test-8.4"
+ mysql: "8.4"
+ requires: [ "build" ]
+ - test:
+ name: "test-mariadb"
+ mysql: "mariadb"
requires: [ "build" ]
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 000000000..96ac27d83
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,33 @@
+name: CI to Zendesk Dockerhub
+
+on:
+ push:
+ tags:
+ - v*.*.*
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: zendesk/checkout@v2
+ with:
+ fetch-depth: '1'
+ - name: Set up QEMU
+ run: |-
+ QEMU_IMAGE=tonistiigi/binfmt:latest
+ docker pull $QEMU_IMAGE
+ docker image inspect $QEMU_IMAGE
+ docker run --rm --privileged $QEMU_IMAGE --install all
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: zendesk/setup-buildx-action@v1.6.0
+ with:
+ install: true
+ - name: Docker Build and push
+ run: |-
+ set -eu -o pipefail
+ echo ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | docker login --username=${{ secrets.DOCKER_HUB_USERNAME }} --password-stdin
+ set -x
+ current_tag=zendesk/maxwell:"${GITHUB_REF##refs/tags/}"
+ latest_tag=zendesk/maxwell:latest
+ docker buildx build --platform=linux/arm64,linux/amd64 --file=Dockerfile --push --tag="$current_tag" --tag="$latest_tag" .
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8a460c339..501958e2d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,27 +1,373 @@
# Maxwell changelog
-### [v1.33.0](https://github.com/zendesk/maxwell/releases/tag/v1.33.0): "tradegy of birds and windows"
+### [v1.42.2](https://github.com/zendesk/maxwell/releases/tag/v1.42.2)
+- update jdk on docker image
+- support rabbitmq + SSL
+- small fixes for latest maria
+- get tests running under mysql 8.4
+- remove Kinesis internal TTL, see #2147 for details
+- support bootstrapping from a replica that doesn't contain the maxwell database
+
+
+
+_Released 2025-01-09_
+
+### [v1.42.1](https://github.com/zendesk/maxwell/releases/tag/v1.42.1)
+
+- bugfix for 1.42.0, mysql 8.0.x and "SHOW BINARY LOG STATUS"
+
+
+
+_Released 2024-12-21_
+
+### [v1.42.0](https://github.com/zendesk/maxwell/releases/tag/v1.42.0)
+
+- initial support for mysql 8.4
+- support partitioning for sns and sqs
+- bugfix for maria
+
+
+
+_Released 2024-12-17_
+
+### [v1.41.2](https://github.com/zendesk/maxwell/releases/tag/v1.41.2)
+
+- Owen Derby is the Nick Clarke of Maxwell parser bugs
+
+
+
+_Released 2024-06-05_
+
+### [v1.41.1](https://github.com/zendesk/maxwell/releases/tag/v1.41.1)
+
+- fix 2 parser issues, one mariadb and one "tablespace" specific
+- upgrade lz4 dep for security
+
+
+
+_Released 2024-03-24_
+
+### [v1.41.0](https://github.com/zendesk/maxwell/releases/tag/v1.41.0)
+
+- javascript filters are now passed a second, optional dicionary
+ argument which persists between filter invocations.
+
+
+
+_Released 2023-11-30_
+
+### [v1.40.6](https://github.com/zendesk/maxwell/releases/tag/v1.40.6)
+
+- fix 2 parser bugs
+- upgrade jackson for security
+
+
+
+_Released 2023-11-04_
+
+### [v1.40.5](https://github.com/zendesk/maxwell/releases/tag/v1.40.5)
+
+- Fix a bug introduced in v1.40.2 in the kafka producer.
+
+
+
+_Released 2023-09-09_
+
+### [v1.40.4](https://github.com/zendesk/maxwell/releases/tag/v1.40.4)
+
+- add support for mariadb's DROP COLUMN IF EXISTS
+
+
+
+_Released 2023-09-01_
+
+### [v1.40.3](https://github.com/zendesk/maxwell/releases/tag/v1.40.3)
+
+- bugfix for "rename tables"
+- bugfix for temporary tables that rollback inside transactions
+- sns+localstack support
+
+
+
+_Released 2023-08-27_
+
+### [v1.40.2](https://github.com/zendesk/maxwell/releases/tag/v1.40.2)
+
+- fix dumb bug in last release
+
+
+
+_Released 2023-06-11_
+
+### [v1.40.0](https://github.com/zendesk/maxwell/releases/tag/v1.40.0)
+
+- add kafka 3.4.0
+- kafka 2.7.0 is now the default kafka library
+- add custom health-check factory jar thing
+
+
+
+_Released 2023-04-02_
+
+### [v1.39.6](https://github.com/zendesk/maxwell/releases/tag/v1.39.6)
+
+- Bugfix issue where SQL query would go missing (#1973)
+- Various parser bugfixes (#1970, #1982, #1987)
+- Fix issue with renaming a primary key column (#1977)
+
+
+
+_Released 2023-03-11_
+
+### [v1.39.5](https://github.com/zendesk/maxwell/releases/tag/v1.39.5)
+
+- a few parser fixes
+
+
+
+_Released 2023-02-08_
+
+### [v1.39.4](https://github.com/zendesk/maxwell/releases/tag/v1.39.4)
+
+- Fix bugs with older versions of mariadb (<10.4)
+
+
+
+_Released 2022-12-07_
+
+### [v1.39.3](https://github.com/zendesk/maxwell/releases/tag/v1.39.3)
+
+- some bugfixes for 1.39.2 and google pubsub
+- couple of security upgrades, including in the docker image
+
+
+
+_Released 2022-12-04_
+
+### [v1.39.2](https://github.com/zendesk/maxwell/releases/tag/v1.39.2)
+
+this is a bug-fix release. some upgrades broke maxwell's http interface and there's
+a bunch of SQL parser fixes in here.
+
+
+
+_Released 2022-11-02_
+
+### [v1.39.1](https://github.com/zendesk/maxwell/releases/tag/v1.39.1)
+
+This is a faily major release, including lots of MariaDB support fixes
+and a few months worth of patches.
+
+- GTID support for MariaDB
+- Improved JSON column handling for MariaDB
+- add `--pubsub_message_ordering_key`, thanks Billy Braga
+- add `--pubsub_emulator`, thanks Billy Braga
+- add `--ignore_missing_schema` for otherwise untenable schema situations.
+- handle TABLESPACE related DDL
+
+
+
+_Released 2022-11-02_
+
+### [v1.38.0](https://github.com/zendesk/maxwell/releases/tag/v1.38.0)
+
+- Maxwell gets the ability to talk to bigtable! I have no idea how well it'll work. I hope it works for you!
+- upgrade protobuf to fix a rabbitmq issue with booleans, I think.
+- rabbitMQ timeouts on connection
+- other fixes.
+- I can't imagine the security department cares about my naming what with what's going on inside 1019. I guess we'll see.
+
+
+
+
+_Released 2022-07-29_
+
+### [v1.37.7](https://github.com/zendesk/maxwell/releases/tag/v1.37.7)
+
+ - Bump viafoura/metrics-datadog 2.0.0-RC3
+
+
+
+_Released 2022-06-21_
+
+### [v1.37.6](https://github.com/zendesk/maxwell/releases/tag/v1.37.6)
+
+- In non-GTID mode, Verify that the master's server hasn't changed out
+ from underneath us. thanks Tamin Khan
+
+
+
+_Released 2022-05-12_
+
+### [v1.37.5](https://github.com/zendesk/maxwell/releases/tag/v1.37.5)
+
+- Upgrade binlog-replicator. pulls in some minor fixes.
+
+
+
+_Released 2022-04-16_
+
+### [v1.37.4](https://github.com/zendesk/maxwell/releases/tag/v1.37.4)
+
+- configure custom producer via environment
+- sns and sqs producers take output config properly
+
+
+
+_Released 2022-04-08_
+
+### [v1.37.3](https://github.com/zendesk/maxwell/releases/tag/v1.37.3)
+
+- fixes for mariadb
+
+
+
+_Released 2022-03-25_
+
+### [v1.37.2](https://github.com/zendesk/maxwell/releases/tag/v1.37.2)
+
+- configurable binlog event queue size
+
+
+
+_Released 2022-03-14_
+
+### [v1.37.1](https://github.com/zendesk/maxwell/releases/tag/v1.37.1)
+
+ - upgrade mysql-connector-j
+
+_Released 2022-03-07_
+
+### [v1.37.0](https://github.com/zendesk/maxwell/releases/tag/v1.37.0)
+
+- Change max size of RowMap buffer to unblock high-efficiency producers
+
+
+
+_Released 2022-01-26_
+
+### [v1.36.0](https://github.com/zendesk/maxwell/releases/tag/v1.36.0)
+
+- fix bug where the millionth binlog would kinda sort "overflow" and the
+ binlog positions would stop moving.
+- My benefactor here asked that I stopped creating cute release names.
+ The security department, mysteriously.
+
+
+
+_Released 2022-01-23_
+
+### [v1.35.5](https://github.com/zendesk/maxwell/releases/tag/v1.35.5)
+
+- log4j, again and agian.
+
+
+
+_Released 2021-12-29_
+
+### [v1.35.4](https://github.com/zendesk/maxwell/releases/tag/v1.35.4)
+
+- log4j turns 2.17.0, happy birthday
+
+
+
+_Released 2021-12-18_
+
+### [v1.35.3](https://github.com/zendesk/maxwell/releases/tag/v1.35.3)
+
+- log4j vulnerability #2
+
+
+
+_Released 2021-12-15_
+
+### [v1.35.2](https://github.com/zendesk/maxwell/releases/tag/v1.35.2)
+
+- better logging when we can't connect on startup
+
+
+
+_Released 2021-12-12_
+
+### [v1.35.1](https://github.com/zendesk/maxwell/releases/tag/v1.35.1)
+
+- log4j upgrade to upgrade past the giant security hole
+
+
+
+_Released 2021-12-10_
+
+### [v1.35.0](https://github.com/zendesk/maxwell/releases/tag/v1.35.0)
+
+- couple of parser fixes
+- docker builds are now multi-platform
+- replication_reconnection_retries configuration option
+- quote table names in bootstrapper properly
+
+
+
+_Released 2021-11-30_
+
+### [v1.34.1](https://github.com/zendesk/maxwell/releases/tag/v1.34.1)
+
+- support for mysql 8's visible/invisible columns
+- support mariadb's if-exists/if-not-exists for partition management
+- add an index for the http endpoint
+
+
+
+_Released 2021-09-21_
+
+### [v1.34.0](https://github.com/zendesk/maxwell/releases/tag/v1.34.0)
+
+- intern a bunch of objects in our in-memory representation of schema.
+ Saves gobs of memory in cases where one has N copies of the same
+ database. Note that this changes the API of Columns, should any
+ embedded Maxwell application be using that.
+- go up to BIGINT for maxwell's auto-increment ids
+
+
+
+_Released 2021-07-29_
+
+### [v1.33.1](https://github.com/zendesk/maxwell/releases/tag/v1.33.1)
+
+- properties may now be fetched from a javascript blob in the env
+- RowMap provides access to primary keys
+- fix an odd NPE in mariaDB init
+
+
+
+_Released 2021-06-02_
+
+### [v1.33.0](https://github.com/zendesk/maxwell/releases/tag/v1.33.0)
- Add HTTP endpoint for runtime reconfiguration
-### [v1.32.0](https://github.com/zendesk/maxwell/releases/tag/v1.32.0): "cmon cmon no one can see you cry"
+_Released 2021-03-29_
+
+### [v1.32.0](https://github.com/zendesk/maxwell/releases/tag/v1.32.0)
- Amazon SNS producer added, thanks Rober Wittman
- kafka 2.7.0 supported
- stackdriver metrics logging available
-### [v1.31.0](https://github.com/zendesk/maxwell/releases/tag/v1.31.0): "84 tent cabin"
+_Released 2021-03-17_
+
+### [v1.31.0](https://github.com/zendesk/maxwell/releases/tag/v1.31.0)
- Add producer for NATS streaming server
-### [v1.30.0](https://github.com/zendesk/maxwell/releases/tag/v1.30.0): "all of this has happened before"
+_Released 2021-02-11_
+
+### [v1.30.0](https://github.com/zendesk/maxwell/releases/tag/v1.30.0)
- support server-sent heartbeating on the binlog connection via --binlog-heartbeat
- can connect to rabbitmq by URL, supports SSL connections
@@ -31,40 +377,52 @@
- fixes for odd azure mysql connection failures
-### [v1.29.2](https://github.com/zendesk/maxwell/releases/tag/v1.29.2): "i now know the meaning of shame"
+_Released 2021-02-05_
+
+### [v1.29.2](https://github.com/zendesk/maxwell/releases/tag/v1.29.2)
- fix for terrible performance regression in bootstrapping
-### [v1.29.1](https://github.com/zendesk/maxwell/releases/tag/v1.29.1): "depluralize"
+_Released 2021-01-27_
+
+### [v1.29.1](https://github.com/zendesk/maxwell/releases/tag/v1.29.1)
- small bugfix release, fixes binlog event type processing in mysql 8
-### [v1.29.0](https://github.com/zendesk/maxwell/releases/tag/v1.29.0): "i don't know, i don't know, i don't know"
+_Released 2020-12-23_
+
+### [v1.29.0](https://github.com/zendesk/maxwell/releases/tag/v1.29.0)
- High Availability support via jgroups-raft
- rework --help text
-### [v1.28.2](https://github.com/zendesk/maxwell/releases/tag/v1.28.2): "fantasy baseball"
+_Released 2020-12-15_
+
+### [v1.28.2](https://github.com/zendesk/maxwell/releases/tag/v1.28.2)
- fix for encryption parsing error on table creation
- some logging around memory usage in RowMapBuffer
-### [v1.28.1](https://github.com/zendesk/maxwell/releases/tag/v1.28.1): "bootras bootras gallliiiii"
+_Released 2020-12-02_
+
+### [v1.28.1](https://github.com/zendesk/maxwell/releases/tag/v1.28.1)
- fix http server issue in 1.28.0
-### [v1.28.0](https://github.com/zendesk/maxwell/releases/tag/v1.28.0): "stardew mania"
+_Released 2020-11-25_
+
+### [v1.28.0](https://github.com/zendesk/maxwell/releases/tag/v1.28.0)
- schema compaction! with the new --max_schemas option, maxwell will
periodically roll up the `maxwell`.`schemas` table, preventing it from
@@ -75,8 +433,10 @@
- various dependency bumps
-### [v1.27.1](https://github.com/zendesk/maxwell/releases/tag/v1.27.1): "red bag? red bag"
+_Released 2020-11-19_
+
+### [v1.27.1](https://github.com/zendesk/maxwell/releases/tag/v1.27.1)
- redis producer gets sentinal support
- fix a double-reconnect race condition
@@ -85,8 +445,10 @@
- miscellaneous dependency bumps
-### [v1.27.0](https://github.com/zendesk/maxwell/releases/tag/v1.27.0): "running water"
+_Released 2020-08-07_
+
+### [v1.27.0](https://github.com/zendesk/maxwell/releases/tag/v1.27.0)
- better support for empty/null passwords
- allow bootstrap utility to query replication_host
@@ -97,13 +459,18 @@
- fresh and clean documentation
-### [v1.26.4](https://github.com/zendesk/maxwell/releases/tag/v1.26.4): "No songs here"
+
+_Released 2020-06-30_
+
+### [v1.26.4](https://github.com/zendesk/maxwell/releases/tag/v1.26.4)
- support now() function with precision
-### [v1.26.3](https://github.com/zendesk/maxwell/releases/tag/v1.26.3): "the worst song in the goddamn world"
+_Released 2020-06-08_
+
+### [v1.26.3](https://github.com/zendesk/maxwell/releases/tag/v1.26.3)
- use pooled redis connections, fixes corruption when redis was accessed
from multiple threads (bootstrap/producer), thanks @lucastex
@@ -111,8 +478,10 @@ from multiple threads (bootstrap/producer), thanks @lucastex
- fix race condition in binlog reconnect logic
-### [v1.26.2](https://github.com/zendesk/maxwell/releases/tag/v1.26.2): "dave the butcher"
+_Released 2020-05-26_
+
+### [v1.26.2](https://github.com/zendesk/maxwell/releases/tag/v1.26.2)
- bootstraps can be scheduled in the future by setting the `started_at`
column, thanks @lucastex
@@ -120,42 +489,54 @@ from multiple threads (bootstrap/producer), thanks @lucastex
for supporting DEFAULT ENCRYPTION
-### [v1.26.1](https://github.com/zendesk/maxwell/releases/tag/v1.26.1): "maybe we can break your ankle / clean and unsuspiciously"
+_Released 2020-05-18_
+
+### [v1.26.1](https://github.com/zendesk/maxwell/releases/tag/v1.26.1)
- fixes for redis re-connection login, thanks much @lucastex
-### [v1.26.0](https://github.com/zendesk/maxwell/releases/tag/v1.26.0): "tip the waitress, feed her cocaine habit"
+_Released 2020-05-07_
+
+### [v1.26.0](https://github.com/zendesk/maxwell/releases/tag/v1.26.0)
- We now support mysql 8's caching_sha2_password authentication scheme
- support for converting JSON field names to camelCase
-### [v1.25.3](https://github.com/zendesk/maxwell/releases/tag/v1.25.3): "bye, bolinas"
+_Released 2020-05-06_
+
+### [v1.25.3](https://github.com/zendesk/maxwell/releases/tag/v1.25.3)
- fixes memory leak in mysql-binlog-connector
- fixes exceptions that occur when a connection passes wait_timeout
-### [v1.25.2](https://github.com/zendesk/maxwell/releases/tag/v1.25.2): "love potion #9"
+_Released 2020-05-02_
+
+### [v1.25.2](https://github.com/zendesk/maxwell/releases/tag/v1.25.2)
- Fixes for a long standing JSON bug in 8.0.19+
-### [v1.25.1](https://github.com/zendesk/maxwell/releases/tag/v1.25.1): "nowhere to put it"
+_Released 2020-05-01_
+
+### [v1.25.1](https://github.com/zendesk/maxwell/releases/tag/v1.25.1)
- issue #1457, ALTER DATABASE with implicit database name
- maxwell now runs on JDK 11 in docker
- exit with status 2 when we can't find binlog files
-### [v1.25.0](https://github.com/zendesk/maxwell/releases/tag/v1.25.0): "mah mah mah my corona. I'm sorry. I'm sorry."
+_Released 2020-04-22_
+
+### [v1.25.0](https://github.com/zendesk/maxwell/releases/tag/v1.25.0)
- swap un-maintained snaq.db with C3P0.
- support eu datadog metrics
@@ -163,8 +544,10 @@ from multiple threads (bootstrap/producer), thanks @lucastex
heartbeats, postition setting)
-### [v1.24.2](https://github.com/zendesk/maxwell/releases/tag/v1.24.2): "#shelterinstyle"
+_Released 2020-03-29_
+
+### [v1.24.2](https://github.com/zendesk/maxwell/releases/tag/v1.24.2)
- bugfix parsing errors: compressed columns, exchange partitions,
parenthesis-enclosed default values, `drop column foo.t`.
@@ -173,30 +556,38 @@ from multiple threads (bootstrap/producer), thanks @lucastex
- fix redis channel interpolation on RPUSH
-### [v1.24.1](https://github.com/zendesk/maxwell/releases/tag/v1.24.1): "pixies in my head all damn week"
+_Released 2020-03-25_
+
+### [v1.24.1](https://github.com/zendesk/maxwell/releases/tag/v1.24.1)
- allow jdbc_options on secondary connections
- fix a crash in bootstrapping / javascript filters
- fix a regression in message.publish.age metric
-### [v1.24.0](https://github.com/zendesk/maxwell/releases/tag/v1.24.0): "la la la la la la low"
+_Released 2020-01-21_
+
+### [v1.24.0](https://github.com/zendesk/maxwell/releases/tag/v1.24.0)
- add comments field to bootstrapping, thanks Tom Collins
- fix sql bug with #comments style comments
-### [v1.23.5](https://github.com/zendesk/maxwell/releases/tag/v1.23.5): "And I get so stuck in my head - Lost in all the lies, nihilistic backslide"
+_Released 2019-12-14_
+
+### [v1.23.5](https://github.com/zendesk/maxwell/releases/tag/v1.23.5)
- Update bootstrap documentation
- Bump drop wizard metrics to support Java versions 10+
-### [v1.23.4](https://github.com/zendesk/maxwell/releases/tag/v1.23.4): "Try to be kinder to people who bore you, You're probably boring them too."
+_Released 2019-12-12_
+
+### [v1.23.4](https://github.com/zendesk/maxwell/releases/tag/v1.23.4)
- Bump and override dependencies to fix security vulnerabilities.
- Update redis-key config options
@@ -204,8 +595,10 @@ from multiple threads (bootstrap/producer), thanks @lucastex
- list changes
-### [v1.23.3](https://github.com/zendesk/maxwell/releases/tag/v1.23.3): "but that's not the way it feels"
+_Released 2019-12-03_
+
+### [v1.23.3](https://github.com/zendesk/maxwell/releases/tag/v1.23.3)
- pubsubDelayMultiplier may now be 1.0
- allow %{database} and %{topic} interpolation into redis producer
@@ -213,68 +606,86 @@ from multiple threads (bootstrap/producer), thanks @lucastex
- setup default client_id in maxwell-bootstrap util
-### [v1.23.2](https://github.com/zendesk/maxwell/releases/tag/v1.23.2): "you enjoy it every time"
+_Released 2019-11-21_
+
+### [v1.23.2](https://github.com/zendesk/maxwell/releases/tag/v1.23.2)
- upgrade jackson
- stop passing maxwell rows through the JS filter. too dangerous.
-### [v1.23.1](https://github.com/zendesk/maxwell/releases/tag/v1.23.1): "the new barrista"
+_Released 2019-10-18_
+
+### [v1.23.1](https://github.com/zendesk/maxwell/releases/tag/v1.23.1)
- Add option for XADD (redis streams) operation
- Add configuration flag for tuning transaction buffer memory
- sectionalize help text
-### [v1.23.0](https://github.com/zendesk/maxwell/releases/tag/v1.23.0): "When it breaks If it breaks We will see"
+_Released 2019-10-12_
+
+### [v1.23.0](https://github.com/zendesk/maxwell/releases/tag/v1.23.0)
- Added AWS FIFO support
- Add retry and batch settings to pubs producer
- Add support for age SLO metrics
-### [v1.22.6](https://github.com/zendesk/maxwell/releases/tag/v1.22.6): "the things that keep your, like, dresses, like"
+_Released 2019-10-08_
+
+### [v1.22.6](https://github.com/zendesk/maxwell/releases/tag/v1.22.6)
- upgrade mysql-connector-java to 8.0.17
- use a newer docker image as base
- list changes
-### [v1.22.5](https://github.com/zendesk/maxwell/releases/tag/v1.22.5): "all of the names"
+_Released 2019-09-20_
+
+### [v1.22.5](https://github.com/zendesk/maxwell/releases/tag/v1.22.5)
- bugfix for bootstrapping off a split replica that doesn't contain a
"maxwell" database
- Fix a parser issue with db.table.column style column names
-### [v1.22.4](https://github.com/zendesk/maxwell/releases/tag/v1.22.4): "Last Christmans, I gave you my heart"
+_Released 2019-09-06_
+
+### [v1.22.4](https://github.com/zendesk/maxwell/releases/tag/v1.22.4)
- Add row type to fallback message
- Upgrade jackson-databind
-### [v1.22.3](https://github.com/zendesk/maxwell/releases/tag/v1.22.3): "my doubt, my failings"
+_Released 2019-08-23_
+
+### [v1.22.3](https://github.com/zendesk/maxwell/releases/tag/v1.22.3)
- fix issue with google pubsub in 1.22.2
-### [v1.22.2](https://github.com/zendesk/maxwell/releases/tag/v1.22.2): "some girls"
+_Released 2019-06-20_
+
+### [v1.22.2](https://github.com/zendesk/maxwell/releases/tag/v1.22.2)
- fix an issue with bootstrapping-on-replicas
- add --output_primary_keys and --output_primary_key_columns
- fix a very minor memory leak with blacklists
-### [v1.22.1](https://github.com/zendesk/maxwell/releases/tag/v1.22.1): "a snow covered field"
+_Released 2019-06-18_
+
+### [v1.22.1](https://github.com/zendesk/maxwell/releases/tag/v1.22.1)
- fix crash in rabbit-mq producer
- better support for maxwell + azure-mysql
@@ -282,8 +693,10 @@ from multiple threads (bootstrap/producer), thanks @lucastex
- some security upgrades
-### [v1.22.0](https://github.com/zendesk/maxwell/releases/tag/v1.22.0): "through the roof, and underground"
+_Released 2019-05-28_
+
+### [v1.22.0](https://github.com/zendesk/maxwell/releases/tag/v1.22.0)
- Bootstrapping has been reworked and is now available in all setups,
including those in which the maxwell store is split from the replicator.
@@ -291,23 +704,29 @@ including those in which the maxwell store is split from the replicator.
- add .partition_string = to javascript filters
-### [v1.21.1](https://github.com/zendesk/maxwell/releases/tag/v1.21.1): "ohhhhhh oh oh"
+_Released 2019-04-16_
+
+### [v1.21.1](https://github.com/zendesk/maxwell/releases/tag/v1.21.1)
- Upgrade binlog connector. Should fix issues around deserialization
errors.
-### [v1.21.0](https://github.com/zendesk/maxwell/releases/tag/v1.21.0): "through the roof"
+_Released 2019-03-29_
+
+### [v1.21.0](https://github.com/zendesk/maxwell/releases/tag/v1.21.0)
- Bootstrapping output no longer contain binlog positions. Please update
any code that relies on this.
- Fix 3 parser issues.
-### [v1.20.0](https://github.com/zendesk/maxwell/releases/tag/v1.20.0): "and so you learn the only way to go is"
+_Released 2019-03-23_
+
+### [v1.20.0](https://github.com/zendesk/maxwell/releases/tag/v1.20.0)
- add support for partitioning by transaction ID thx @hexene
- add support for a kafka "fallback" topic to write to
@@ -318,46 +737,58 @@ errors.
in AFTER column statements
-### [v1.19.7](https://github.com/zendesk/maxwell/releases/tag/v1.19.7): "in every corner of your room"
+_Released 2019-02-28_
+
+### [v1.19.7](https://github.com/zendesk/maxwell/releases/tag/v1.19.7)
- fix a parser error with empty sql comments
- interpret latin-1 as windows-1252, not iso-whatever, thx @borleaandrei
-### [v1.19.6](https://github.com/zendesk/maxwell/releases/tag/v1.19.6): "set up for you"
+_Released 2019-01-25_
+
+### [v1.19.6](https://github.com/zendesk/maxwell/releases/tag/v1.19.6)
- Further fixes for GTID-reconnection issues.
- Crash sanely when GTID-enabled maxwell is connected to clearly the wrong master,
thanks @acampoh
-### [v1.19.5](https://github.com/zendesk/maxwell/releases/tag/v1.19.5): "when there is trap"
+_Released 2019-01-20_
+
+### [v1.19.5](https://github.com/zendesk/maxwell/releases/tag/v1.19.5)
- Fixes for unreliable connections wrt to GTID events; previously we
restart in any old position, now we throw away the current transaction
and restart the replicator again at the head of the GTID event.
-### [v1.19.4](https://github.com/zendesk/maxwell/releases/tag/v1.19.4): "and underground"
+_Released 2019-01-15_
+
+### [v1.19.4](https://github.com/zendesk/maxwell/releases/tag/v1.19.4)
- Fixes for a maxwell database not making it through the blacklist
- Add `output_null_zerodates` parameter to control how we treat
'0000-00-00'
-### [v1.19.3](https://github.com/zendesk/maxwell/releases/tag/v1.19.3): "through the roof"
+_Released 2019-01-12_
+
+### [v1.19.3](https://github.com/zendesk/maxwell/releases/tag/v1.19.3)
- Add a universal backpressure mechanism. This should help people who
were running into out-of-memory situations while bootstrapping.
-### [v1.19.2](https://github.com/zendesk/maxwell/releases/tag/v1.19.2): "the same I wore last night"
+_Released 2018-12-19_
+
+### [v1.19.2](https://github.com/zendesk/maxwell/releases/tag/v1.19.2)
- Include schema_id in bootstrap events
- add more logging around binlog connector losing connection
@@ -368,16 +799,20 @@ were running into out-of-memory situations while bootstrapping.
- list changes
-### [v1.19.1](https://github.com/zendesk/maxwell/releases/tag/v1.19.1): "the swoop here doesn't change things one bit"
+_Released 2018-12-02_
+
+### [v1.19.1](https://github.com/zendesk/maxwell/releases/tag/v1.19.1)
- Handle mysql bit literals in DEFAULT statements
- blacklist out CREATE ROLE etc
- upgrade dependencies to pick up security issues
-### [v1.19.0](https://github.com/zendesk/maxwell/releases/tag/v1.19.0): "whole lotta milka"
+_Released 2018-11-12_
+
+### [v1.19.0](https://github.com/zendesk/maxwell/releases/tag/v1.19.0)
- mysql 8 support!
- utf8 enum values are supported now
@@ -388,8 +823,10 @@ were running into out-of-memory situations while bootstrapping.
are a terrible idea?
-### [v1.18.0](https://github.com/zendesk/maxwell/releases/tag/v1.18.0): "hello from the Andes"
+_Released 2018-10-27_
+
+### [v1.18.0](https://github.com/zendesk/maxwell/releases/tag/v1.18.0)
- memory optimizations for large schemas (especially shareded schemas with lots of duplicates)
- add support for an http endpoint to support Prometheus metrics
@@ -401,15 +838,19 @@ were running into out-of-memory situations while bootstrapping.
- add message.publish.age metric
-### [v1.17.1](https://github.com/zendesk/maxwell/releases/tag/v1.17.1): "ay, ay, ay"
+_Released 2018-09-15_
+
+### [v1.17.1](https://github.com/zendesk/maxwell/releases/tag/v1.17.1)
- fix a regression around filters + bootstrapping
- fix a regression around filters + database-only-ddl
-### [v1.17.0](https://github.com/zendesk/maxwell/releases/tag/v1.17.0): "monday, not sunday tuesday"
+_Released 2018-07-03_
+
+### [v1.17.0](https://github.com/zendesk/maxwell/releases/tag/v1.17.0)
v1.17.0 brings a new level of configurability by allowing you to inject
a bit of javascript into maxwell's processing. Should be useful! Also:
@@ -417,16 +858,20 @@ a bit of javascript into maxwell's processing. Should be useful! Also:
- fix regression for Alibaba RDS tables
-### [v1.16.1](https://github.com/zendesk/maxwell/releases/tag/v1.16.1): "the 90 degree angle thing"
+_Released 2018-06-28_
+
+### [v1.16.1](https://github.com/zendesk/maxwell/releases/tag/v1.16.1)
- Fix Bootstrapping for JSON columns
- add --recapture_schema flag for when ya wanna start over
- add kafka 1.0 libraries, make them default
-### [v1.16.0](https://github.com/zendesk/maxwell/releases/tag/v1.16.0): "kind of sort of a reference to something"
+_Released 2018-06-21_
+
+### [v1.16.0](https://github.com/zendesk/maxwell/releases/tag/v1.16.0)
v1.16.0 brings a rewrite of Maxwell's filtering system, giving it a
concise list of rules that are executed in sequence. It's now possible
@@ -435,8 +880,10 @@ value, and probably some other use cases.
See http://maxwells-daemon.io/config/#filtering for details.
-### [v1.15.0](https://github.com/zendesk/maxwell/releases/tag/v1.15.0): "I'm sure I'm being supportive here."
+_Released 2018-06-15_
+
+### [v1.15.0](https://github.com/zendesk/maxwell/releases/tag/v1.15.0)
This is a bug-fix release, but it's big enough I'm giving it a minor
version.
@@ -453,22 +900,28 @@ start of a row, and points the replicator at the next-position.
Much thanks to Tim, Likun and others in sorting this mess out.
-### [v1.14.7](https://github.com/zendesk/maxwell/releases/tag/v1.14.7): "casamir pulaski day"
+_Released 2018-06-13_
+
+### [v1.14.7](https://github.com/zendesk/maxwell/releases/tag/v1.14.7)
- add RowMap#getRowQuery, thx @saimon7
- revert alpine-linux docker image fiasco
- fix RawJSONString not serializable, thx @niuhaifeng
-### [v1.14.6](https://github.com/zendesk/maxwell/releases/tag/v1.14.6): "gimme one sec, I need to grab something"
+_Released 2018-06-03_
+
+### [v1.14.6](https://github.com/zendesk/maxwell/releases/tag/v1.14.6)
- Fix docker image
-### [v1.14.5](https://github.com/zendesk/maxwell/releases/tag/v1.14.5): "he looks funny, he moves funny"
+_Released 2018-05-15_
+
+### [v1.14.5](https://github.com/zendesk/maxwell/releases/tag/v1.14.5)
- reduce docker image footprint
- add benchmarking framework
@@ -476,32 +929,42 @@ Much thanks to Tim, Likun and others in sorting this mess out.
- fix parser error on UPGRADE PARTITIONING
-### [v1.14.4](https://github.com/zendesk/maxwell/releases/tag/v1.14.4): "chinese food"
+_Released 2018-05-15_
+
+### [v1.14.4](https://github.com/zendesk/maxwell/releases/tag/v1.14.4)
- Fix race condition in SchemaCapturer
-### [v1.14.3](https://github.com/zendesk/maxwell/releases/tag/v1.14.3): "what's for lunch?"
+
+_Released 2018-05-07_
+
+### [v1.14.3](https://github.com/zendesk/maxwell/releases/tag/v1.14.3)
- Enable jvm metrics
-### [v1.14.2](https://github.com/zendesk/maxwell/releases/tag/v1.14.2): "bork bork bork"
+_Released 2018-05-04_
+### [v1.14.2](https://github.com/zendesk/maxwell/releases/tag/v1.14.2)
- fix regression in 1.14.1 around bootstrapping host detection
- fix heartbeating code around table includes
-### [v1.14.1](https://github.com/zendesk/maxwell/releases/tag/v1.14.1): "half asleep in frog pajamas"
+_Released 2018-05-02_
+
+### [v1.14.1](https://github.com/zendesk/maxwell/releases/tag/v1.14.1)
- bootstraps can now take a client_id
- improved config validation for embedded mode
-### [v1.14.0](https://github.com/zendesk/maxwell/releases/tag/v1.14.0): "cats, cats, more cats. sadness at lack of cats."
+_Released 2018-05-01_
+
+### [v1.14.0](https://github.com/zendesk/maxwell/releases/tag/v1.14.0)
- new feature `--output_xoffset` to uniquely identify rows within transactions,
thx Jens Gyti
@@ -509,15 +972,22 @@ Much thanks to Tim, Likun and others in sorting this mess out.
- Bug fixes around dates pre 1000 AD
-### [v1.13.5](https://github.com/zendesk/maxwell/releases/tag/v1.13.5): "cyclone keni is real"
+
+_Released 2018-04-24_
+
+### [v1.13.5](https://github.com/zendesk/maxwell/releases/tag/v1.13.5)
- Support environment variable based configuration
-### [v1.13.4](https://github.com/zendesk/maxwell/releases/tag/v1.13.4): "it was just a dream"
+_Released 2018-04-11_
+
+### [v1.13.4](https://github.com/zendesk/maxwell/releases/tag/v1.13.4)
- Added possibility to do not declare the rabbitmq exchange.
-### [v1.13.3](https://github.com/zendesk/maxwell/releases/tag/v1.13.3): "winner winner chicken dinner"
+_Released 2018-04-03_
+
+### [v1.13.3](https://github.com/zendesk/maxwell/releases/tag/v1.13.3)
- Add logging for binlog errors
@@ -526,16 +996,19 @@ Much thanks to Tim, Likun and others in sorting this mess out.
- Always write null fields in primary key fields
- Bugfix: fix http_path_prefix command line option issue
-### [v1.13.2](https://github.com/zendesk/maxwell/releases/tag/v1.13.2): "I just bought them to sleep in"
+_Released 2018-04-03_
+### [v1.13.2](https://github.com/zendesk/maxwell/releases/tag/v1.13.2)
- fix a bug with CHARACTER SET = DEFAULT
- maxwell now eclipse-friendly.
- configurable bind-address for maxwell's http server
-### [v1.13.1](https://github.com/zendesk/maxwell/releases/tag/v1.13.1): "line up your exes in song"
+_Released 2018-03-06_
+
+### [v1.13.1](https://github.com/zendesk/maxwell/releases/tag/v1.13.1)
- redis producer now supports LPUSH, thx @m-denton
- RowMap can now contain artbitrary attributes for embedded maxwell, thx @jkgeyti
@@ -546,8 +1019,10 @@ Much thanks to Tim, Likun and others in sorting this mess out.
- support for --daemon
-### [v1.13.0](https://github.com/zendesk/maxwell/releases/tag/v1.13.0): "sorry, I burned your clothes"
+_Released 2018-02-20_
+
+### [v1.13.0](https://github.com/zendesk/maxwell/releases/tag/v1.13.0)
- proper SSL connection support, thanks @cadams5
- support for including original SQL in insert/update/deletes, thanks @saimon7
@@ -556,8 +1031,10 @@ Much thanks to Tim, Likun and others in sorting this mess out.
- fix for bug when two databases share a single table
-### [v1.12.0](https://github.com/zendesk/maxwell/releases/tag/v1.12.0): "Cold Feet, literally and metaphorically."
+_Released 2018-02-01_
+
+### [v1.12.0](https://github.com/zendesk/maxwell/releases/tag/v1.12.0)
- Support for injecting a custom producer, thanks @tomcollinsproject
- New producer for Amazon SQS, thanks @vikrant2mahajan
@@ -569,14 +1046,19 @@ Much thanks to Tim, Likun and others in sorting this mess out.
- SQL parser bugfix for values like +1.234, thanks @hexene
-### [v1.11.0](https://github.com/zendesk/maxwell/releases/tag/v1.11.0): "the latest, the greatest"
+_Released 2018-01-09_
+
+### [v1.11.0](https://github.com/zendesk/maxwell/releases/tag/v1.11.0)
- default kafka client upgrades to 0.11.0.1
- fix the encryption issue (https://github.com/zendesk/maxwell/issues/803)
-### [v1.10.9](https://github.com/zendesk/maxwell/releases/tag/v1.10.9): "no one left behind"
+
+_Released 2017-11-22_
+
+### [v1.10.9](https://github.com/zendesk/maxwell/releases/tag/v1.10.9)
We recommend all v1.10.7 and v1.10.8 users upgrade to v1.10.9.
@@ -585,38 +1067,49 @@ We recommend all v1.10.7 and v1.10.8 users upgrade to v1.10.9.
- Reduce docker image size
-### [v1.10.8](https://github.com/zendesk/maxwell/releases/tag/v1.10.8): "what doesn't kill you makes you stronger"
+_Released 2017-10-30_
+
+### [v1.10.8](https://github.com/zendesk/maxwell/releases/tag/v1.10.8)
- Fix docker builds
- Add Google Cloud Pub/Sub producer
- RabbitMQ producer enhancements
-### [v1.10.7](https://github.com/zendesk/maxwell/releases/tag/v1.10.7): "it's never too l8!"
+
+_Released 2017-10-12_
+
+### [v1.10.7](https://github.com/zendesk/maxwell/releases/tag/v1.10.7)
- Java 8 upgrade
- Diagnostic health check endpoint
- Encryption
- Documentation update: encryption, kinesis producer, schema storage fundamentals, etc.
-### [v1.10.6](https://github.com/zendesk/maxwell/releases/tag/v1.10.6): "a new starter is here"
+_Released 2017-10-11_
+
+### [v1.10.6](https://github.com/zendesk/maxwell/releases/tag/v1.10.6)
- Binlog-connector upgrade
- Bug-fix: when using literal string for an option that accepts Regex, Regex characters are no longer special
- If master recovery is enabled, Maxwell cleans up old positions for the same server and client id
-### [v1.10.5](https://github.com/zendesk/maxwell/releases/tag/v1.10.5): "half asleep on her couch"
+_Released 2017-08-14_
+
+### [v1.10.5](https://github.com/zendesk/maxwell/releases/tag/v1.10.5)
- Shyko's binlog-connector is now the default and only replication
backend available for maxwell.
-### [v1.10.4](https://github.com/zendesk/maxwell/releases/tag/v1.10.4): "shutdown --harder"
+_Released 2017-07-25_
+
+### [v1.10.4](https://github.com/zendesk/maxwell/releases/tag/v1.10.4)
Notable changes:
@@ -633,22 +1126,29 @@ Notable changes:
binlog_connector mode (thanks Geoff Lywood).
-### [v1.10.3](https://github.com/zendesk/maxwell/releases/tag/v1.10.3): "1.10.2-and-a-bit"
+_Released 2017-07-10_
+
+### [v1.10.3](https://github.com/zendesk/maxwell/releases/tag/v1.10.3)
- tiny release to fix a units error in the `replication.lag` metric
(subtracting seconds from milliseconds)
-### [v1.10.2](https://github.com/zendesk/maxwell/releases/tag/v1.10.2): "just in time for tomorrow"
+_Released 2017-06-06_
+
+### [v1.10.2](https://github.com/zendesk/maxwell/releases/tag/v1.10.2)
- added metrics: "replication.queue.time" and "inflightmessages.count"
- renamed "time.overall" metric to "message.publish.time"
- documentation updates (thanks Chintan Tank)
-### [v1.10.1](https://github.com/zendesk/maxwell/releases/tag/v1.10.1): "forgive and forget"
+
+_Released 2017-06-04_
+
+### [v1.10.1](https://github.com/zendesk/maxwell/releases/tag/v1.10.1)
The observable changes in this minor release are a new configuration for Kafka/Kinesis producer to abort processing on publish errors, and support of Kafka 0.10.2. Also a bunch of good refactoring has been done for heartbeat processing. List of changes:
@@ -658,7 +1158,9 @@ The observable changes in this minor release are a new configuration for Kafka/K
- Allow for empty double-quoted string literals for database schema changes
- Ignore Kafka/Kinesis producer errors based on new configuration ignore_producer_error
-### [v1.10.0](https://github.com/zendesk/maxwell/releases/tag/v1.10.0): "slightly more ones than zeroes"
+_Released 2017-05-26_
+
+### [v1.10.0](https://github.com/zendesk/maxwell/releases/tag/v1.10.0)
This is a small release, primarily around a change to how schemas are
stored. Maxwell now stores the `last_heartbeat_read` with each entry
@@ -677,7 +1179,10 @@ Other minor changes:
- log4j version bump (allows for one entry per line JSON logging)
-### [v1.9.0](https://github.com/zendesk/maxwell/releases/tag/v1.9.0): "now with added whimsy"
+
+_Released 2017-05-09_
+
+### [v1.9.0](https://github.com/zendesk/maxwell/releases/tag/v1.9.0)
Maxwell 1.9 adds one main feature: monitoring support, contributed by
Scott Ferguson. Multiple backends can be configured, read the updated
@@ -691,8 +1196,10 @@ There's also some bugfixes:
- minor logging improvements
-### [v1.8.2](https://github.com/zendesk/maxwell/releases/tag/v1.8.2): "just as the postcards wept"
+_Released 2017-04-26_
+
+### [v1.8.2](https://github.com/zendesk/maxwell/releases/tag/v1.8.2)
Bugfix release.
@@ -701,34 +1208,44 @@ Bugfix release.
- use seconds instead of milliseconds for DDL messages
-### [v1.8.1](https://github.com/zendesk/maxwell/releases/tag/v1.8.1): "famous is faster, don't have to be talented"
+_Released 2017-04-11_
+
+### [v1.8.1](https://github.com/zendesk/maxwell/releases/tag/v1.8.1)
- performance improves in capturing and restoring schema, thx Joren
Minnaert
- Allow for capturing from a separate mysql host (adds support for using
Maxscale as a replication proxy), thx Adam Szkoda
-### [v1.8.0](https://github.com/zendesk/maxwell/releases/tag/v1.8.0): "upbeat, honest, contradictory"
+_Released 2017-02-20_
+
+### [v1.8.0](https://github.com/zendesk/maxwell/releases/tag/v1.8.0)
In version 1.8.0 Maxwell gains alpha support for GTID-based positions!
All praise due to Henry Cai.
-### [v1.7.2](https://github.com/zendesk/maxwell/releases/tag/v1.7.2): "comparing self to better"
+_Released 2017-02-14_
+
+### [v1.7.2](https://github.com/zendesk/maxwell/releases/tag/v1.7.2)
- Fix a bug found where maxwell could cache the wrong TABLE_MAP_ID for a
binlog event, leading to crashes or in some cases data mismatches.
-### [v1.7.1](https://github.com/zendesk/maxwell/releases/tag/v1.7.1): "blame it on your seratonin"
+_Released 2017-01-30_
+
+### [v1.7.1](https://github.com/zendesk/maxwell/releases/tag/v1.7.1)
- bootstrapping now can take a `--where` clause
- performance improvements in the kafka producer
-### [v1.7.0](https://github.com/zendesk/maxwell/releases/tag/v1.7.0): "lucky me, lucky mud"
+_Released 2017-01-24_
+
+### [v1.7.0](https://github.com/zendesk/maxwell/releases/tag/v1.7.0)
Maxwell 1.7 brings 2 major new, alpha features. The first is Mysql 5.7
support, including JSON column type support and handling of 5.7 SQL, but
@@ -744,7 +1261,10 @@ There's also some bugfixes:
- Amazon RDS heartbeat events now tick maxwell's position, thx Scott Ferguson
- allow CHECK() statements inside column definitions
-### [v1.6.0](https://github.com/zendesk/maxwell/releases/tag/v1.6.0): "give me a quest"
+
+_Released 2017-01-07_
+
+### [v1.6.0](https://github.com/zendesk/maxwell/releases/tag/v1.6.0)
This is mostly a bugfix release, but it gets a minor version bump due to
a single change of behavior: dates and timestamps which mysql may
@@ -757,8 +1277,10 @@ Other bugfixes:
restart
- allow pointing maxwell to a pre-existing database
-### [v1.5.2](https://github.com/zendesk/maxwell/releases/tag/v1.5.2): "french banana"
+_Released 2016-12-29_
+
+### [v1.5.2](https://github.com/zendesk/maxwell/releases/tag/v1.5.2)
- add support for kafka 0.10.1 @ smferguson
- master recovery: cleanup positions from previous master; prevent
@@ -766,7 +1288,10 @@ Other bugfixes:
- fix a bug that would trigger in certain cases when dropping a column
that was part of the primary-key
-### [v1.5.1](https://github.com/zendesk/maxwell/releases/tag/v1.5.1): "1.5.1 is just 1.5.1"
+
+_Released 2016-12-07_
+
+### [v1.5.1](https://github.com/zendesk/maxwell/releases/tag/v1.5.1)
This is a bugfix release.
- fixes for bootstrapping with an alternative maxwell-schema name and an
@@ -776,8 +1301,10 @@ This is a bugfix release.
- Get the bootstrapping process to output NULL values.
- fix a quoting issue in the bootstrap code, thanks @mylesjao.
-### [v1.5.0](https://github.com/zendesk/maxwell/releases/tag/v1.5.0): "someone, somewhere, is still smoking cigarettes, damnit"
+_Released 2016-11-24_
+
+### [v1.5.0](https://github.com/zendesk/maxwell/releases/tag/v1.5.0)
- CHANGE: Kafka producer no longer ships with hard-coded defaults.
Please ensure you have "compression.type", "metadata.fetch.timeout.ms", and "retries"
@@ -785,34 +1312,45 @@ This is a bugfix release.
- bugfix: fix a regression in handling `ALTER TABLE change c int after b` statements
- warn on servers with missing server_id
-### [v1.4.2](https://github.com/zendesk/maxwell/releases/tag/v1.4.2): "drawer cat is back"
+_Released 2016-11-07_
+
+### [v1.4.2](https://github.com/zendesk/maxwell/releases/tag/v1.4.2)
- kafka 0.10.0 support, as well as a re-working of the --kafka_version
command line option.
-### [v1.4.1](https://github.com/zendesk/maxwell/releases/tag/v1.4.1): "cat snores"
+_Released 2016-11-01_
+
+### [v1.4.1](https://github.com/zendesk/maxwell/releases/tag/v1.4.1)
- support per-table topics, Thanks @smferguson and @sschatts.
- fix a parser issue with DROP COLUMN CASCADE, thanks @smferguson
-### [v1.4.0](https://github.com/zendesk/maxwell/releases/tag/v1.4.0): "deep, insomniac character flaws"
+
+_Released 2016-10-27_
+
+### [v1.4.0](https://github.com/zendesk/maxwell/releases/tag/v1.4.0)
1.4.0 brings us two nice new features:
- partition-by-column: see --kafka_partition_columns. Thanks @smferguson
- output schema changes as JSON: see --output_ddl. Thanks @xmlking
- As well as a fix around race conditions on shutdown.
-### [v1.3.0](https://github.com/zendesk/maxwell/releases/tag/v1.3.0): "yogg-saron"
+_Released 2016-10-21_
+
+### [v1.3.0](https://github.com/zendesk/maxwell/releases/tag/v1.3.0)
- support for fractional DATETIME, TIME, TIMESTAMP columns, thanks @Dagnan
- support for outputting server_id & thread_id, thanks @sagiba
- fix a race condition in bootstrap support
-### [v1.2.2](https://github.com/zendesk/maxwell/releases/tag/v1.2.2): "bats wearing frog pajamas"
+_Released 2016-10-03_
+
+### [v1.2.2](https://github.com/zendesk/maxwell/releases/tag/v1.2.2)
- Maxwell will now include by default fields with NULL values (as null
fields). To disable this and restore the old functionality where fields
@@ -823,16 +1361,20 @@ This is a bugfix release.
mismatched client_id
- Fix a bug when using CHANGE COLUMN on a primary key
-### [v1.2.1](https://github.com/zendesk/maxwell/releases/tag/v1.2.1): "point-ones are a sad and inevitable fact"
+_Released 2016-09-23_
+
+### [v1.2.1](https://github.com/zendesk/maxwell/releases/tag/v1.2.1)
This is a bugfix release.
- fix a parser bug around ALTER TABLE CHARACTER SET
- fix bin/maxwell to pull in the proper version of the kafka-clients
library
-### [v1.2.0](https://github.com/zendesk/maxwell/releases/tag/v1.2.0): "just here, not to talk to you"
+_Released 2016-09-15_
+
+### [v1.2.0](https://github.com/zendesk/maxwell/releases/tag/v1.2.0)
1.2.0 is a major release of Maxwell that introduces master recovery
features; when a slave is promoted to master, Maxwell is now capable of
@@ -843,14 +1385,18 @@ It also upgrades the kafka producer library to 0.9. If you're using
maxwell with a kafka 0.8 server, you must now pass the `--kafka0.8` flag
to maxwell.
-### [v1.1.6](https://github.com/zendesk/maxwell/releases/tag/v1.1.6): "pithy"
+_Released 2016-09-12_
+
+### [v1.1.6](https://github.com/zendesk/maxwell/releases/tag/v1.1.6)
- minor bugfix in which maxwell with --replay mode was trying to write
heartbeats
-### [v1.1.5](https://github.com/zendesk/maxwell/releases/tag/v1.1.5): "my brain is a polluted mess"
+_Released 2016-09-07_
+
+### [v1.1.5](https://github.com/zendesk/maxwell/releases/tag/v1.1.5)
- @dadah89 adds --output_binlog_position to optionally output the
position with the row
@@ -862,15 +1408,20 @@ to maxwell.
- lay the ground work for doing master recovery; we add a heartbeat into
the positions table that we can co-ordinate around.
-### [v1.1.4](https://github.com/zendesk/maxwell/releases/tag/v1.1.4): "george flunk"
+_Released 2016-09-04_
+
+### [v1.1.4](https://github.com/zendesk/maxwell/releases/tag/v1.1.4)
- add support for a bunch more charsets (gbk, big5, notably)
- fix Maxwell's handling of kafka errors - previously we were trying to
crash Maxwell by throwing a RuntimeException out of the Kafka
Producer, but this was a failure. Now we log and skip all errors.
-### [v1.1.3](https://github.com/zendesk/maxwell/releases/tag/v1.1.3): "the button I push to not have to go out"
+
+_Released 2016-08-05_
+
+### [v1.1.3](https://github.com/zendesk/maxwell/releases/tag/v1.1.3)
This is a bugfix release, which fixes:
- https://github.com/zendesk/maxwell/issues/376, a problem parsing
@@ -882,8 +1433,10 @@ This is a bugfix release, which fixes:
- https://github.com/zendesk/maxwell/issues/373, we were incorrectly
expecting heartbeats to work under 5.1
-### [v1.1.2](https://github.com/zendesk/maxwell/releases/tag/v1.1.2): "scribbled notes on red pages"
+_Released 2016-07-14_
+
+### [v1.1.2](https://github.com/zendesk/maxwell/releases/tag/v1.1.2)
- pick up latest mysql-connector-j, fixes #369
- fix an issue where maxwell could skip ahead positions if a leader failed.
@@ -891,14 +1444,18 @@ This is a bugfix release, which fixes:
of very large transactions / rows inside transactions
- kinder, gentler help text when you specify an option incorrectly
-### [v1.1.1](https://github.com/zendesk/maxwell/releases/tag/v1.1.1): scribbled notes on blue pages
+_Released 2016-06-27_
+
+### [v1.1.1](https://github.com/zendesk/maxwell/releases/tag/v1.1.1)
- fixes a race condition setting the binlog position that would get
maxwell stuck
-### [v1.1.0](https://github.com/zendesk/maxwell/releases/tag/v1.1.0): "sleep away the afternoon"
+_Released 2016-05-23_
+
+### [v1.1.0](https://github.com/zendesk/maxwell/releases/tag/v1.1.0)
- much more efficient processing of schema updates storage, especially when dealing with large schemas.
- @lileeyao added --exclude-columns and the --jdbc_options features
@@ -908,30 +1465,41 @@ This is a bugfix release, which fixes:
- bugfix: unsigned integer columns were captured incorrectly. 1.1 will
recapture the schema and attempt to correct the error.
-### [v1.1.0-pre4](https://github.com/zendesk/maxwell/releases/tag/v1.1.0-pre4): "buck buck buck buck buck buck-AH!"
+_Released 2016-05-20_
+
+### [v1.1.0-pre4](https://github.com/zendesk/maxwell/releases/tag/v1.1.0-pre4)
- Eddie McLean gives some helpful patches around bootstrapping
- Bugfixes for the patch-up-the-schema code around unsigned ints
-### [v1.1.0-pre3](https://github.com/zendesk/maxwell/releases/tag/v1.1.0-pre3):
+
+_Released 2016-05-06_
+
+### [v1.1.0-pre3](https://github.com/zendesk/maxwell/releases/tag/v1.1.0-pre3)
- forgot to include some updates that back-patch unsigned column
problems
-### [v1.1.0-pre2](https://github.com/zendesk/maxwell/releases/tag/v1.1.0-pre2): "yawn yawn"
+_Released 2016-05-05_
+
+### [v1.1.0-pre2](https://github.com/zendesk/maxwell/releases/tag/v1.1.0-pre2)
- fix performance issues when capturing schema in AWS Aurora
- fix a bug in capturing unsigned integer columns
-### [v1.0.1](https://github.com/zendesk/maxwell/releases/tag/v1.0.1): "bag of oversized daisies"
+_Released 2016-05-04_
+
+### [v1.0.1](https://github.com/zendesk/maxwell/releases/tag/v1.0.1)
- fixes a parsing bug with `CURRENT_TIMESTAMP()`
-### [v1.0.0](https://github.com/zendesk/maxwell/releases/tag/v1.0.0): "Maxwell learns to speak"
+_Released 2016-04-12_
+
+### [v1.0.0](https://github.com/zendesk/maxwell/releases/tag/v1.0.0)
Since v0.17.0, Maxwell has gotten:
- bootstrapping support
@@ -944,27 +1512,35 @@ Since v0.17.0, Maxwell has gotten:
and I, Osheroff, think the damn thing is stable enough for a 1.0. So
there.
-### [v1.0.0-RC3](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-RC3): "C'mon and take it"
+_Released 2016-03-11_
+
+### [v1.0.0-RC3](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-RC3)
pull in support for replication heartbeats. helps in the flakier
network environs.
-### [v1.0.0-RC2](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-RC2): "same thing, just without the v"
+_Released 2016-03-08_
+
+### [v1.0.0-RC2](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-RC2)
- fixes the way ALTER DATABASE charset= was handled
- adds proper handling of ALTER TABLE CONVERT TO CHARSET
-### [v1.0.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-RC1): "Richard Buckner's release"
+_Released 2016-02-20_
+
+### [v1.0.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-RC1)
- modifications to the way the bootstrap utility works
- fix a race condition crash bug in bootstrapping
- fix a parser bug
-### [v1.0.0-PRE2](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-PRE2): "an embarassment of riches"
+_Released 2016-02-11_
+
+### [v1.0.0-PRE2](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-PRE2)
1.0.0-PRE2 brings in a lot of changes that got merged while we were
testing out PRE1. so, hey.
@@ -975,13 +1551,18 @@ testing out PRE1. so, hey.
- add `--blacklist_tables` option to fully ignore excessive schema changes (Nicolas Maquet)
- bootstrap rows now have 'bootstrap-insert' type
-### [v1.0.0-PRE1](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-PRE1): "drunk conversations with sober people"
+
+_Released 2016-01-30_
+
+### [v1.0.0-PRE1](https://github.com/zendesk/maxwell/releases/tag/v1.0.0-PRE1)
- Here we have the preview release of @nmaquet's excellent work around
bootstrapping initial versions of mysql tables.
-### [v0.17.0](https://github.com/zendesk/maxwell/releases/tag/v0.17.0): "wrists of William"
+_Released 2016-01-09_
+
+### [v0.17.0](https://github.com/zendesk/maxwell/releases/tag/v0.17.0)
v0.17 is a large bugfix release with one new feature.
- FEATURE: allow specifying an alternative mysql schema-storage server and
@@ -992,8 +1573,10 @@ v0.17 is a large bugfix release with one new feature.
- BUGFIX: many more SQL-parser fixes. We are mostly through some
thousands of lines of SQL produced by mysql-test.
-### [v0.16.2](https://github.com/zendesk/maxwell/releases/tag/v0.16.2): "The best laid plans"
+_Released 2016-01-07_
+
+### [v0.16.2](https://github.com/zendesk/maxwell/releases/tag/v0.16.2)
This is a large-ish bugfix release.
- Support, with reservations, binlog_row_image=MINIMAL
@@ -1003,8 +1586,10 @@ This is a large-ish bugfix release.
- support UCS2 (start trying to operate ok on the mysql-test suite)
- use ObjectOutputStream.reset to fix memory leaks when buffering to disk
-### [v0.16.1](https://github.com/zendesk/maxwell/releases/tag/v0.16.1): "me and room service"
+_Released 2015-12-16_
+
+### [v0.16.1](https://github.com/zendesk/maxwell/releases/tag/v0.16.1)
This is a bug-fix-roundup release:
- support ALTER DATABASE
@@ -1013,57 +1598,86 @@ This is a bug-fix-roundup release:
- some modifications to the overflow-to-disk logic; we buffer the input
and output, and we fix a memory leak
-### [v0.16.0](https://github.com/zendesk/maxwell/releases/tag/v0.16.0): "Kristian Kaufmann's version"
+_Released 2015-12-11_
+
+### [v0.16.0](https://github.com/zendesk/maxwell/releases/tag/v0.16.0)
Version 0.16.0 introduces a feature where UPDATE statements will now
show both the new row image and the old values of the fields that
changed. Thanks @kristiankaufmann
-### [v0.15.0](https://github.com/zendesk/maxwell/releases/tag/v0.15.0): "the littlest little city"
+
+_Released 2015-12-10_
+
+### [v0.15.0](https://github.com/zendesk/maxwell/releases/tag/v0.15.0)
- fix a parse problem with indices ordered by ASC/DESC
-### [v0.15.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v0.15.0-RC1): "it's later than you think"
+_Released 2015-12-07_
+
+### [v0.15.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v0.15.0-RC1)
- large transactions now buffer to disk instead of crushing maxwell.
- support ALGORITHM=[algo], LOCK=[lock] for 5.6 alters
-### [v0.14.6](https://github.com/zendesk/maxwell/releases/tag/v0.14.6): "It's about being American. Sort of."
+_Released 2015-12-04_
+
+### [v0.14.6](https://github.com/zendesk/maxwell/releases/tag/v0.14.6)
- fix TIME column support
- fix parsing on millisecond precision column defintions
- fix CREATE SCHEMA parsing
-### [v0.14.5](https://github.com/zendesk/maxwell/releases/tag/v0.14.5): "false is the new true"
+_Released 2015-11-27_
+
+### [v0.14.5](https://github.com/zendesk/maxwell/releases/tag/v0.14.5)
- handle BOOLEAN columns with true/false defaults
-### [v0.14.4](https://github.com/zendesk/maxwell/releases/tag/v0.14.4): "You'd think we'd be at 1.0 by now, wouldn't you?"
+
+_Released 2015-11-25_
+
+### [v0.14.4](https://github.com/zendesk/maxwell/releases/tag/v0.14.4)
- fixes parsing of "mysql comments" (`/*! .. */`)
- More performance improvements, another 10% in a tight loop.
-### [v0.14.3](https://github.com/zendesk/maxwell/releases/tag/v0.14.3): "Peanuts. My girlfriend thinks about peanuts."
+
+_Released 2015-11-24_
+
+### [v0.14.3](https://github.com/zendesk/maxwell/releases/tag/v0.14.3)
- fixes a regression in 0.14.2 that creates duplicate copies of the "mysql" database in the schema.
-### [v0.14.2](https://github.com/zendesk/maxwell/releases/tag/v0.14.2): "Maxwell Sandvik 88"
+
+_Released 2015-11-23_
+
+### [v0.14.2](https://github.com/zendesk/maxwell/releases/tag/v0.14.2)
- capture the mysql database along with the rest of the schema. Eliding it was a bad premature optimization that led to crashes when tables in the mysql database changed.
-### [v0.14.1](https://github.com/zendesk/maxwell/releases/tag/v0.14.1): "be liberal in what you accept. Even if nonsensical."
+
+_Released 2015-11-20_
+
+### [v0.14.1](https://github.com/zendesk/maxwell/releases/tag/v0.14.1)
- fixes a parser bug around named PRIMARY KEYs.
-### [v0.14.0](https://github.com/zendesk/maxwell/releases/tag/v0.14.0): "the slow but inevitable slide"
+
+_Released 2015-11-17_
+
+### [v0.14.0](https://github.com/zendesk/maxwell/releases/tag/v0.14.0)
This release introduces row filters, allowing you to include or exclude tables from maxwell's output based on names or regular expressions.
-### [v0.13.1](https://github.com/zendesk/maxwell/releases/tag/v0.13.1): "well that was somewhat expected"
+
+_Released 2015-11-03_
+
+### [v0.13.1](https://github.com/zendesk/maxwell/releases/tag/v0.13.1)
v0.13.1 is a bug fix of v0.13.0 -- fixes a bug where long rows were truncated.
@@ -1072,7 +1686,10 @@ v0.13.0 contains:
- @davidsheldon contributed some nice bug fixes around `CREATE TABLE ... IF NOT EXISTS`, which were previously generating new, bogus copies of the schema.
- we now include a "scavenger thread" that will lazily clean out old, deleted schemas.
-### [v0.13.0](https://github.com/zendesk/maxwell/releases/tag/v0.13.0): "Malkovich Malkovich Malkovich Sheldon?"
+
+_Released 2015-10-29_
+
+### [v0.13.0](https://github.com/zendesk/maxwell/releases/tag/v0.13.0)
Lucky release number 13 brings some reasonably big changes:
- Big performance boost for maxwell: 75% faster in some benchmarks
@@ -1081,15 +1698,24 @@ Lucky release number 13 brings some reasonably big changes:
_This release has a pretty bad bug. do not use._
-### [v0.12.0](https://github.com/zendesk/maxwell/releases/tag/v0.12.0): "what do I call them? Slippers? Why, are you jealous?"
+
+_Released 2015-10-29_
+
+### [v0.12.0](https://github.com/zendesk/maxwell/releases/tag/v0.12.0)
- add support for BIT columns.
-### [v0.11.4](https://github.com/zendesk/maxwell/releases/tag/v0.11.4): "13 steps"
+
+_Released 2015-10-16_
+
+### [v0.11.4](https://github.com/zendesk/maxwell/releases/tag/v0.11.4)
this is another bugfix release that fixes a problem where the replication thread can die in the middle of processing a transaction event. I really need to fix this at a lower level, ie the open-replicator level.
-### [v0.11.3](https://github.com/zendesk/maxwell/releases/tag/v0.11.3): ".. and the other half is to take the bugs out"
+
+_Released 2015-09-30_
+
+### [v0.11.3](https://github.com/zendesk/maxwell/releases/tag/v0.11.3)
this is a bugfix release:
- fix problems with table creation options inside alter statements ( `ALTER TABLE foo auto_increment=10` )
@@ -1097,7 +1723,10 @@ this is a bugfix release:
the test suite should also be way more reliable, not like you care.
-### [v0.11.2](https://github.com/zendesk/maxwell/releases/tag/v0.11.2): "savage acts of unprovoked violence are bad"
+
+_Released 2015-09-29_
+
+### [v0.11.2](https://github.com/zendesk/maxwell/releases/tag/v0.11.2)
This is a bugfix release. It includes:
- soft deletions of maxwell.schemas to fix A->B->A master swapping without creating intense replication delay
@@ -1105,175 +1734,292 @@ This is a bugfix release. It includes:
- kill off maxwell if the position thread dies
- fix a bug where maxwell could pick up a copy of schema from a different server_id (curse you operator precedence!)
-### [v0.11.1](https://github.com/zendesk/maxwell/releases/tag/v0.11.1): "dog snoring loudly"
+
+_Released 2015-09-18_
+
+### [v0.11.1](https://github.com/zendesk/maxwell/releases/tag/v0.11.1)
- maxwell gets a very minimal pass at detecting when a master has changed, in which it will kill off schemas and positions from a server_id that no longer is valid. this should prevent the worst of cases.
-### [v0.11.0](https://github.com/zendesk/maxwell/releases/tag/v0.11.0): "cat waving gently"
+
+_Released 2015-09-16_
+
+### [v0.11.0](https://github.com/zendesk/maxwell/releases/tag/v0.11.0)
This release of Maxwell preserves transaction information in the kafka stream by adding a `xid` key in the JSON object, as well as a `commit` key for the final row inside the transaction.
It also contains a bugfix around server_id handling.
-### [v0.10.1](https://github.com/zendesk/maxwell/releases/tag/v0.10.1): "all 64 of your bases belong to... shut up, internet parrot."
+
+_Released 2015-09-15_
+
+### [v0.10.1](https://github.com/zendesk/maxwell/releases/tag/v0.10.1)
- proper support for BLOB, BINARY, VARBINARY columns (base 64 encoded)
- fix a problem with the SQL parser where specifying encoding or collation in a string column in the wrong order would crash
- make table option parsing more lenient
-### [v0.11.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-RC1): "goin' faster than a rollercoaster"
+
+_Released 2015-09-11_
+
+### [v0.11.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-RC1)
- merge master fixes
-### [v0.10.0](https://github.com/zendesk/maxwell/releases/tag/v0.10.0): "The first word is French"
+
+_Released 2015-09-09_
+
+### [v0.11.0-PRE4](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE4)
+
+- bugfix on v0.11.0-PRE3
+
+
+_Released 2015-09-09_
+
+### [v0.10.0](https://github.com/zendesk/maxwell/releases/tag/v0.10.0)
- Mysql 5.6 checksum support!
- some more bugfixes with the SQL parser
-### [v0.11.0-PRE4](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE4): "except for that other thing"
-- bugfix on v0.11.0-PRE3
+_Released 2015-09-09_
-### [v0.11.0-PRE3](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE3): "nothing like a good night's sleep"
+### [v0.11.0-PRE3](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE3)
- handle SAVEPOINT within transactions
- downgrade unhandled SQL to a warning
-### [v0.11.0-PRE2](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE2): "you really need to name a *PRE* release something cutesy?"
+
+_Released 2015-09-08_
+
+### [v0.11.0-PRE2](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE2)
- fixes for myISAM "transactions"
-### [v0.11.0-PRE1](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE1): "A slow traffic jam towards the void"
+
+_Released 2015-09-03_
+
+### [v0.11.0-PRE1](https://github.com/zendesk/maxwell/releases/tag/v0.11.0-PRE1)
- fix a server_id bug (was always 1 in maxwell.schemas)
- JSON output now includes transaction IDs
-### [v0.10.0-RC4](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC4): "Inspiring confidence"
+
+_Released 2015-09-02_
+
+### [v0.10.0-RC4](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC4)
- deal with BINARY flag in string column creation.
-### [v0.9.5](https://github.com/zendesk/maxwell/releases/tag/v0.9.5): "Long story short, that's why I'm late"
+
+_Released 2015-08-31_
+
+### [v0.9.5](https://github.com/zendesk/maxwell/releases/tag/v0.9.5)
- handle the BINARY flag in column creation
-### [v0.10.0-RC3](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC3): "Except for that one thing"
+
+_Released 2015-08-31_
+
+### [v0.10.0-RC3](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC3)
- handle "TRUNCATE [TABLE_NAME]" statements
-### [v0.10.0-RC2](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC2): "RC2 is always a good sign."
+
+_Released 2015-08-27_
+
+### [v0.10.0-RC2](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC2)
- fixes a bug with checksum processing.
-### [v0.10.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC1): "verify all the things"
+
+_Released 2015-08-26_
+
+### [v0.10.0-RC1](https://github.com/zendesk/maxwell/releases/tag/v0.10.0-RC1)
- upgrade to open-replicator 1.3.0-RC1, which brings binlog checksum (and thus easy 5.6.1) support to maxwell.
-### [v0.9.4](https://github.com/zendesk/maxwell/releases/tag/v0.9.4): "we've been here before"
+
+_Released 2015-08-04_
+
+### [v0.9.4](https://github.com/zendesk/maxwell/releases/tag/v0.9.4)
- allow a configurable number (including unlimited) of schemas to be stored
-### [v0.9.3](https://github.com/zendesk/maxwell/releases/tag/v0.9.3): "some days it's just better to stay in bed"
+
+_Released 2015-07-27_
+
+### [v0.9.3](https://github.com/zendesk/maxwell/releases/tag/v0.9.3)
- bump open-replicator to 1.2.3, which allows processing of single rows greater than 2^24 bytes
-### [v0.9.2](https://github.com/zendesk/maxwell/releases/tag/v0.9.2): "Cat's tongue"
+
+_Released 2015-07-14_
+
+### [v0.9.2](https://github.com/zendesk/maxwell/releases/tag/v0.9.2)
- bump open-replicator buffer to 50mb by default
- log to STDERR, not STDOUT
- `--output_file` option for file producer
-### [v0.9.1](https://github.com/zendesk/maxwell/releases/tag/v0.9.1): "bugs, bugs, bugs, lies, statistics"
+
+_Released 2015-07-10_
+
+### [v0.9.1](https://github.com/zendesk/maxwell/releases/tag/v0.9.1)
- Maxwell is now aware that column names are case-insenstive
- fix a nasty bug in which maxwell would store the wrong position after it lost its connection to the master.
-### [v0.9.0](https://github.com/zendesk/maxwell/releases/tag/v0.9.0): Vanchi says "eat"
+
+_Released 2015-06-22_
+
+### [v0.9.0](https://github.com/zendesk/maxwell/releases/tag/v0.9.0)
Also, vanchi is so paranoid he's worried immediately about this.
- mysql 5.6 support (without checksum support, yet)
- fix a bunch of miscellaneous bugs @akshayi1 found (REAL, BOOL, BOOLEAN types, TRUNCATE TABLE)
-### [v0.8.1](https://github.com/zendesk/maxwell/releases/tag/v0.8.1): "Pascal says Bonjour"
+
+_Released 2015-06-18_
+
+### [v0.8.1](https://github.com/zendesk/maxwell/releases/tag/v0.8.1)
- minor bugfix release around mysql connections going away.
-### [v0.8.0](https://github.com/zendesk/maxwell/releases/tag/v0.8.0): the cat never shuts up
+
+_Released 2015-06-16_
+
+### [v0.8.0](https://github.com/zendesk/maxwell/releases/tag/v0.8.0)
- add "ts" field to row output
- add --config option for passing a different config file
- support int1, int2, int4, int8 columns
-### [v0.7.2](https://github.com/zendesk/maxwell/releases/tag/v0.7.2): "all the sql ladies"
+
+_Released 2015-06-09_
+
+### [v0.7.2](https://github.com/zendesk/maxwell/releases/tag/v0.7.2)
- handle inline sql comments
- ignore more user management SQL
-### [v0.7.1](https://github.com/zendesk/maxwell/releases/tag/v0.7.1): "not hoarders"
+
+_Released 2015-05-29_
+
+### [v0.7.1](https://github.com/zendesk/maxwell/releases/tag/v0.7.1)
- only keep 5 most recent schemas
-### [v0.7.0](https://github.com/zendesk/maxwell/releases/tag/v0.7.0): 0.7.0, "alameda"
+
+_Released 2015-05-15_
+
+### [v0.7.0](https://github.com/zendesk/maxwell/releases/tag/v0.7.0)
- handle CURRENT_TIMESTAMP parsing properly
- better binlog position sync behavior
-### [v0.6.3](https://github.com/zendesk/maxwell/releases/tag/v0.6.3): 0.6.3
+
+_Released 2015-04-28_
+
+### [v0.6.3](https://github.com/zendesk/maxwell/releases/tag/v0.6.3)
- better blacklist for CREATE TRIGGER
-### [v0.6.2](https://github.com/zendesk/maxwell/releases/tag/v0.6.2): v0.6.2
+
+_Released 2015-04-13_
+
+### [v0.6.2](https://github.com/zendesk/maxwell/releases/tag/v0.6.2)
- maxwell now ignores SAVEPOINT statements.
-### [v0.6.1](https://github.com/zendesk/maxwell/releases/tag/v0.6.1): v0.6.1
+
+_Released 2015-04-13_
+
+### [v0.6.1](https://github.com/zendesk/maxwell/releases/tag/v0.6.1)
- fixes a bug with parsing length-limited indexes.
-### [v0.6.0](https://github.com/zendesk/maxwell/releases/tag/v0.6.0): kafkakafkakafa
+
+_Released 2015-04-13_
+
+### [v0.6.0](https://github.com/zendesk/maxwell/releases/tag/v0.6.0)
Version 0.6.0 has Maxwell outputting a JSON kafka key, so that one can use Kafka's neat "store the last copy of a key" retention policy. It also fixes a couple of bugs in the query parsing path.
-### [v0.5.0](https://github.com/zendesk/maxwell/releases/tag/v0.5.0): 0.5.0 -- "People who put commas in column names deserve undefined behavior"
+
+_Released 2015-04-09_
+
+### [v0.5.0](https://github.com/zendesk/maxwell/releases/tag/v0.5.0)
- maxwell now captures primary keys on tables. We'll use this to form kafka key names later.
- maxwell now outputs to a single topic, hashing the data by database name to keep a database's updates in order.
-### [v0.4.0](https://github.com/zendesk/maxwell/releases/tag/v0.4.0): 0.4.0, "unboxed cat"
+
+_Released 2015-04-06_
+
+### [v0.4.0](https://github.com/zendesk/maxwell/releases/tag/v0.4.0)
v0.4.0 fixes some bugs with long-lived mysql connections by adding connection pooling support.
-### [v0.3.0](https://github.com/zendesk/maxwell/releases/tag/v0.3.0): 0.3.0
+
+_Released 2015-03-25_
+
+### [v0.3.0](https://github.com/zendesk/maxwell/releases/tag/v0.3.0)
This version fixes a fairly nasty bug in which the binlog-position flush thread was sharing a connection with the rest of the system, leading to crashes.
It also enables kafka gzip compression by default.
-### [v0.2.2](https://github.com/zendesk/maxwell/releases/tag/v0.2.2): 0.2.2
+
+_Released 2015-03-24_
+
+### [v0.2.2](https://github.com/zendesk/maxwell/releases/tag/v0.2.2)
Version 0.2.2 sets up the LANG environment variable, which fixes a bug in utf-8 handling.
-### [v0.2.1](https://github.com/zendesk/maxwell/releases/tag/v0.2.1): v0.2.1
+
+_Released 2015-03-22_
+
+### [v0.2.1](https://github.com/zendesk/maxwell/releases/tag/v0.2.1)
version 0.2.1 makes Maxwell ignore CREATE INDEX ddl statements and others.
-### [v0.2.0](https://github.com/zendesk/maxwell/releases/tag/v0.2.0): 0.2.0
+
+_Released 2015-03-21_
+
+### [v0.2.0](https://github.com/zendesk/maxwell/releases/tag/v0.2.0)
This release gets Maxwell storing the last-written binlog position inside the mysql master itself.
-### [v0.1.4](https://github.com/zendesk/maxwell/releases/tag/v0.1.4): 0.1.4
+
+_Released 2015-03-18_
+
+### [v0.1.4](https://github.com/zendesk/maxwell/releases/tag/v0.1.4)
support --position_file param
-### [v0.1.3](https://github.com/zendesk/maxwell/releases/tag/v0.1.3): 0.1.3
+
+_Released 2015-03-09_
+
+### [v0.1.3](https://github.com/zendesk/maxwell/releases/tag/v0.1.3)
Adds kafka command line options.
-### [v0.1.1](https://github.com/zendesk/maxwell/releases/tag/v0.1.1): 0.1.1
+
+_Released 2015-03-09_
+
+### [v0.1.1](https://github.com/zendesk/maxwell/releases/tag/v0.1.1)
v0.1.1, a small bugfix release.
-### [v0.1](https://github.com/zendesk/maxwell/releases/tag/v0.1): 0.1
+
+_Released 2015-03-06_
+
+### [v0.1](https://github.com/zendesk/maxwell/releases/tag/v0.1)
This is the first possible release of Maxwell that might work. It includes some exceedingly basic kafka support, and JSON output of binlog deltas.
+
+_Released 2015-03-04_
+
diff --git a/Dockerfile b/Dockerfile
index 75ff164fe..6b552e60e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,6 @@
-FROM maven:3.6-jdk-11
-ENV MAXWELL_VERSION=1.33.0 KAFKA_VERSION=1.0.0
+FROM maven:3.9.9-eclipse-temurin-23 AS builder
+ENV MAXWELL_VERSION=1.42.3 KAFKA_VERSION=1.0.0
+
RUN apt-get update \
&& apt-get -y upgrade \
@@ -18,6 +19,20 @@ RUN cd /workspace \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* /workspace/ /root/.m2/ \
&& echo "$MAXWELL_VERSION" > /REVISION
+# Build clean image with non-root priveledge
+FROM openjdk:23-jdk-slim
+
+RUN apt-get update \
+ && apt-get -y upgrade
+
+COPY --from=builder /app /app
+COPY --from=builder /REVISION /REVISION
+
WORKDIR /app
+RUN useradd -u 1000 maxwell -d /app
+RUN chown 1000:1000 /app
+
+USER 1000
+
CMD [ "/bin/bash", "-c", "bin/maxwell-docker" ]
diff --git a/LICENSE b/LICENSE
index 17ba95a7b..544b2b4d2 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
- Copyright 2015 Zendesk
+ Copyright 2025 Zendesk
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/Makefile b/Makefile
index c4d9fcc7e..db1b58d2a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-KAFKA_VERSION ?= 1.0.0
+KAFKA_VERSION ?= 2.7.0
KAFKA_PROFILE = kafka-${KAFKA_VERSION}
export JAVA_TOOL_OPTIONS = -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
@@ -16,9 +16,13 @@ clean:
depclean: clean
rm -f $(CLASSPATH)
-package: depclean kafka-0.8.2.2 kafka-0.9.0.1 kafka-0.10.0.1 kafka-0.10.2.1 kafka-0.11.0.1 kafka-1.0.0 kafka-2.7.0
+package: depclean kafka-0.8.2.2 kafka-0.9.0.1 kafka-0.10.0.1 kafka-0.10.2.1 kafka-0.11.0.1 kafka-1.0.0 kafka-2.7.0 kafka-3.4.0
@# TODO: this is inefficient, we really just want to copy the jars...
mvn package -DskipTests=true
kafka-%:
mvn compile -P kafka-$(*)
+
+FORCE:
+docs: FORCE
+ mvn javadoc:javadoc
diff --git a/README.md b/README.md
index baafcf6fe..29dfdb2fc 100644
--- a/README.md
+++ b/README.md
@@ -1,44 +1,33 @@
-
-
+This is __Maxwell's daemon__, a [change data capture](https://www.confluent.io/blog/how-change-data-capture-works-patterns-solutions-implementation/) application
+that reads MySQL binlogs and writes data changes as JSON to Kafka, Kinesis, and other streaming platforms.
-This is Maxwell's daemon, an application that reads MySQL binlogs and writes
-row updates as JSON to Kafka, Kinesis, or other streaming platforms. Maxwell has
-low operational overhead, requiring nothing but mysql and a place to write to.
-Its common use cases include ETL, cache building/expiring, metrics collection,
-search indexing and inter-service communication. Maxwell gives you some of the
-benefits of event sourcing without having to re-architect your entire platform.
-Download:
-[https://github.com/zendesk/maxwell/releases/download/v1.33.0/maxwell-1.33.0.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.33.0/maxwell-1.33.0.tar.gz)
-
-Source:
-[https://github.com/zendesk/maxwell](https://github.com/zendesk/maxwell)
-
+[↓ Download](https://github.com/zendesk/maxwell/releases/download/v1.42.3/maxwell-1.42.3.tar.gz) \|
+[⚝ Source / Community](https://github.com/zendesk/maxwell) \|
+[☝ Getting Started](/quickstart) \|
+[☷ Reference](/config)
+
+__What's it for?__
+
+- ETL of all sorts
+- maintaining an audit log of all changes to your database
+- cache building/expiring
+- search indexing
+- inter-service communication
-```
- mysql> insert into `test`.`maxwell` set id = 1, daemon = 'Stanislaw Lem';
- maxwell: {
- "database": "test",
- "table": "maxwell",
- "type": "insert",
- "ts": 1449786310,
- "xid": 940752,
- "commit": true,
- "data": { "id":1, "daemon": "Stanislaw Lem" }
- }
-```
+
+__It goes like this:__
```
- mysql> update test.maxwell set daemon = 'firebus! firebus!' where id = 1;
- maxwell: {
+ mysql> update `test`.`maxwell` set mycol = 55, daemon = 'Stanislaw Lem';
+ maxwell -> kafka:
+ {
"database": "test",
"table": "maxwell",
"type": "update",
- "ts": 1449786341,
- "xid": 940786,
- "commit": true,
- "data": {"id":1, "daemon": "Firebus! Firebus!"},
- "old": {"daemon": "Stanislaw Lem"}
+ "ts": 1449786310,
+ "data": { "id":1, "daemon": "Stanislaw Lem", "mycol": 55 },
+ "old": { "mycol":, 23, "daemon": "what once was" }
}
```
diff --git a/bin/maxwell b/bin/maxwell
index d12a1fea7..b1987b897 100755
--- a/bin/maxwell
+++ b/bin/maxwell
@@ -18,7 +18,7 @@ fi
CLASSPATH="$CLASSPATH:$lib_dir/*"
-KAFKA_VERSION="1.0.0"
+KAFKA_VERSION="2.7.0"
function use_kafka() {
wanted="$1"
@@ -86,7 +86,6 @@ if [ -z "$kafka_client_jar" -o "$(echo "$kafka_client_jar" | wc -l)" -gt 1 ]; th
ls -1 "$kafka_client_dir" | sed -e 's/^kafka-clients-/ - /' -e 's/\.jar$//'
exit 1
else
- echo "Using kafka version: $KAFKA_VERSION"
CLASSPATH="$CLASSPATH:$kafka_client_jar"
fi
diff --git a/build/Gemfile b/build/Gemfile
index 5f99ae3f9..cfa19ad8e 100644
--- a/build/Gemfile
+++ b/build/Gemfile
@@ -1,6 +1,6 @@
source 'https://rubygems.org'
-gem 'octokit'
+gem 'octokit', '>= 4.6.0'
gem 'netrc'
gem 'mime-types'
diff --git a/build/Gemfile.lock b/build/Gemfile.lock
index a475e5b79..ab9571ce1 100644
--- a/build/Gemfile.lock
+++ b/build/Gemfile.lock
@@ -1,19 +1,38 @@
GEM
remote: https://rubygems.org/
specs:
- addressable (2.3.8)
- faraday (0.9.2)
+ addressable (2.8.0)
+ public_suffix (>= 2.0.2, < 5.0)
+ faraday (1.5.0)
+ faraday-em_http (~> 1.0)
+ faraday-em_synchrony (~> 1.0)
+ faraday-excon (~> 1.1)
+ faraday-httpclient (~> 1.0.1)
+ faraday-net_http (~> 1.0)
+ faraday-net_http_persistent (~> 1.1)
+ faraday-patron (~> 1.0)
multipart-post (>= 1.2, < 3)
+ ruby2_keywords (>= 0.0.4)
+ faraday-em_http (1.0.0)
+ faraday-em_synchrony (1.0.0)
+ faraday-excon (1.1.0)
+ faraday-httpclient (1.0.1)
+ faraday-net_http (1.0.1)
+ faraday-net_http_persistent (1.1.0)
+ faraday-patron (1.0.0)
mime-types (3.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2016.0521)
- multipart-post (2.0.0)
+ multipart-post (2.1.1)
netrc (0.11.0)
- octokit (4.2.0)
- sawyer (~> 0.6.0, >= 0.5.3)
- sawyer (0.6.0)
- addressable (~> 2.3.5)
- faraday (~> 0.8, < 0.10)
+ octokit (4.21.0)
+ faraday (>= 0.9)
+ sawyer (~> 0.8.0, >= 0.5.3)
+ public_suffix (4.0.6)
+ ruby2_keywords (0.0.4)
+ sawyer (0.8.2)
+ addressable (>= 2.3.5)
+ faraday (> 0.8, < 2.0)
PLATFORMS
ruby
@@ -21,7 +40,7 @@ PLATFORMS
DEPENDENCIES
mime-types
netrc
- octokit
+ octokit (>= 4.6.0)
BUNDLED WITH
- 1.13.7
+ 1.17.3
diff --git a/build/mkchangelog b/build/mkchangelog
index 4eebc1cad..21815989b 100755
--- a/build/mkchangelog
+++ b/build/mkchangelog
@@ -16,13 +16,15 @@ while releases.any?
prerelease = !!(r[:tag_name] =~ /-\w+$/)
next if prerelease && r[:body].strip.empty?
out = <<-EOL
-### [%{tag_name}](%{html_url}): %{munged_name}
+### [%{tag_name}](%{html_url})
%{body}
+
+_Released %{date}_
EOL
r[:body].gsub!(/\r\n/, "\n")
- r[:munged_name] = r[:name].gsub(/^(\S+)\s+(.*)/, '\2')
+ r[:date] = r[:created_at].to_s.split(' ').first
puts out.strip % r
puts
end
diff --git a/config.properties.example b/config.properties.example
index a862236bc..1715ab0b2 100644
--- a/config.properties.example
+++ b/config.properties.example
@@ -11,7 +11,7 @@ password=maxwell
# *** general ***
-# choose where to produce data to. stdout|file|kafka|kinesis|pubsub|sqs|rabbitmq|redis
+# choose where to produce data to. stdout|file|kafka|kinesis|pubsub|sqs|rabbitmq|redis|bigquery
#producer=kafka
# set the log level. note that you can configure things further in log4j2.xml
@@ -218,6 +218,12 @@ kafka.acks=1
#pubsub_topic=maxwell
#ddl_pubsub_topic=maxwell_ddl
+# *** bigquery ***
+
+#bigquery_project_id=myproject
+#bigquery_dataset=mydataset
+#bigquery_table=mytable
+
# *** rabbit-mq ***
#rabbitmq_host=rabbitmq_hostname
@@ -225,6 +231,7 @@ kafka.acks=1
#rabbitmq_user=guest
#rabbitmq_pass=guest
#rabbitmq_virtual_host=/
+#rabbitmq_handshake_timeout=20000
#rabbitmq_exchange=maxwell
#rabbitmq_exchange_type=fanout
#rabbitmq_exchange_durable=false
@@ -232,6 +239,7 @@ kafka.acks=1
#rabbitmq_routing_key_template=%db%.%table%
#rabbitmq_message_persistent=false
#rabbitmq_declare_exchange=true
+#rabbitmq_use_ssl=false
# *** redis ***
@@ -277,6 +285,12 @@ kafka.acks=1
#
#filter= exclude: *.*, include: foo.*, include: bar.baz, include: foo.bar.col_eg = "value_to_match"
+
+# If you are running maxwell without permissions to view certain parts of your schema,
+# you may encounter "Coulndn't find database XXX" errors. In *only* those situations,
+# turn this option on and use filters to only include what you need.
+#ignore_missing_schema = false
+
# javascript filter
# maxwell can run a bit of javascript for each row if you need very custom filtering/data munging.
# See http://maxwells-daemon.io/filtering/#javascript_filters for more details
diff --git a/docs/docs/config.md b/docs/docs/config.md
index 5dcd47f3c..1b6ded4b1 100644
--- a/docs/docs/config.md
+++ b/docs/docs/config.md
@@ -1,8 +1,7 @@
# Reference
***
-At the minimum, you will need to specify 'host', 'user', 'password', 'producer'.
-The kafka producer requires 'kafka.bootstrap.servers', the kinesis producer requires 'kinesis_stream'.
+Configuration options are set either via command line or the "config.properties" file.
##general
@@ -53,7 +52,7 @@ option | argument | descripti
producer | [PRODUCER_TYPE](#producer_type) | type of producer to use | stdout
custom_producer.factory | CLASS_NAME | fully qualified custom producer factory class, see [example](https://github.com/zendesk/maxwell/blob/master/src/example/com/zendesk/maxwell/example/producerfactory/CustomProducerFactory.java) |
producer_ack_timeout | [PRODUCER_ACK_TIMEOUT](#ack_timeout) | time in milliseconds before async producers consider a message lost |
-producer_partition_by | [PARTITION_BY](#partition_by) | input to kafka/kinesis partition function | database
+producer_partition_by | [PARTITION_BY](#partition_by) | input to kafka/kinesis/sns/sqs partition function | database
producer_partition_columns | STRING | if partitioning by 'column', a comma separated list of columns |
producer_partition_by_fallback | [PARTITION_BY_FALLBACK](#partition_by_fallback) | required when producer_partition_by=column. Used when the column is missing |
ignore_producer_error | BOOLEAN | When false, Maxwell will terminate on kafka/kinesis/pubsub publish errors (aside from RecordTooLargeException). When true, errors are only logged. See also dead_letter_topic | true
@@ -78,30 +77,39 @@ kafka_partition_hash | [ default | murmur3 ] | hash func
kafka_key_format | [ array | hash ] | how maxwell outputs kafka keys, either a hash or an array of hashes | hash
ddl_kafka_topic | STRING | if output_ddl is true, kafka topic to write DDL changes to | *kafka_topic*
+_See also:_ [Kafka Producer Documentation](/producers#kafka)
+
## kinesis producer
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
kinesis_stream | STRING | kinesis stream name |
+_See also:_ [Kinesis Producer Documentation](/producers#kinesis)
## sqs producer
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
sqs_queue_uri | STRING | SQS Queue URI |
+_See also:_ [SQS Producer Documentation](/producers#sqs)
+
## sns producer
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
sns_topic | STRING | The SNS topic to publish to. FIFO topics should end with `.fifo` |
sns_attrs | STRING | Properties to set as attributes on the SNS message |
+_See also:_ [SNS Producer Documentation](/producers#sns)
+
## nats producer
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
nats_url | STRING | Comma separated list of nats urls. may include [user:password style auth](https://docs.nats.io/developing-with-nats/security/userpass#connecting-with-a-user-password-in-the-url) | nats://localhost:4222
nats_subject | STRING | Nats subject hierarchy. [Topic substitution](/producers/#topic-substitution) available. | `%{database}.%{table}`
+_See also:_ [Nats Producer Documentation](/producers#nats)
+
## pubsub producer
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
@@ -110,6 +118,7 @@ pubsub_platform_id | STRING | Google Cloud platform id associate
ddl_pubsub_topic | STRING | Google Cloud pub-sub topic to send DDL events to |
pubsub_request_bytes_threshold | LONG | Set number of bytes until batch is send | 1
pubsub_message_count_batch_size| LONG | Set number of messages until batch is send | 1
+pubsub_message_ordering_key | STRING | Google Cloud pub-sub ordering key template (also enables message ordering when set) |
pubsub_publish_delay_threshold | LONG | Set time passed in millis until batch is send | 1
pubsub_retry_delay | LONG | Controls the delay in millis before sending the first retry message | 100
pubsub_retry_delay_multiplier | FLOAT | Controls the increase in retry delay per retry | 1.3
@@ -118,6 +127,18 @@ pubsub_initial_rpc_timeout | LONG | Controls the timeout in seconds fo
pubsub_rpc_timeout_multiplier | FLOAT | Controls the change in RPC timeout | 1.0
pubsub_max_rpc_timeout | LONG | Puts a limit on the value in seconds of the RPC timeout | 600
pubsub_total_timeout | LONG | Puts a limit on the value in seconds of the retry delay, so that the RetryDelayMultiplier can't increase the retry delay higher than this amount | 600
+pubsub_emulator | STRING | Google Cloud pub-sub emulator host to send events to |
+
+_See also:_ [PubSub Producer Documentation](/producers#google-cloud-pubsub)
+
+## bigquery producer
+option | argument | description | default
+-------------------------------|-------------------------------------| --------------------------------------------------- | -------
+bigquery_project_id | STRING | Google Cloud bigquery project id |
+bigquery_dataset | STRING | Google Cloud bigquery dataset id |
+bigquery_table | STRING | Google Cloud bigquery table id |
+
+_See also:_ [PubSub Producer Documentation](/producers#google-cloud-bigquery)
## rabbitmq producer
option | argument | description | default
@@ -127,6 +148,7 @@ rabbitmq_pass | STRING | Password of Rabbitmq connection |
rabbitmq_host | STRING | Host of Rabbitmq machine
rabbitmq_port | INT | Port of Rabbitmq machine |
rabbitmq_virtual_host | STRING | Virtual Host of Rabbitmq |
+rabbitmq_handshake_timeout | STRING | Handshake timeout of Rabbitmq connection in milliseconds |
rabbitmq_exchange | STRING | Name of exchange for rabbitmq publisher |
rabbitmq_exchange_type | STRING | Exchange type for rabbitmq |
rabbitmq_exchange_durable | BOOLEAN | Exchange durability. | false
@@ -134,6 +156,9 @@ rabbitmq_exchange_autodelete | BOOLEAN | If set, the exchange is deleted wh
rabbitmq_routing_key_template | STRING | A string template for the routing key, `%db%` and `%table%` will be substituted. | `%db%.%table%`.
rabbitmq_message_persistent | BOOLEAN | Eanble message persistence. | false
rabbitmq_declare_exchange | BOOLEAN | Should declare the exchange for rabbitmq publisher | true
+rabbitmq_use_ssl | BOOLEAN | If true, will connect to the server using SSL. | false
+
+_See also:_ [RabbitMQ Producer Documentation](/producers#rabbitmq)
## redis producer
option | argument | description | default
@@ -148,6 +173,8 @@ redis_stream_json_key | STRING | Redis XADD Stream Me
redis_sentinels | STRING | Redis sentinels list in format host1:port1,host2:port2,host3:port3... Must be only used with redis_sentinel_master_name
redis_sentinel_master_name | STRING | Redis sentinel master name. Must be only used with redis_sentinels
+_See also:_ [Redis Producer Documentation](/producers#redis)
+
# formatting
option | argument | description | default
@@ -162,6 +189,7 @@ output_server_id | BOOLEAN | records include server_id
output_thread_id | BOOLEAN | records include thread_id | false
output_schema_id | BOOLEAN | records include schema_id, schema_id is the id of the latest schema tracked by maxwell and doesn't relate to any mysql tracked value | false
output_row_query | BOOLEAN | records include INSERT/UPDATE/DELETE statement. Mysql option "binlog_rows_query_log_events" must be enabled | false
+row_query_max_length | INT | The maximum number of characters output in the "query" field. The rest will be truncated.
output_primary_keys | BOOLEAN | DML records include list of values that make up a row's primary key | false
output_primary_key_columns | BOOLEAN | DML records include list of columns that make up a row's primary key | false
output_ddl | BOOLEAN | output DDL (table-alter, table-create, etc) events | false
@@ -173,6 +201,8 @@ option | argument | descripti
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
filter | STRING | filter rules, eg `exclude: db.*, include: *.tbl, include: *./bar(bar)?/, exclude: foo.bar.col=val` |
+_See also:_ [filtering](/filtering)
+
# encryption
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
@@ -186,6 +216,8 @@ ha | | enable ma
jgroups_config | string | location of xml configuration file for jGroups | $PWD/raft.xml
raft_member_id | string | uniquely identify this node within jgroups-raft cluster |
+_See also:_ [High Availability](/high_availability)
+
# monitoring / metrics
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
@@ -206,15 +238,19 @@ metrics_datadog_apikey | STRING | the datadog api key to use when metrics_data
metrics_datadog_site | STRING | the site to publish metrics to when metrics_datadog_type = `http` | us
metrics_datadog_host | STRING | the host to publish metrics to when metrics_datadog_type = `udp` | localhost
metrics_datadog_port | INT | the port to publish metrics to when metrics_datadog_type = `udp` | 8125
+custom_health.factory | CLASS_NAME | fully qualified maxwell health check factory class, see [example](https://github.com/zendesk/maxwell/blob/master/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheckFactory.java) |
+
+_See also:_ [Monitoring](/monitoring)
# misc
option | argument | description | default
-------------------------------|-------------------------------------| --------------------------------------------------- | -------
-bootstrapper | [async | sync | none] | bootstrapper type. See bootstrapping docs. | async
-init_position | FILE:POSITION[:HEARTBEAT] | ignore the information in maxwell.positions and start at the given binlog position. Not available in config.properties. |
+bootstrapper | [async | sync | none] | bootstrapper type. See [bootstrapping docs](/bootstrapping). | async
+init_position | FILE:POSITION[:HEARTBEAT] | ignore the information in maxwell.positions and start at the given binlog position. Not available in config.properties. [see note](/deployment#-init_position)|
replay | BOOLEAN | enable maxwell's read-only "replay" mode: don't store a binlog position or schema changes. Not available in config.properties. |
buffer_memory_usage | FLOAT | Determines how much memory the Maxwell event buffer will use from the jvm max memory. Size of the buffer is: buffer_memory_usage * -Xmx" | 0.25
http_config | BOOLEAN | enable http config endpoint for config updates without restart | false
+binlog_event_queue_size | INT | Size of queue to buffer events parsed from binlog | 5000
@@ -289,79 +325,4 @@ A get request will return the live config state
}
```
-### Deployment scenarios
-***
-
-At a minimum, Maxwell needs row-level-replication turned on into order to
-operate:
-
-```
-[mysqld]
-server_id=1
-log-bin=master
-binlog_format=row
-```
-
-#### GTID support
-As of 1.8.0, Maxwell contains support for
-[GTID-based replication](https://dev.mysql.com/doc/refman/5.6/en/replication-gtids.html).
-Enable it with the `--gtid_mode` configuration param.
-
-Here's how you might configure your mysql server for GTID mode:
-
-```
-$ vi my.cnf
-
-[mysqld]
-server_id=1
-log-bin=master
-binlog_format=row
-gtid-mode=ON
-log-slave-updates=ON
-enforce-gtid-consistency=true
-```
-
-When in GTID-mode, Maxwell will transparently pick up a new replication
-position after a master change. Note that you will still have to re-point
-maxwell to the new master.
-
-GTID support in Maxwell is considered beta-quality at the moment; notably,
-Maxwell is unable to transparently upgrade from a traditional-replication
-scenario to a GTID-replication scenario; currently, when you enable gtid mode
-Maxwell will recapture the schema and GTID-position from "wherever the master
-is at".
-
-
-#### RDS configuration
-To run Maxwell against RDS, (either Aurora or Mysql) you will need to do the following:
-
-- set binlog_format to "ROW". Do this in the "parameter groups" section. For a Mysql-RDS instance this parameter will be
- in a "DB Parameter Group", for Aurora it will be in a "DB Cluster Parameter Group".
-- setup RDS binlog retention as described [here](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.Concepts.MySQL.html).
- The tl;dr is to execute `call mysql.rds_set_configuration('binlog retention hours', 24)` on the server.
-
-#### Split server roles
-
-Maxwell uses MySQL for 3 different functions:
-
-1. A host to store the captured schema in (`--host`).
-2. A host to replicate from (`--replication_host`).
-3. A host to capture the schema from (`--schema_host`).
-
-Often, all three hosts are the same. `host` and `replication_host` should be different
-if maxwell is chained off a replica. `schema_host` should only be used when using the
-maxscale replication proxy.
-
-#### Multiple Maxwell Instances
-
-Maxwell can operate with multiple instances running against a single master, in
-different configurations. This can be useful if you wish to have producers
-running in different configurations, for example producing different groups of
-tables to different topics. Each instance of Maxwell must be configured with a
-unique `client_id`, in order to store unique binlog positions.
-
-With MySQL 5.5 and below, each replicator (be it mysql, maxwell, whatever) must
-also be configured with a unique `replica_server_id`. This is a 32-bit integer
-that corresponds to mysql's `server_id` parameter. The value you configure
-should be unique across all mysql and maxwell instances.
diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md
new file mode 100644
index 000000000..ebc622111
--- /dev/null
+++ b/docs/docs/deployment.md
@@ -0,0 +1,88 @@
+# Runtime reconfiguration
+
+If you've already got binlogs enabled and don't want to restart your mysql to
+configure maxwell, try this:
+
+```
+mysql> set global binlog_format=ROW;
+mysql> set global binlog_row_image=FULL;
+```
+
+note: `binlog_format` is a session-based property. You will need to shutdown all active connections to fully convert
+to row-based replication.
+
+
+# GTID support
+Maxwell contains support for
+[GTID-based replication](https://dev.mysql.com/doc/refman/5.6/en/replication-gtids.html).
+Enable it with the `--gtid_mode` configuration param.
+
+Here's how you might configure your mysql server for GTID mode:
+
+```
+$ vi my.cnf
+
+[mysqld]
+server_id=1
+log-bin=master
+binlog_format=row
+gtid-mode=ON
+log-slave-updates=ON
+enforce-gtid-consistency=true
+```
+
+When in GTID-mode, Maxwell will transparently pick up a new replication
+position after a master change. Note that you will still have to re-point
+maxwell to the new master (or use a floating VIP)
+
+
+# RDS
+To run Maxwell against RDS, (either Aurora or Mysql) you will need to do the following:
+
+- set binlog_format to "ROW". Do this in the "parameter groups" section. For a Mysql-RDS instance this parameter will be
+ in a "DB Parameter Group", for Aurora it will be in a "DB Cluster Parameter Group".
+- setup RDS binlog retention as described [here](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.Concepts.MySQL.html).
+ The tl;dr is to execute `call mysql.rds_set_configuration('binlog retention hours', 24)` on the server.
+
+# Replicating and storing schema from different servers
+
+Maxwell uses MySQL for 3 different functions:
+
+1. A host to store the captured schema in (`--host`).
+2. A host to replicate binlogs from (`--replication_host`).
+3. A host to capture the schema from (`--schema_host`).
+
+Often, all three hosts are the same. `host` and `replication_host` should be different
+if maxwell is chained off a replica. `schema_host` should only be used when using the
+maxscale replication proxy.
+
+# Multiple Maxwells
+
+Maxwell can operate with multiple instances running against a single master, in
+different configurations. This can be useful if you wish to have producers
+running in different configurations, for example producing different groups of
+tables to different topics. Each instance of Maxwell must be configured with a
+unique `client_id`, in order to store unique binlog positions.
+
+Each version of Maxwell must also be configured with a unique
+`replica_server_id`. This is a 32-bit integer that corresponds to mysql's
+`server_id` parameter. The value should be unique across all maxwell instances
+and also be unique from any mysql `server_id` values.
+
+# `--init_position`
+
+This is a dangerous option that you really shouldn't use unless you know what
+you're doing. It allows you to "rewind" history and go back to a certain point
+in the binlog. This can work, but you should be aware that Maxwell must have
+already "visited" that binlog position; simply specifying an arbitrary position
+in the binlog will lead to Maxwell crashing.
+
+
+# Running with limited permissions
+
+If the user you're running maxwell as can't view part of the database because of limited
+permissions, Maxwell may be unable to capture information on part of the schem a and
+the replication stream can break with "Can't find table: XXX" errors. In this case
+you can enable the `ignore_missing_schema` flag *and* configure a filter that will exclude
+any databases/tables you don't have permission to view.
+
diff --git a/docs/docs/embedding.md b/docs/docs/embedding.md
index e47be58d0..38b1705b5 100644
--- a/docs/docs/embedding.md
+++ b/docs/docs/embedding.md
@@ -1,9 +1,10 @@
# Embedding Maxwell
***
-Maxwell typically runs as a command-line program. However, for advanced use it
-is possible to run maxwell from any JVM-based language. Currently the source of
-truth is the source code (there is no published API documentation). Pull requests
-to better document embedded Maxwell uses are welcome.
+Maxwell typically runs as a command-line program. However, for advanced uses it
+is possible to run maxwell from any JVM-based language.
+
+Some fairly incomplete API documentation is available here:
+[https://maxwells-daemon.io/apidocs](https://maxwells-daemon.io/apidocs)
# Compatibility caveat
***
diff --git a/docs/docs/filtering.md b/docs/docs/filtering.md
index 49e7dd9f0..9edcbe53f 100644
--- a/docs/docs/filtering.md
+++ b/docs/docs/filtering.md
@@ -56,10 +56,26 @@ Also note that this is the feature I most regret writing.
If you need more flexibility than the native filters provide, you can write a small chunk of
javascript for Maxwell to pass each row through with `--javascript FILE`. This file should contain
at least a javascript function named `process_row`. This function will be passed a [`WrappedRowMap`]()
-object and is free to make filtering and data munging decisions:
+object that represents the current row and a [`LinkedHashMap`]() which represents a global state and is free to make filtering and data munging decisions:
```
-function process_row(row) {
+function process_row(row, state) {
+ // Updating the state object
+ if ( row.database == "test" && row.table == "lock") {
+ var haslock = row.data.get("haslock");
+ if ( haslock == "false" ) {
+ state.put("haslock", "false");
+ } else if( haslock == "true" ) {
+ state.put("haslock", "true");
+ }
+ }
+
+ // Suppressing rows based on state
+ if(state.get("haslock") == "true") {
+ row.suppress();
+ }
+
+ // Filter and Change based on actual data
if ( row.database == "test" && row.table == "bar" ) {
var username = row.data.get("username");
if ( username == "osheroff" )
diff --git a/docs/docs/monitoring.md b/docs/docs/monitoring.md
index 3e5f86012..1d4c03fd3 100644
--- a/docs/docs/monitoring.md
+++ b/docs/docs/monitoring.md
@@ -18,12 +18,12 @@ metric | description
`messages.failed.meter` | a measure of the rate at which messages failed to send Kafka
`row.meter` | a measure of the rate at which rows arrive to Maxwell from the binlog connector
**Gauges**
-`replication.lag` | the time elapsed between the database transaction commit and the time it was processed by Maxwell, in milliseconds
+`replication.lag` | the time elapsed between the database transaction commit and the time it was processed by Maxwell
`inflightmessages.count` | the number of messages that are currently in-flight (awaiting acknowledgement from the destination, or ahead of messages which are)
**Timers**
-`message.publish.time` | the time it took to send a given record to Kafka, in milliseconds
-`message.publish.age` | the time between an event occurring on the DB and being published to kafka, in milliseconds. Note: since MySQL timestamps are accurate to the second, this is only accurate to +/- 500ms.
-`replication.queue.time` | the time it took to enqueue a given binlog event for processing, in milliseconds
+`message.publish.time` | the time it took to send a given record to Kafka
+`message.publish.age` | the time between an event occurring on the DB and being published to kafka. Note: since MySQL timestamps are accurate to the second, this is only accurate to +/- 500ms.
+`replication.queue.time` | the time it took to enqueue a given binlog event for processing
# HTTP Endpoints
***
@@ -31,11 +31,18 @@ When the HTTP server is enabled the following endpoints are exposed:
| endpoint | description |
|:---------------|:-------------------------------------------------------------------------------|
-| `/metrics` | return all metrics as JSON |
-| `/prometheus` | return all metrics as Prometheus format |
+| `/metrics` | GET all metrics as JSON |
+| `/prometheus` | GET all metrics as Prometheus format |
| `/healthcheck` | run Maxwell's healthchecks. Considered unhealthy if >0 messages have failed in the last 15 minutes. |
| `/ping` | a simple ping test, responds with `pong` |
+| `/diagnostics` | for kafka, send a fake message that measures the client to server latency |
+## Custom Health Check
+Similar to the custom producer, developers can provide their own implementation of a health check.
+
+In order to register your custom health check, you must implement the `MaxwellHealthCheckFactory` interface, which is responsible for creating your custom `MaxwellHealthCheck`. Next, set the `custom_health.factory` configuration property to your `MaxwellHealthCheckFactory`'s fully qualified class name. Then add the custom `MaxwellHealthCheckFactory` JAR and all its dependencies to the $MAXWELL_HOME/lib directory.
+
+Custom health check factory and health check examples can be found here: [https://github.com/zendesk/maxwell/tree/master/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory](https://github.com/zendesk/maxwell/tree/master/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory)
# JMX Configuration
***
diff --git a/docs/docs/producers.md b/docs/docs/producers.md
index e1720be88..5fcbb735b 100644
--- a/docs/docs/producers.md
+++ b/docs/docs/producers.md
@@ -1,9 +1,6 @@
# Kafka
***
-The Kafka producer is perhaps the most production hardened of all the producers,
-having run on high traffic instances at WEB scale.
-
## Topic
Maxwell writes to a kafka topic named "maxwell" by default. It is configurable
via `--kafka_topic`. The given topic can be a plain string or a dynamic
@@ -36,7 +33,9 @@ the "new producer" configuration, as described here:
[http://kafka.apache.org/documentation.html#newproducerconfigs](http://kafka.apache.org/documentation.html#newproducerconfigs)
-## Highest throughput
+## Example kafka configs
+
+### Highest throughput
These properties would give high throughput performance.
@@ -46,7 +45,7 @@ kafka.compression.type = snappy
kafka.retries=0
```
-## Most reliable
+### Most reliable
For at-least-once delivery, you will want something more like:
@@ -57,7 +56,7 @@ kafka.retries = 5 # or some larger number
And you will also want to set `min.insync.replicas` on Maxwell's output topic.
-## Keys
+## Key format
Maxwell generates keys for its Kafka messages based upon a mysql row's primary key in JSON format:
@@ -72,10 +71,10 @@ as a source of truth.
# Partitioning
***
-Both Kafka and AWS Kinesis support the notion of partitioned streams.
+Kafka, AWS Kinesis/SNS/SQS support the notion of partitioned streams.
Because they like to make our lives hard, Kafka calls its two units "topics"
-and "partitions", and Kinesis calls them "streams" and "shards. They're the
-same thing, though. Maxwell is generally configured to write to N
+and "partitions", Kinesis calls them "streams" and "shards", and SNS/SQS calls them "group id".
+They're the same thing, though. Maxwell is generally configured to write to N
partitions/shards on one topic/stream, and how it distributes to those N
partitions/shards can be controlled by `producer_partition_by`.
@@ -112,7 +111,7 @@ your updates), you must set both:
When partitioning by column Maxwell will treat the values for the specified
columns as strings, concatenate them and use that value to partition the data.
-## Kafka partitioning
+### Kafka partitioning
A binlog event's partition is determined by the selected hash function and hash string as follows
@@ -216,7 +215,12 @@ See the [AWS docs](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/cr
In case you need to set up a different region also along with credentials then default one, see the [AWS docs](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/setup-credentials.html#setup-credentials-setting-region).
## Options
-Set the output queue in the `config.properties` by setting the `sqs_queue_uri` property to full SQS queue uri from AWS console.
+Set the output queue in the `config.properties` by setting the following properites
+
+- **sqs_signing_region**: the region to use for SigV4 signing of requests. e.g. `us-east-1`
+- **sqs_service_endpoint**: the service endpoint either with or without the protocol (e.g. `https://sns.us-west-1.amazonaws.com` or `sns.us-west-1.amazonaws.com`)
+- **sqs_queue_uri**: the full SQS queue uri from AWS console. e.g. `https://sqs.us-east-1.amazonaws.com/xxxxxxxxxxxx/maxwell`
+
The producer uses the [AWS SQS SDK](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/sqs/AmazonSQSClient.html).
@@ -258,6 +262,34 @@ for DDL updates by setting the `ddl_pubsub_topic` property.
The producer uses the [Google Cloud Java Library for Pub/Sub](https://github.com/GoogleCloudPlatform/google-cloud-java/tree/master/google-cloud-pubsub) and uses its built-in configurations.
+# Google Cloud BigQuery
+***
+To stream data into Google Cloud Bigquery, first there must be a table created on bigquery in order to stream the data
+into defined as `bigquery_project_id.bigquery_dataset.bigquery_table`. The schema of the table must match the outputConfig. The column types should be defined as below
+
+- database: string
+- table: string
+- type: string
+- ts: integer
+- xid: integer
+- xoffset: integer
+- commit: boolean
+- position: string
+- gtid: string
+- server_id: integer
+- primary_key: string
+- data: string
+- old: string
+
+See the Google Cloud Platform docs for the [latest examples of which permissions are needed](https://cloud.google.com/bigquery/docs/access-control), as well as [how to properly configure service accounts](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances).
+
+Set the output stream in `config.properties` by setting the `bigquery_project_id`, `bigquery_dataset` and `bigquery_table` properties.
+
+The producer uses the [Google Cloud Java Bigquery Storage Library for Bigquery](https://github.com/googleapis/java-bigquerystorage) [Bigquery Storage Write API documenatation](https://cloud.google.com/bigquery/docs/write-api).
+To use the Storage Write API, you must have `bigquery.tables.updateData` permissions.
+
+This producer is using the Default Stream with at-least once semantics for greater data resiliency and fewer scaling restrictions
+
# RabbitMQ
***
To produce messages to RabbitMQ, you will need to specify a host in `config.properties` with `rabbitmq_host`. This is the only required property, everything else falls back to a sane default.
@@ -267,6 +299,7 @@ The remaining configurable properties are:
- `rabbitmq_user` - defaults to **guest**
- `rabbitmq_pass` - defaults to **guest**
- `rabbitmq_virtual_host` - defaults to **/**
+- `rabbitmq_handshake_timeout` - defaults to **10000**
- `rabbitmq_exchange` - defaults to **maxwell**
- `rabbitmq_exchange_type` - defaults to **fanout**
- `rabbitmq_exchange_durable` - defaults to **false**
@@ -275,20 +308,17 @@ The remaining configurable properties are:
- This config controls the routing key, where `%db%` and `%table%` are placeholders that will be substituted at runtime
- `rabbitmq_message_persistent` - defaults to **false**
- `rabbitmq_declare_exchange` - defaults to **true**
+- `rabbitmq_use_ssl` - defaults to **false**
For more details on these options, you are encouraged to the read official RabbitMQ documentation here: [https://www.rabbitmq.com/documentation.html](https://www.rabbitmq.com/documentation.html)
# Redis
***
-Set the output stream in `config.properties` by setting the `redis_type`
-property to either `pubsub`, `xadd`, `lpush` or `rpsuh`. The `redis_key` is
-used as a channel for `pubsub`, as stream key for `xadd` and as key for `lpush`
-and `rpush`.
-Maxwell writes to a Redis channel named "maxwell" by default. It can be static,
-e.g. 'maxwell', or dynamic, e.g. `namespace_%{database}_%{table}`. In the
-latter case 'database' and 'table' will be replaced with the values for the row
-being processed. This can be changed with the `redis_pub_channel`, `redis_list_key` and `redis_stream_key` option.
+Choose type of redis data structure to create to by setting `redis_type` to one of:
+`pubsub`, `xadd`, `lpush` or `rpush`. The default is `pubsub`.
+
+`redis_key` defaults to "maxwell" and supports [topic substitution](#topic-substitution)
Other configurable properties are:
@@ -304,11 +334,11 @@ Other configurable properties are:
# Custom Producer
***
-If none of the producers packaged with Maxwell meet your requirements, a custom producer can be added at runtime. The producer is responsible for processing the raw database rows. Note that your producer may receive DDL and heartbeat rows as well, but your producer can easily filter them out (see example).
+If none of the producers packaged with Maxwell meet your requirements, a custom producer can be added at runtime.
-In order to register your custom producer, you must implement the `ProducerFactory` interface, which is responsible for creating your custom `AbstractProducer`. Next, set the `custom_producer.factory` configuration property to your `ProducerFactory`'s fully qualified class name. Then add the custom `ProducerFactory` and all its dependencies to the $MAXWELL_HOME/lib directory.
+In order to register your custom producer, you must implement the `ProducerFactory` interface, which is responsible for creating your custom `AbstractProducer`. Next, set the `custom_producer.factory` configuration property to your `ProducerFactory`'s fully qualified class name. Then add the custom `ProducerFactory` JAR and all its dependencies to the $MAXWELL_HOME/lib directory.
-Your custom producer will likely require configuration properties as well. For that, use the `custom_producer.*` property namespace. Those properties will be exposed to your producer via `MaxwellConfig.customProducerProperties`.
+Your custom producer will likely require configuration properties as well. For that, use the `custom_producer.*` (or `CUSTOM_PRODUCER_*` if using env-variable configuration) property namespace. Those properties will be available to your producer via `MaxwellConfig.customProducerProperties`.
Custom producer factory and producer examples can be found here: [https://github.com/zendesk/maxwell/tree/master/src/example/com/zendesk/maxwell/example/producerfactory](https://github.com/zendesk/maxwell/tree/master/src/example/com/zendesk/maxwell/example/producerfactory)
diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md
index 255dff328..180099750 100644
--- a/docs/docs/quickstart.md
+++ b/docs/docs/quickstart.md
@@ -1,13 +1,13 @@
# Download
***
-- Download binary distro: [https://github.com/zendesk/maxwell/releases/download/v1.33.0/maxwell-1.33.0.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.33.0/maxwell-1.33.0.tar.gz)
+- Download binary distro: [https://github.com/zendesk/maxwell/releases/download/v1.42.3/maxwell-1.42.3.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.42.3/maxwell-1.42.3.tar.gz)
- Sources and bug tracking is available on github: [https://github.com/zendesk/maxwell](https://github.com/zendesk/maxwell)
**curl**:
```
-curl -sLo - https://github.com/zendesk/maxwell/releases/download/v1.33.0/maxwell-1.33.0.tar.gz \
+curl -sLo - https://github.com/zendesk/maxwell/releases/download/v1.42.3/maxwell-1.42.3.tar.gz \
| tar zxvf -
-cd maxwell-1.33.0
+cd maxwell-1.42.3
```
**docker**:
@@ -25,38 +25,26 @@ brew install maxwell
# Configure Mysql
***
-*Server Config:* Ensure server_id is set, and that row-based replication is on.
-
```
-$ vi my.cnf
+
+# /etc/my.cnf
[mysqld]
-server_id=1
-log-bin=master
+# maxwell needs binlog_format=row
binlog_format=row
+server_id=1
+log-bin=master
```
-Or on a running server:
-
-```
-mysql> set global binlog_format=ROW;
-mysql> set global binlog_row_image=FULL;
-```
-
-note: `binlog_format` is a session-based property. You will need to shutdown all active connections to fully convert
-to row-based replication.
-
-*Permissions:* Maxwell needs permissions to act as a replica, and to write to the `maxwell` database.
```
mysql> CREATE USER 'maxwell'@'%' IDENTIFIED BY 'XXXXXX';
-mysql> GRANT ALL ON maxwell.* TO 'maxwell'@'%';
-mysql> GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'maxwell'@'%';
-
-# or for running maxwell locally:
-
mysql> CREATE USER 'maxwell'@'localhost' IDENTIFIED BY 'XXXXXX';
+
+mysql> GRANT ALL ON maxwell.* TO 'maxwell'@'%';
mysql> GRANT ALL ON maxwell.* TO 'maxwell'@'localhost';
+
+mysql> GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'maxwell'@'%';
mysql> GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'maxwell'@'localhost';
```
@@ -112,6 +100,15 @@ bin/maxwell --user='maxwell' --password='XXXXXX' --host='127.0.0.1' \
--pubsub_topic='maxwell'
```
+## Google Cloud Bigquery
+
+```
+bin/maxwell --user='maxwell' --password='XXXXXX' --host='127.0.0.1' \
+ --producer=bigquery --bigquery_project_id='$BIGQUERY_PROJECT_ID' \
+ --bigquery_dataset='$BIGQUERY_DATASET' \
+ --bigquery_table='$BIGQUERY_TABLE'
+```
+
## RabbitMQ
```
diff --git a/docs/maxwell_theme/main.html b/docs/maxwell_theme/main.html
index 11fceb3d3..2add34f14 100644
--- a/docs/maxwell_theme/main.html
+++ b/docs/maxwell_theme/main.html
@@ -1,7 +1,7 @@
{% extends "base.html" %}
{%- block content %}
-
{% include "toc.html" %}
-
+
{% include "toc.html" %}
+
{% include "content.html" %}
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index fbd993139..b72c07862 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -6,8 +6,9 @@ extra_css:
- maxwell.css
nav:
- 'Quick Start': 'quickstart.md'
- - 'Configuration':
+ - 'Documentation':
- 'Reference': 'config.md'
+ - 'Deployment': 'deployment.md'
- 'Producers': 'producers.md'
- 'Filtering': 'filtering.md'
- 'Bootstrapping': 'bootstrapping.md'
diff --git a/docs/push b/docs/push
index a14cf02cc..6723462e6 100755
--- a/docs/push
+++ b/docs/push
@@ -1,7 +1,10 @@
#!/bin/bash
set -eux
+make docs
+
cd $(dirname $0)
./build
+cp -a ../target/site/apidocs site/
(cd site && git add . && git commit -m 'update docs' && git push origin HEAD:gh-pages)
./clean
diff --git a/kinesis-producer-library.properties.example b/kinesis-producer-library.properties.example
index 8bc3f36c2..a3c47968d 100644
--- a/kinesis-producer-library.properties.example
+++ b/kinesis-producer-library.properties.example
@@ -258,7 +258,7 @@ RecordMaxBufferedTime = 100
# Default: 30000
# Minimum: 100
# Maximum (inclusive): 9223372036854775807
-RecordTtl = 30000
+RecordTtl = 3600000
# Which region to send records to.
#
diff --git a/pom.xml b/pom.xml
index b140046dd..b6d18b05e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,10 +1,10 @@
-
+
+4.0.0com.zendeskmaxwell
- 1.33.0
+ 1.42.3jarmaxwell
@@ -41,8 +41,12 @@
UTF-8
- 3.7.7
+ 3.12.40.28.3
+ 1.12.537
+ 2.15.2
+ 1.8
+ 1.8
@@ -52,7 +56,7 @@
org.apache.kafkakafka-clients
- 0.8.2.2
+ 3.7.2
@@ -62,7 +66,7 @@
org.apache.kafkakafka-clients
- 0.9.0.1
+ 3.7.2
@@ -72,7 +76,7 @@
org.apache.kafkakafka-clients
- 0.10.0.1
+ 3.7.2
@@ -82,7 +86,7 @@
org.apache.kafkakafka-clients
- 0.10.2.1
+ 3.7.2
@@ -92,7 +96,7 @@
org.apache.kafkakafka-clients
- 0.11.0.1
+ 3.7.2
@@ -105,7 +109,7 @@
org.apache.kafkakafka-clients
- 1.0.0
+ 3.7.2
@@ -118,13 +122,32 @@
org.apache.kafkakafka-clients
- 2.7.0
+ 3.7.2
+
+
+
+
+ kafka-3.4.0
+
+ true
+
+
+
+ org.apache.kafka
+ kafka-clients
+ 3.7.2
+
+
+ com.zendesk
+ mysql-binlog-connector-java
+ 0.30.1
+ com.mchangec3p0
@@ -153,7 +176,7 @@
mysqlmysql-connector-java
- 8.0.17
+ 8.0.28org.apache.commons
@@ -171,39 +194,41 @@
4.8-1
- org.hamcrest
- hamcrest-all
- 1.3
- test
+ net.sf.jopt-simple
+ jopt-simple
+ 5.0.4
- junit
- junit
- 4.13.1
- test
+ com.fasterxml.jackson.core
+ jackson-core
+ ${jackson.version}
- org.mockito
- mockito-core
- ${mockito.version}
- test
+ com.fasterxml.jackson.core
+ jackson-databind
+ ${jackson.version}
- org.mockito
- mockito-inline
- ${mockito.version}
- test
+ com.fasterxml.jackson.core
+ jackson-annotations
+ ${jackson.version}
- com.github.stefanbirkner
- system-rules
- 1.18.0
- test
+ org.jgroups
+ jgroups-raft
+ 1.0.0.Final
+
+ org.openjdk.nashorn
+ nashorn-core
+ 15.3
+
+
+
org.apache.logging.log4jlog4j-core
- 2.13.2
+ 2.17.1org.apache.logging.log4j
@@ -215,35 +240,20 @@
jul-to-slf4j1.7.30
-
- com.zendesk
- mysql-binlog-connector-java
- 0.24.0
-
-
- net.sf.jopt-simple
- jopt-simple
- 5.0.4
-
-
- net.snaq
- dbpool
- 7.0-jdk7
- com.djdch.log4jlog4j-staticshutdown1.1.0
- com.fasterxml.jackson.core
- jackson-core
- 2.11.2
+ org.apache.commons
+ commons-lang3
+ 3.11com.fasterxml.jackson.corejackson-databind
- 2.11.2
+ 2.12.1com.vividsolutions
@@ -251,34 +261,76 @@
1.13
- net.jpountz.lz4
- lz4
- 1.3.0
+ org.lz4
+ lz4-java
+ 1.8.0
+
+
+ me.tongfei
+ progressbar
+ 0.6.0
+
+
+
+
+ com.google.cloud
+ google-cloud-bigquerystorage
+ 2.14.2
+
+
+ com.google.cloud
+ google-cloud-bigquery
+ 2.13.3
+
+
+ com.amazonaws
+ aws-java-sdk-core
+ ${aws-java.version}
+
+
+ com.amazonaws
+ aws-java-sdk-sns
+ ${aws-java.version}
+
+
+ com.amazonaws
+ aws-java-sdk-sqs
+ ${aws-java.version}
+
+
+ com.amazonaws
+ aws-java-sdk-sts
+ ${aws-java.version}
+
+
+ com.amazonaws
+ amazon-kinesis-producer
+ 0.15.7
+
+
+ javax.xml.bind
+ jaxb-api
+ 2.3.0com.rabbitmqamqp-client
- 5.7.3
+ 5.18.0com.google.cloudgoogle-cloud-pubsub
- 1.108.6
+ 1.120.24io.natsjnats2.8.0
-
- com.google.guava
- guava
- 30.1.1-jre
- com.google.protobufprotobuf-java
- 3.13.0
+ 3.21.7io.dropwizard.metrics
@@ -290,10 +342,17 @@
jedis3.5.1
+
+
+
+ io.dropwizard.metrics
+ metrics-core
+ 4.1.17
+ io.dropwizard.metricsmetrics-servlets
- 4.1.14
+ 4.1.17io.dropwizard.metrics
@@ -303,12 +362,12 @@
org.eclipse.jettyjetty-server
- 9.4.37.v20210219
+ 10.0.14org.eclipse.jettyjetty-servlet
- 9.4.37.v20210219
+ 10.0.12com.viafoura
@@ -345,16 +404,10 @@
simpleclient_servlet0.9.0
-
-
- org.jgroups
- jgroups-raft
- 1.0.0.Final
- io.opencensus
- opencensus-contrib-dropwizard
+ opencensus-contrib-dropwizard${opencensus.version}
@@ -373,6 +426,39 @@
opencensus-exporter-stats-stackdriver${opencensus.version}
+
+
+
+ org.hamcrest
+ hamcrest-all
+ 1.3
+ test
+
+
+ junit
+ junit
+ 4.13.1
+ test
+
+
+ org.mockito
+ mockito-core
+ ${mockito.version}
+ test
+
+
+ org.mockito
+ mockito-inline
+ ${mockito.version}
+ test
+
+
+ com.github.stefanbirkner
+ system-rules
+ 1.18.0
+ test
+
+
@@ -387,6 +473,16 @@
ossrhhttps://oss.sonatype.org/true
+ 10
+ true
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ --add-opens java.base/java.util=ALL-UNNAMED
@@ -419,7 +515,7 @@
org.apache.maven.pluginsmaven-javadoc-plugin
- 3.2.0
+ 3.3.111
diff --git a/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheck.java b/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheck.java
new file mode 100644
index 000000000..7daa7ee8f
--- /dev/null
+++ b/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheck.java
@@ -0,0 +1,15 @@
+package com.zendesk.maxwell.example.maxwellhealthcheckfactory;
+
+import com.zendesk.maxwell.monitoring.MaxwellHealthCheck;
+import com.zendesk.maxwell.producer.AbstractProducer;
+
+public class CustomMaxwellHealthCheck extends MaxwellHealthCheck {
+ public CustomMaxwellHealthCheck(AbstractProducer producer) {
+ super(producer);
+ }
+
+ @Override
+ protected Result check() throws Exception {
+ return Result.unhealthy("I am always unhealthy");
+ }
+}
diff --git a/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheckFactory.java b/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheckFactory.java
new file mode 100644
index 000000000..f085f5a76
--- /dev/null
+++ b/src/example/com/zendesk/maxwell/example/maxwellhealthcheckfactory/CustomMaxwellHealthCheckFactory.java
@@ -0,0 +1,14 @@
+package com.zendesk.maxwell.example.maxwellhealthcheckfactory;
+
+import com.zendesk.maxwell.monitoring.MaxwellHealthCheck;
+import com.zendesk.maxwell.monitoring.MaxwellHealthCheckFactory;
+import com.zendesk.maxwell.producer.AbstractProducer;
+
+public class CustomMaxwellHealthCheckFactory implements MaxwellHealthCheckFactory
+{
+ @Override
+ public MaxwellHealthCheck createHealthCheck(AbstractProducer producer)
+ {
+ return new CustomMaxwellHealthCheck(producer);
+ }
+}
diff --git a/src/main/antlr4/imports/column_definitions.g4 b/src/main/antlr4/imports/column_definitions.g4
index 0275db82d..e1860be4f 100644
--- a/src/main/antlr4/imports/column_definitions.g4
+++ b/src/main/antlr4/imports/column_definitions.g4
@@ -81,6 +81,7 @@ column_options:
| collation
| default_value
| primary_key
+ | visibility
| ON UPDATE ( CURRENT_TIMESTAMP current_timestamp_length? | now_function )
| UNIQUE KEY?
| KEY
@@ -89,7 +90,7 @@ column_options:
| COMMENT string_literal
| COLUMN_FORMAT (FIXED|DYNAMIC|COMPRESSED|DEFAULT)
| STORAGE (DISK|MEMORY|DEFAULT)
- | (VIRTUAL | STORED)
+ | (VIRTUAL | PERSISTENT | STORED)
| (GENERATED ALWAYS)? AS skip_parens
| reference_definition
| CHECK skip_parens
@@ -103,6 +104,7 @@ enum_value: string_literal;
charset_def: character_set | ASCII;
character_set: ((CHARACTER SET) | CHARSET) charset_name;
+visibility: VISIBLE | INVISIBLE;
nullability: (NOT NULL | NULL);
default_value: DEFAULT
diff --git a/src/main/antlr4/imports/generate_tokens.rb b/src/main/antlr4/imports/generate_tokens.rb
index efa0e0ac9..e0f2e4453 100755
--- a/src/main/antlr4/imports/generate_tokens.rb
+++ b/src/main/antlr4/imports/generate_tokens.rb
@@ -26,6 +26,7 @@
SCHEMA
SMALLINT
TABLE
+TABLES
TINYINT
TEMPORARY
@@ -154,6 +155,7 @@
DYNAMIC
FIXED
COMPRESSED
+COMPRESSION
REDUNDANT
COMPACT
TABLESPACE
@@ -192,6 +194,7 @@
COPY
INPLACE
INSTANT
+NOCOPY
VISIBLE
INVISIBLE
@@ -265,6 +268,7 @@
VALIDATION
VIRTUAL
+PERSISTENT
STORED
GENERATED
ALWAYS
@@ -275,6 +279,9 @@
START
TRANSACTION
+
+WAIT
+NOWAIT
)
diff --git a/src/main/antlr4/imports/mysql_alter_database.g4 b/src/main/antlr4/imports/mysql_alter_database.g4
index 69a2415ff..bd6ebc5cc 100644
--- a/src/main/antlr4/imports/mysql_alter_database.g4
+++ b/src/main/antlr4/imports/mysql_alter_database.g4
@@ -5,5 +5,7 @@ import mysql_literal_tokens, mysql_idents;
alter_database: ALTER (DATABASE | SCHEMA) name? alter_database_definition;
alter_database_definition:
(default_character_set | default_collation)+
- | UPGRADE DATA DIRECTORY NAME;
+ | UPGRADE DATA DIRECTORY NAME
+ | alter_encryption;
+alter_encryption: ENCRYPTION '='? string_literal;
diff --git a/src/main/antlr4/imports/mysql_alter_table.g4 b/src/main/antlr4/imports/mysql_alter_table.g4
index bffd17b35..c286c9cb6 100644
--- a/src/main/antlr4/imports/mysql_alter_table.g4
+++ b/src/main/antlr4/imports/mysql_alter_table.g4
@@ -2,10 +2,13 @@ grammar mysql_alter_table;
import mysql_literal_tokens, mysql_idents, column_definitions, mysql_partition;
-alter_table: alter_table_preamble alter_specifications? alter_partition_specification?;
+alter_table: alter_table_preamble alter_specifications? alter_partition_specification? alter_post_flags?;
-alter_table_preamble: ALTER alter_flags? TABLE table_name;
+alter_table_preamble: ALTER alter_flags? TABLE table_name wait_flag?;
alter_flags: (ONLINE | OFFLINE | IGNORE);
+wait_flag:
+ (WAIT integer | NOWAIT);
+
alter_specifications: alter_specification (',' alter_specification)*;
alter_specification:
@@ -25,20 +28,21 @@ alter_specification:
;
// the various alter_table commands available
-add_column: ADD COLUMN? column_definition col_position?;
-add_column_parens: ADD COLUMN? '(' (column_definition|index_definition) (',' (column_definition|index_definition))* ')';
+add_column: ADD COLUMN? if_not_exists? column_definition col_position?;
+add_column_parens: ADD COLUMN? if_not_exists? '(' (column_definition|index_definition) (',' (column_definition|index_definition))* ')';
change_column: CHANGE COLUMN? full_column_name column_definition col_position?;
-drop_column: DROP COLUMN? full_column_name CASCADE?;
+if_exists: IF EXISTS;
+drop_column: DROP COLUMN? if_exists? full_column_name CASCADE?;
modify_column: MODIFY COLUMN? column_definition col_position?;
-drop_key: DROP FOREIGN? (INDEX|KEY) name;
+drop_key: DROP FOREIGN? (INDEX|KEY) if_exists? name;
drop_primary_key: DROP PRIMARY KEY;
alter_rename_table: RENAME (TO | AS)? table_name;
convert_to_character_set: CONVERT TO charset_token charset_name collation?;
rename_column: RENAME COLUMN name TO name;
alter_partition_specification:
- ADD PARTITION skip_parens
- | DROP PARTITION partition_names
+ ADD PARTITION if_not_exists? skip_parens
+ | DROP PARTITION if_exists? partition_names
| TRUNCATE PARTITION partition_names
| DISCARD PARTITION partition_names TABLESPACE
| IMPORT PARTITION partition_names TABLESPACE
@@ -57,20 +61,29 @@ alter_partition_specification:
ignored_alter_specifications:
ADD index_definition
| ALTER INDEX name (VISIBLE | INVISIBLE)
- | ALTER COLUMN? name ((SET DEFAULT literal) | (DROP DEFAULT))
- | DROP INDEX index_name
+ | ALTER COLUMN? name ((SET DEFAULT literal) | (DROP DEFAULT) | (SET (VISIBLE | INVISIBLE)))
+ | DROP INDEX if_exists? index_name
| DISABLE KEYS
| ENABLE KEYS
| ORDER BY alter_ordering (',' alter_ordering)*
| FORCE
| DISCARD TABLESPACE
| IMPORT TABLESPACE
- | ALGORITHM '='? algorithm_type
- | LOCK '='? lock_type
| RENAME (INDEX|KEY) name TO name
+ | DROP CHECK name
+ | DROP CONSTRAINT name
+ | alter_post_flag
;
- algorithm_type: DEFAULT | INPLACE | COPY | INSTANT;
- lock_type: DEFAULT | NONE | SHARED | EXCLUSIVE;
+
+alter_post_flags:
+ alter_post_flag (',' alter_post_flag)*;
+
+alter_post_flag:
+ ALGORITHM '='? algorithm_type
+ | LOCK '='? lock_type;
+
+algorithm_type: DEFAULT | INPLACE | COPY | INSTANT | NOCOPY;
+lock_type: DEFAULT | NONE | SHARED | EXCLUSIVE;
partition_names: id (',' id)*;
alter_ordering: alter_ordering_column (ASC|DESC)?;
diff --git a/src/main/antlr4/imports/mysql_create_table.g4 b/src/main/antlr4/imports/mysql_create_table.g4
index d6cf86aca..fa14b32f7 100644
--- a/src/main/antlr4/imports/mysql_create_table.g4
+++ b/src/main/antlr4/imports/mysql_create_table.g4
@@ -26,6 +26,7 @@ table_creation_option:
| creation_checksum
| creation_collation
| creation_comment
+ | creation_compression
| creation_connection
| creation_data_directory
| creation_delay_key_write
@@ -65,12 +66,13 @@ creation_max_rows: MAX_ROWS '='? integer;
creation_min_rows: MIN_ROWS '='? integer;
creation_pack_keys: PACK_KEYS '='? (integer | DEFAULT);
creation_password: PASSWORD '='? string_literal;
+creation_compression: COMPRESSION '='? string_literal;
creation_row_format: ROW_FORMAT '='? (DEFAULT | DEFAULT | DYNAMIC | FIXED | COMPRESSED | REDUNDANT | COMPACT);
creation_stats_auto_recalc: STATS_AUTO_RECALC '='? (DEFAULT | INTEGER_LITERAL);
creation_stats_persistent: STATS_PERSISTENT '='? (DEFAULT | INTEGER_LITERAL);
-creation_stats_sample_pages: STATS_SAMPLE_PAGES '='? INTEGER_LITERAL;
+creation_stats_sample_pages: STATS_SAMPLE_PAGES '='? (DEFAULT | INTEGER_LITERAL);
creation_storage_option: STORAGE (DISK | MEMORY | DEFAULT);
-creation_tablespace: TABLESPACE string;
+creation_tablespace: tablespace;
creation_union: UNION '='? '(' name (',' name)* ')';
creation_encryption: ENCRYPTION '='? string_literal;
creation_start_transaction: START TRANSACTION;
diff --git a/src/main/antlr4/imports/mysql_idents.g4 b/src/main/antlr4/imports/mysql_idents.g4
index dcc414f45..fe9b5bdc4 100644
--- a/src/main/antlr4/imports/mysql_idents.g4
+++ b/src/main/antlr4/imports/mysql_idents.g4
@@ -18,10 +18,11 @@ user_token: (IDENT | QUOTED_IDENT | string_literal);
name: ( id | tokens_available_for_names | INTEGER_LITERAL | DBL_STRING_LITERAL );
name_all_tokens: ( id | all_tokens | INTEGER_LITERAL | DBL_STRING_LITERAL );
id: ( IDENT | QUOTED_IDENT );
-literal: (float_literal | integer_literal | string_literal | byte_literal | NULL | TRUE | FALSE);
-literal_with_weirdo_multistring: (float_literal | integer_literal | string_literal+ | byte_literal | NULL | TRUE | FALSE);
+literal: (float_literal | broken_float_literal | integer_literal | string_literal | byte_literal | NULL | TRUE | FALSE);
+literal_with_weirdo_multistring: (float_literal | broken_float_literal | integer_literal | string_literal+ | byte_literal | NULL | TRUE | FALSE);
float_literal: ('+'|'-')? INTEGER_LITERAL? '.' INTEGER_LITERAL;
+broken_float_literal: ('+'|'-')? INTEGER_LITERAL '.';
integer_literal: ('+'|'-')? INTEGER_LITERAL;
string_literal: (STRING_LITERAL | DBL_STRING_LITERAL);
byte_literal:
@@ -39,6 +40,8 @@ default_collation: DEFAULT? collation;
charset_token: (CHARSET | (CHARACTER SET) | (CHAR SET));
collation: COLLATE '='? (IDENT | string_literal | QUOTED_IDENT | DEFAULT);
+tablespace: TABLESPACE '='? (id | string_literal);
+
if_not_exists: IF NOT EXISTS;
diff --git a/src/main/antlr4/imports/mysql_indices.g4 b/src/main/antlr4/imports/mysql_indices.g4
index 8662c8c15..38da0fb95 100644
--- a/src/main/antlr4/imports/mysql_indices.g4
+++ b/src/main/antlr4/imports/mysql_indices.g4
@@ -2,23 +2,26 @@ grammar mysql_indices;
import mysql_literal_tokens, mysql_idents;
+if_not_exists:
+ IF NOT EXISTS;
+
index_definition:
(index_type_1 | index_type_pk | index_type_3 | index_type_4 | index_type_5 | index_type_check);
index_type_1:
- index_or_key index_name? index_type? index_column_list index_options*;
+ index_or_key if_not_exists? index_name? index_type? index_column_list index_options*;
index_type_pk:
- index_constraint? PRIMARY KEY (index_type | index_name)* index_column_list index_options*;
+ index_constraint? PRIMARY KEY if_not_exists? (index_type | index_name)* index_column_list index_options*;
index_type_3:
- index_constraint? UNIQUE index_or_key? index_name? index_type? index_column_list index_options*;
+ index_constraint? UNIQUE if_not_exists? index_or_key? index_name? index_type? index_column_list index_options*;
index_type_4:
- (FULLTEXT | SPATIAL) index_or_key? index_name? index_column_list index_options*;
+ (FULLTEXT | SPATIAL) index_or_key? if_not_exists? index_name? index_column_list index_options*;
index_type_5:
- index_constraint? FOREIGN KEY index_name? index_column_list reference_definition;
+ index_constraint? FOREIGN KEY if_not_exists? index_name? index_column_list reference_definition;
index_type_check:
@@ -43,7 +46,8 @@ index_columns: index_column (',' index_column )*;
index_column:
name index_column_partial_def? index_column_asc_or_desc?
- | '(' CAST skip_parens ')';
+ | skip_parens index_column_asc_or_desc?;
+
index_column_partial_def: '(' index_column_partial_length ')';
index_column_partial_length: INTEGER_LITERAL+;
diff --git a/src/main/antlr4/imports/mysql_literal_tokens.g4 b/src/main/antlr4/imports/mysql_literal_tokens.g4
index d057ff070..411b28413 100644
--- a/src/main/antlr4/imports/mysql_literal_tokens.g4
+++ b/src/main/antlr4/imports/mysql_literal_tokens.g4
@@ -1,8 +1,8 @@
// This file is automatically generated by src/main/antlr4/imports/generate_tokens.rb
grammar mysql_literal_tokens;
-tokens_available_for_names: (ACTION | AFTER | ALGORITHM | ALWAYS | ASCII | AUTO_INCREMENT | AVG_ROW_LENGTH | BEGIN | BIT | BOOL | BOOLEAN | BTREE | BYTE | CAST | CHARSET | CHECKSUM | COALESCE | COLUMNS | COLUMN_FORMAT | COMMENT | COMPACT | COMPRESSED | CONNECTION | COPY | DATA | DATE | DATETIME | DEFINER | DELAY_KEY_WRITE | DIRECTORY | DISABLE | DISCARD | DISK | DYNAMIC | ENABLE | ENCRYPTION | ENGINE | ENUM | EXCHANGE | EXCLUSIVE | FIRST | FIXED | FULL | GEOMETRY | GEOMETRYCOLLECTION | HASH | IMPORT | INPLACE | INSERT_METHOD | INSTANT | INVISIBLE | INVOKER | JSON | KEY_BLOCK_SIZE | LAST | LINESTRING | LIST | MAX_ROWS | MEMORY | MERGE | MIN_ROWS | MODIFY | MULTILINESTRING | MULTIPOINT | MULTIPOLYGON | NAME | NATIONAL | NCHAR | NO | NONE | NOW | NVARCHAR | OFFLINE | ONLINE | PACK_KEYS | PARSER | PARTIAL | PARTITIONING | PARTITIONS | PASSWORD | POINT | POLYGON | REBUILD | REDUNDANT | REMOVE | REORGANIZE | REPAIR | ROW_FORMAT | SECURITY | SERIAL | SHARED | SIGNED | SIMPLE | SRID | START | STATS_AUTO_RECALC | STATS_PERSISTENT | STATS_SAMPLE_PAGES | STORAGE | SUBPARTITION | SUBPARTITIONS | TABLESPACE | TEMPORARY | TEMPTABLE | TEXT | TIME | TIMESTAMP | TRANSACTION | TRUNCATE | UNDEFINED | UNICODE | UPGRADE | VALIDATION | VIEW | VISIBLE | WITHOUT | YEAR);
-all_tokens: (ACTION | ADD | AFTER | ALGORITHM | ALTER | ALWAYS | ANALYZE | AS | ASC | ASCII | AUTO_INCREMENT | AVG_ROW_LENGTH | BEGIN | BIGINT | BINARY | BIT | BLOB | BOOL | BOOLEAN | BTREE | BY | BYTE | CASCADE | CAST | CHANGE | CHAR | CHARACTER | CHARSET | CHECK | CHECKSUM | COALESCE | COLLATE | COLUMN | COLUMNS | COLUMN_FORMAT | COMMENT | COMPACT | COMPRESSED | CONNECTION | CONSTRAINT | CONVERT | COPY | CREATE | CURRENT_TIMESTAMP | CURRENT_USER | DATA | DATABASE | DATE | DATETIME | DECIMAL | DEFAULT | DEFINER | DELAY_KEY_WRITE | DELETE | DESC | DIRECTORY | DISABLE | DISCARD | DISK | DOUBLE | DROP | DYNAMIC | ENABLE | ENCRYPTION | ENGINE | ENUM | EXCHANGE | EXCLUSIVE | EXISTS | FALSE | FIRST | FIXED | FLOAT | FLOAT4 | FLOAT8 | FORCE | FOREIGN | FULL | FULLTEXT | GENERATED | GEOMETRY | GEOMETRYCOLLECTION | HASH | IF | IGNORE | IMPORT | INDEX | INPLACE | INSERT_METHOD | INSTANT | INT | INT1 | INT2 | INT3 | INT4 | INT8 | INTEGER | INTO | INVISIBLE | INVOKER | JSON | KEY | KEYS | KEY_BLOCK_SIZE | LAST | LIKE | LINEAR | LINESTRING | LIST | LOCALTIME | LOCALTIMESTAMP | LOCK | LONG | LONGBLOB | LONGTEXT | MATCH | MAX_ROWS | MEDIUMBLOB | MEDIUMINT | MEDIUMTEXT | MEMORY | MERGE | MIDDLEINT | MIN_ROWS | MODIFY | MULTILINESTRING | MULTIPOINT | MULTIPOLYGON | NAME | NATIONAL | NCHAR | NO | NONE | NOT | NOW | NULL | NUMERIC | NVARCHAR | OFFLINE | ON | ONLINE | OPTIMIZE | OR | ORDER | PACK_KEYS | PARSER | PARTIAL | PARTITION | PARTITIONING | PARTITIONS | PASSWORD | POINT | POLYGON | PRECISION | PRIMARY | RANGE | REAL | REBUILD | REDUNDANT | REFERENCES | REMOVE | RENAME | REORGANIZE | REPAIR | REPLACE | RESTRICT | ROW_FORMAT | SCHEMA | SECURITY | SERIAL | SET | SHARED | SIGNED | SIMPLE | SMALLINT | SPATIAL | SQL | SRID | START | STATS_AUTO_RECALC | STATS_PERSISTENT | STATS_SAMPLE_PAGES | STORAGE | STORED | SUBPARTITION | SUBPARTITIONS | TABLE | TABLESPACE | TEMPORARY | TEMPTABLE | TEXT | TIME | TIMESTAMP | TINYBLOB | TINYINT | TINYTEXT | TO | TRANSACTION | TRUE | TRUNCATE | UNDEFINED | UNICODE | UNION | UNIQUE | UNSIGNED | UPDATE | UPGRADE | USING | VALIDATION | VARBINARY | VARCHAR | VARYING | VIEW | VIRTUAL | VISIBLE | WITH | WITHOUT | YEAR | ZEROFILL);
+tokens_available_for_names: (ACTION | AFTER | ALGORITHM | ALWAYS | ASCII | AUTO_INCREMENT | AVG_ROW_LENGTH | BEGIN | BIT | BOOL | BOOLEAN | BTREE | BYTE | CAST | CHARSET | CHECKSUM | COALESCE | COLUMNS | COLUMN_FORMAT | COMMENT | COMPACT | COMPRESSED | COMPRESSION | CONNECTION | COPY | DATA | DATE | DATETIME | DEFINER | DELAY_KEY_WRITE | DIRECTORY | DISABLE | DISCARD | DISK | DYNAMIC | ENABLE | ENCRYPTION | ENGINE | ENUM | EXCHANGE | EXCLUSIVE | FIRST | FIXED | FULL | GEOMETRY | GEOMETRYCOLLECTION | HASH | IMPORT | INPLACE | INSERT_METHOD | INSTANT | INVISIBLE | INVOKER | JSON | KEY_BLOCK_SIZE | LAST | LINESTRING | LIST | MAX_ROWS | MEMORY | MERGE | MIN_ROWS | MODIFY | MULTILINESTRING | MULTIPOINT | MULTIPOLYGON | NAME | NATIONAL | NCHAR | NO | NOCOPY | NONE | NOW | NOWAIT | NVARCHAR | OFFLINE | ONLINE | PACK_KEYS | PARSER | PARTIAL | PARTITIONING | PARTITIONS | PASSWORD | POINT | POLYGON | REBUILD | REDUNDANT | REMOVE | REORGANIZE | REPAIR | ROW_FORMAT | SECURITY | SERIAL | SHARED | SIGNED | SIMPLE | SRID | START | STATS_AUTO_RECALC | STATS_PERSISTENT | STATS_SAMPLE_PAGES | STORAGE | SUBPARTITION | SUBPARTITIONS | TABLES | TABLESPACE | TEMPORARY | TEMPTABLE | TEXT | TIME | TIMESTAMP | TRANSACTION | TRUNCATE | UNDEFINED | UNICODE | UPGRADE | VALIDATION | VIEW | VISIBLE | WAIT | WITHOUT | YEAR);
+all_tokens: (ACTION | ADD | AFTER | ALGORITHM | ALTER | ALWAYS | ANALYZE | AS | ASC | ASCII | AUTO_INCREMENT | AVG_ROW_LENGTH | BEGIN | BIGINT | BINARY | BIT | BLOB | BOOL | BOOLEAN | BTREE | BY | BYTE | CASCADE | CAST | CHANGE | CHAR | CHARACTER | CHARSET | CHECK | CHECKSUM | COALESCE | COLLATE | COLUMN | COLUMNS | COLUMN_FORMAT | COMMENT | COMPACT | COMPRESSED | COMPRESSION | CONNECTION | CONSTRAINT | CONVERT | COPY | CREATE | CURRENT_TIMESTAMP | CURRENT_USER | DATA | DATABASE | DATE | DATETIME | DECIMAL | DEFAULT | DEFINER | DELAY_KEY_WRITE | DELETE | DESC | DIRECTORY | DISABLE | DISCARD | DISK | DOUBLE | DROP | DYNAMIC | ENABLE | ENCRYPTION | ENGINE | ENUM | EXCHANGE | EXCLUSIVE | EXISTS | FALSE | FIRST | FIXED | FLOAT | FLOAT4 | FLOAT8 | FORCE | FOREIGN | FULL | FULLTEXT | GENERATED | GEOMETRY | GEOMETRYCOLLECTION | HASH | IF | IGNORE | IMPORT | INDEX | INPLACE | INSERT_METHOD | INSTANT | INT | INT1 | INT2 | INT3 | INT4 | INT8 | INTEGER | INTO | INVISIBLE | INVOKER | JSON | KEY | KEYS | KEY_BLOCK_SIZE | LAST | LIKE | LINEAR | LINESTRING | LIST | LOCALTIME | LOCALTIMESTAMP | LOCK | LONG | LONGBLOB | LONGTEXT | MATCH | MAX_ROWS | MEDIUMBLOB | MEDIUMINT | MEDIUMTEXT | MEMORY | MERGE | MIDDLEINT | MIN_ROWS | MODIFY | MULTILINESTRING | MULTIPOINT | MULTIPOLYGON | NAME | NATIONAL | NCHAR | NO | NOCOPY | NONE | NOT | NOW | NOWAIT | NULL | NUMERIC | NVARCHAR | OFFLINE | ON | ONLINE | OPTIMIZE | OR | ORDER | PACK_KEYS | PARSER | PARTIAL | PARTITION | PARTITIONING | PARTITIONS | PASSWORD | PERSISTENT | POINT | POLYGON | PRECISION | PRIMARY | RANGE | REAL | REBUILD | REDUNDANT | REFERENCES | REMOVE | RENAME | REORGANIZE | REPAIR | REPLACE | RESTRICT | ROW_FORMAT | SCHEMA | SECURITY | SERIAL | SET | SHARED | SIGNED | SIMPLE | SMALLINT | SPATIAL | SQL | SRID | START | STATS_AUTO_RECALC | STATS_PERSISTENT | STATS_SAMPLE_PAGES | STORAGE | STORED | SUBPARTITION | SUBPARTITIONS | TABLE | TABLES | TABLESPACE | TEMPORARY | TEMPTABLE | TEXT | TIME | TIMESTAMP | TINYBLOB | TINYINT | TINYTEXT | TO | TRANSACTION | TRUE | TRUNCATE | UNDEFINED | UNICODE | UNION | UNIQUE | UNSIGNED | UPDATE | UPGRADE | USING | VALIDATION | VARBINARY | VARCHAR | VARYING | VIEW | VIRTUAL | VISIBLE | WAIT | WITH | WITHOUT | YEAR | ZEROFILL);
ACTION: A C T I O N;
ADD: A D D;
@@ -42,6 +42,7 @@ COLUMN_FORMAT: C O L U M N '_' F O R M A T;
COMMENT: C O M M E N T;
COMPACT: C O M P A C T;
COMPRESSED: C O M P R E S S E D;
+COMPRESSION: C O M P R E S S I O N;
CONNECTION: C O N N E C T I O N;
CONSTRAINT: C O N S T R A I N T;
CONVERT: C O N V E R T;
@@ -136,9 +137,11 @@ NAME: N A M E;
NATIONAL: N A T I O N A L;
NCHAR: N C H A R;
NO: N O;
+NOCOPY: N O C O P Y;
NONE: N O N E;
NOT: N O T;
NOW: N O W;
+NOWAIT: N O W A I T;
NULL: N U L L;
NUMERIC: N U M E R I C;
NVARCHAR: N V A R C H A R;
@@ -155,6 +158,7 @@ PARTITION: P A R T I T I O N;
PARTITIONING: P A R T I T I O N I N G;
PARTITIONS: P A R T I T I O N S;
PASSWORD: P A S S W O R D;
+PERSISTENT: P E R S I S T E N T;
POINT: P O I N T;
POLYGON: P O L Y G O N;
PRECISION: P R E C I S I O N;
@@ -191,6 +195,7 @@ STORED: S T O R E D;
SUBPARTITION: S U B P A R T I T I O N;
SUBPARTITIONS: S U B P A R T I T I O N S;
TABLE: T A B L E;
+TABLES: T A B L E S;
TABLESPACE: T A B L E S P A C E;
TEMPORARY: T E M P O R A R Y;
TEMPTABLE: T E M P T A B L E;
@@ -219,6 +224,7 @@ VARYING: V A R Y I N G;
VIEW: V I E W;
VIRTUAL: V I R T U A L;
VISIBLE: V I S I B L E;
+WAIT: W A I T;
WITH: W I T H;
WITHOUT: W I T H O U T;
YEAR: Y E A R;
diff --git a/src/main/antlr4/imports/mysql_rename.g4 b/src/main/antlr4/imports/mysql_rename.g4
index bd3d9c1d2..7aadac987 100644
--- a/src/main/antlr4/imports/mysql_rename.g4
+++ b/src/main/antlr4/imports/mysql_rename.g4
@@ -1,6 +1,6 @@
grammar mysql_rename;
import mysql_literal_tokens, mysql_idents;
-rename_table: RENAME TABLE rename_table_spec (',' rename_table_spec)*;
+rename_table: RENAME (TABLE | TABLES) rename_table_spec (',' rename_table_spec)*;
rename_table_spec: table_name TO table_name;
diff --git a/src/main/java/com/zendesk/maxwell/BufferedMaxwell.java b/src/main/java/com/zendesk/maxwell/BufferedMaxwell.java
index 3957a5001..ef7a7b867 100644
--- a/src/main/java/com/zendesk/maxwell/BufferedMaxwell.java
+++ b/src/main/java/com/zendesk/maxwell/BufferedMaxwell.java
@@ -9,14 +9,27 @@
import java.util.concurrent.TimeUnit;
/**
- * Created by ben on 8/27/16.
+ * A subclass of {@link Maxwell} that buffers rows in-memory for consumption by the caller
*/
public class BufferedMaxwell extends Maxwell {
+ /**
+ * Initializer for a buffered Maxwell instance. Sets up buffered producer.
+ * @param config Maxwell configuration
+ * @throws SQLException if we have db issues
+ * @throws URISyntaxException if we can't build a database URI
+ */
public BufferedMaxwell(MaxwellConfig config) throws SQLException, URISyntaxException {
super(config);
config.producerType = "buffer";
}
+ /**
+ * Poll for a RowMap to be producer by the maxwell instance.
+ * @param ms poll time to wait in milliseconds before timing out
+ * @return RowMap
+ * @throws IOException if we have issues building a producer
+ * @throws InterruptedException if we are interrupted
+ */
public RowMap poll(long ms) throws IOException, InterruptedException {
BufferedProducer p = (BufferedProducer) this.context.getProducer();
return p.poll(ms, TimeUnit.MILLISECONDS);
diff --git a/src/main/java/com/zendesk/maxwell/CaseSensitivity.java b/src/main/java/com/zendesk/maxwell/CaseSensitivity.java
index a4337c4cc..5eea5a1c6 100644
--- a/src/main/java/com/zendesk/maxwell/CaseSensitivity.java
+++ b/src/main/java/com/zendesk/maxwell/CaseSensitivity.java
@@ -1,5 +1,9 @@
package com.zendesk.maxwell;
+/**
+ * Describes the 3 difference case sensitivity settings on a Mysql server.
+ * Case sensitivity rules apply to databases and tables, not columns.
+ */
public enum CaseSensitivity { CASE_SENSITIVE, CONVERT_TO_LOWER, CONVERT_ON_COMPARE };
diff --git a/src/main/java/com/zendesk/maxwell/Maxwell.java b/src/main/java/com/zendesk/maxwell/Maxwell.java
index 4de9d6d33..04b0741d4 100644
--- a/src/main/java/com/zendesk/maxwell/Maxwell.java
+++ b/src/main/java/com/zendesk/maxwell/Maxwell.java
@@ -22,6 +22,11 @@
import java.util.ArrayList;
import java.util.List;
+/**
+ * The main Maxwell class. Instantiate and call `.run` or `.start` to start Maxwell.
+ * @see #run()
+ * @see #start()
+ */
public class Maxwell implements Runnable {
protected MaxwellConfig config;
protected MaxwellContext context;
@@ -29,6 +34,12 @@ public class Maxwell implements Runnable {
static final Logger LOGGER = LoggerFactory.getLogger(Maxwell.class);
+ /**
+ * Intialize a top level Maxwell runner
+ * @param config Maxwell configuration
+ * @throws SQLException If Maxwell can't connect
+ * @throws URISyntaxException If there's a problem with the database configuration
+ */
public Maxwell(MaxwellConfig config) throws SQLException, URISyntaxException {
this(new MaxwellContext(config));
}
@@ -38,6 +49,9 @@ protected Maxwell(MaxwellContext context) throws SQLException, URISyntaxExceptio
this.context = context;
}
+ /**
+ * run Maxwell, catching all Exceptions.
+ */
public void run() {
try {
start();
@@ -46,11 +60,18 @@ public void run() {
}
}
+ /**
+ * restarts a stopped Maxwell instance. rebuilds all connections, threads, etc.
+ * @throws Exception If Maxwell can't initialize its context
+ */
public void restart() throws Exception {
this.context = new MaxwellContext(config);
start();
}
+ /**
+ * Stop the currently running Maxwell
+ */
public void terminate() {
Thread terminationThread = this.context.terminate();
if (terminationThread != null) {
@@ -101,9 +122,9 @@ private Position attemptMasterRecovery() throws Exception {
}
private void logColumnCastError(ColumnDefCastException e) throws SQLException, SchemaStoreException {
- try ( Connection conn = context.getSchemaConnectionPool().getConnection() ) {
- LOGGER.error("checking for schema inconsistencies in " + e.database + "." + e.table);
- SchemaCapturer capturer = new SchemaCapturer(conn, context.getCaseSensitivity(), e.database, e.table);
+ LOGGER.error("checking for schema inconsistencies in " + e.database + "." + e.table);
+ try ( Connection conn = context.getSchemaConnectionPool().getConnection();
+ SchemaCapturer capturer = new SchemaCapturer(conn, context.getCaseSensitivity(), e.database, e.table)) {
Schema recaptured = capturer.capture();
Table t = this.replicator.getSchema().findDatabase(e.database).findTable(e.table);
List diffs = new ArrayList<>();
@@ -119,6 +140,17 @@ private void logColumnCastError(ColumnDefCastException e) throws SQLException, S
}
}
+ /**
+ * Determines initial replication position
+ *
+ *
Retrieve stored position from `maxwell`.`positons`
+ *
Attempt master recovery
+ *
Use previous client_id's position. See https://github.com/zendesk/maxwell/issues/782
+ *
Capture the current master position
+ *
+ * @return Binlog position to start replicating at
+ * @throws Exception Various SQL and IO exceptions
+ */
protected Position getInitialPosition() throws Exception {
/* first method: do we have a stored position for this server? */
Position initial = this.context.getInitialPosition();
@@ -174,10 +206,22 @@ private void logBanner(AbstractProducer producer, Position initialPosition) {
LOGGER.info(String.format(bootString, getMaxwellVersion(), producerName, initialPosition.toString()));
}
+ /**
+ * Hook for subclasses to execute code after all initialization is complete,
+ * but before replication starts.
+ */
protected void onReplicatorStart() {}
+
+ /**
+ * Hook for subclasses to execute code before termination of the instance
+ */
protected void onReplicatorEnd() {}
+ /**
+ * Start maxwell
+ * @throws Exception If maxwell stops due to an Exception
+ */
public void start() throws Exception {
try {
this.startInner();
@@ -241,8 +285,11 @@ private void startInner() throws Exception {
context.getHeartbeatNotifier(),
config.scripting,
context.getFilter(),
+ context.getConfig().getIgnoreMissingSchema(),
config.outputConfig,
- config.bufferMemoryUsage
+ config.bufferMemoryUsage,
+ config.replicationReconnectionRetries,
+ config.binlogEventQueueSize
);
context.setReplicator(replicator);
@@ -259,6 +306,10 @@ private void startInner() throws Exception {
}
+ /**
+ * The main entry point for Maxwell
+ * @param args command line arguments
+ */
public static void main(String[] args) {
try {
Logging.setupLogBridging();
@@ -287,18 +338,19 @@ public void run() {
}
} catch ( SQLException e ) {
// catch SQLException explicitly because we likely don't care about the stacktrace
- LOGGER.error("SQLException: " + e.getLocalizedMessage());
+ LOGGER.error("SQLException: " + e.getLocalizedMessage(), e);
System.exit(1);
} catch ( URISyntaxException e ) {
// catch URISyntaxException explicitly as well to provide more information to the user
LOGGER.error("Syntax issue with URI, check for misconfigured host, port, database, or JDBC options (see RFC 2396)");
- LOGGER.error("URISyntaxException: " + e.getLocalizedMessage());
+ LOGGER.error("URISyntaxException: " + e.getLocalizedMessage(), e);
System.exit(1);
} catch ( ServerException e ) {
- LOGGER.error("Maxwell couldn't find the requested binlog, exiting...");
+ LOGGER.error("Maxwell couldn't find the requested binlog, exiting...", e);
System.exit(2);
} catch ( Exception e ) {
e.printStackTrace();
+ LOGGER.error("Maxwell saw an exception and is exiting...", e);
System.exit(1);
}
}
diff --git a/src/main/java/com/zendesk/maxwell/MaxwellCompatibilityError.java b/src/main/java/com/zendesk/maxwell/MaxwellCompatibilityError.java
index 74e34b4ee..e55711604 100644
--- a/src/main/java/com/zendesk/maxwell/MaxwellCompatibilityError.java
+++ b/src/main/java/com/zendesk/maxwell/MaxwellCompatibilityError.java
@@ -1,5 +1,8 @@
package com.zendesk.maxwell;
+/**
+ * Thrown when Maxwell can't operate with the mysql server configured as it is.
+ */
public class MaxwellCompatibilityError extends Exception {
public MaxwellCompatibilityError(String message) {
super(message);
diff --git a/src/main/java/com/zendesk/maxwell/MaxwellConfig.java b/src/main/java/com/zendesk/maxwell/MaxwellConfig.java
index d2be7be72..e96f37fe9 100644
--- a/src/main/java/com/zendesk/maxwell/MaxwellConfig.java
+++ b/src/main/java/com/zendesk/maxwell/MaxwellConfig.java
@@ -6,161 +6,643 @@
import com.zendesk.maxwell.filtering.Filter;
import com.zendesk.maxwell.filtering.InvalidFilterException;
import com.zendesk.maxwell.monitoring.MaxwellDiagnosticContext;
+import com.zendesk.maxwell.monitoring.MaxwellHealthCheckFactory;
import com.zendesk.maxwell.producer.EncryptionMode;
import com.zendesk.maxwell.producer.MaxwellOutputConfig;
import com.zendesk.maxwell.producer.ProducerFactory;
+import com.zendesk.maxwell.replication.BinlogConnectorReplicator;
import com.zendesk.maxwell.replication.BinlogPosition;
import com.zendesk.maxwell.replication.Position;
import com.zendesk.maxwell.scripting.Scripting;
import com.zendesk.maxwell.util.AbstractConfig;
import com.zendesk.maxwell.util.MaxwellOptionParser;
import joptsimple.OptionSet;
+import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.threeten.bp.Duration;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.regex.Pattern;
+/**
+ * Configuration object for Maxwell
+ */
public class MaxwellConfig extends AbstractConfig {
static final Logger LOGGER = LoggerFactory.getLogger(MaxwellConfig.class);
+ /**
+ * String that describes an environment key that, if set, will enable Maxwell's GTID mode
+ *
+ * Primarily used for test environment setup.
+ *
+ */
public static final String GTID_MODE_ENV = "GTID_MODE";
+ /**
+ * If non-null, specify a mysql to replicate from.
+ * If non, fallback to {@link #maxwellMysql}
+ */
public MaxwellMysqlConfig replicationMysql;
+
+ /**
+ * Number of times to attempt connecting the replicator before giving up
+ */
+ public int replicationReconnectionRetries;
+
+ /**
+ * If non-null, specify a mysql server to capture schema from
+ * If non, fallback to {@link #maxwellMysql}
+ */
public MaxwellMysqlConfig schemaMysql;
+ /**
+ * Specify a "root" maxwell server
+ */
public MaxwellMysqlConfig maxwellMysql;
+
+ /**
+ * Configuration for including/excluding rows
+ */
public Filter filter;
+
+ /**
+ * Ignore any missing database / table schemas, unless they're
+ * included as part of filters. Default false. Don't use unless
+ * you really really need to.
+ */
+ public Boolean ignoreMissingSchema;
+
+ /**
+ * Attempt to use Mysql GTIDs to keep track of position
+ */
public Boolean gtidMode;
+ /**
+ * Name of database in which to store maxwell data (default `maxwell`)
+ */
public String databaseName;
- public String includeDatabases, excludeDatabases, includeTables, excludeTables, excludeColumns, blacklistDatabases, blacklistTables, includeColumnValues;
+ /**
+ * filter out these columns
+ */
+ public String excludeColumns;
+
+ /**
+ * Maxwell filters
+ */
public String filterList;
+ /**
+ * If non-null, generate a producer with this factory
+ */
public ProducerFactory producerFactory; // producerFactory has precedence over producerType
+
+ /**
+ * Available to customer producers for configuration.
+ * Setup with all properties prefixed `customer_producer.`
+ */
public final Properties customProducerProperties;
+
+ /**
+ * Available to customer producers for configuration.
+ * Setup with all properties prefixed `customer_producer.`
+ */
+ public MaxwellHealthCheckFactory customHealthFactory;
+
+ /**
+ * String describing desired producer type: "kafka", "kinesis", etc.
+ */
public String producerType;
+ /**
+ * Properties object containing all configuration options beginning with "kafka."
+ */
public final Properties kafkaProperties;
+
+ /**
+ * Main kafka topic to produce to
+ */
public String kafkaTopic;
+
+ /**
+ * Kafka topic to send undeliverable rows to
+ */
public String deadLetterTopic;
+
+ /**
+ * Kafka topic to send schema changes (DDL) to
+ */
public String ddlKafkaTopic;
+
+ /**
+ * "hash" or "array" -- defines format of kafka key
+ */
public String kafkaKeyFormat;
+
+ /**
+ * "default" or "murmur3", defines partition-choice hash function
+ */
public String kafkaPartitionHash;
- public String kafkaPartitionKey;
- public String kafkaPartitionColumns;
- public String kafkaPartitionFallback;
+
+ /**
+ * "async" or "sync", describes bootstrapping behavior
+ */
public String bootstrapperType;
+
+ /**
+ * size of queue for buffered producer
+ */
public int bufferedProducerSize;
+ /**
+ * database|table|primary_key|transaction_id|column|random
+ * Input for partition choice function
+ */
public String producerPartitionKey;
+
+ /**
+ * when producerPartitionKey is "column", list of columns to partition by
+ */
public String producerPartitionColumns;
+
+ /**
+ * when producerPartitionKey is "column", database|table|primary_key to fall back to
+ * (when column is unavailable)
+ */
public String producerPartitionFallback;
+ /**
+ * Kinesis stream name
+ */
public String kinesisStream;
+
+ /**
+ * If true, pass key through {@link DigestUtils#md5Hex} before sending to Kinesis.
+ * Limits the size of the kinesis key, iirc.
+ */
public boolean kinesisMd5Keys;
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellSQSProducer} Queue URI
+ */
public String sqsQueueUri;
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellSQSProducer} Queue Service Endpoint URL
+ */
+ public String sqsServiceEndpoint;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellSQSProducer} Queue Signing region
+ */
+ public String sqsSigningRegion;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellSQSProducer} topic
+ */
public String snsTopic;
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellSQSProducer}
+ * ["table"|"database"] -- if set, interpolate either/or table / database into the message
+ */
public String snsAttrs;
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} project id
+ */
public String pubsubProjectId;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} topic
+ */
public String pubsubTopic;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} DDL topic
+ */
public String ddlPubsubTopic;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} bytes request threshold
+ */
public Long pubsubRequestBytesThreshold;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} message count batch size
+ */
public Long pubsubMessageCountBatchSize;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} message ordering key template (will enable message ordering if specified)
+ */
+ public String pubsubMessageOrderingKey;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} publish delay threshold
+ */
public Duration pubsubPublishDelayThreshold;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} retry delay
+ */
public Duration pubsubRetryDelay;
- public Double pubsubRetryDelayMultiplier;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} retry delay multiplier
+ */
+ public Float pubsubRetryDelayMultiplier;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} max retry delay
+ */
public Duration pubsubMaxRetryDelay;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} initial RPC timeout
+ */
public Duration pubsubInitialRpcTimeout;
- public Double pubsubRpcTimeoutMultiplier;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} RPC timeout multiplier
+ */
+ public Float pubsubRpcTimeoutMultiplier;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} max RPC timeout
+ */
public Duration pubsubMaxRpcTimeout;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} total timeout
+ */
public Duration pubsubTotalTimeout;
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellPubsubProducer} emulator host to use, if specified
+ */
+ public String pubsubEmulator;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellBigQueryProducer} project id
+ */
+ public String bigQueryProjectId;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellBigQueryProducer} dataset
+ */
+ public String bigQueryDataset;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellBigQueryProducer} table
+ */
+ public String bigQueryTable;
+
+
+ /**
+ * Used in all producers deriving from {@link com.zendesk.maxwell.producer.AbstractAsyncProducer}.
+ * In milliseconds, time a message can spend in the {@link com.zendesk.maxwell.producer.InflightMessageList}
+ * without server acknowledgement before being considered lost.
+ */
public Long producerAckTimeout;
+ /**
+ * output file path for the {@link com.zendesk.maxwell.producer.FileProducer}
+ */
public String outputFile;
+
+ /**
+ * Controls output features and formats
+ */
public MaxwellOutputConfig outputConfig;
+
+ /**
+ * string representation of java log level
+ */
public String log_level;
+ /**
+ * container for maxwell metric collection
+ */
public MetricRegistry metricRegistry;
+
+ /**
+ * container for maxwell health checks
+ */
public HealthCheckRegistry healthCheckRegistry;
+ /**
+ * http port for metrics/admin server
+ */
public int httpPort;
+
+ /**
+ * bind adress for metrics/admin server
+ */
public String httpBindAddress;
+
+ /**
+ * path prefix for metrics/admin server
+ */
public String httpPathPrefix;
+
+ /**
+ * path prefix for metrics server
+ */
public String metricsPrefix;
+
+ /**
+ * string describing how to report metrics.
+ */
public String metricsReportingType;
+
+ /**
+ * for slf4j metrics reporter, how often to report
+ */
public Long metricsSlf4jInterval;
+
+ /**
+ * How to report metrics to datadog, either "udp" or "http"
+ */
public String metricsDatadogType;
+
+ /**
+ * list of additional tags to send to datadog, as tag:value,tag:value
+ */
public String metricsDatadogTags;
+
+ /**
+ * datadog apikey used when reporting type is http
+ */
public String metricsDatadogAPIKey;
+
+ /**
+ * "us" or "eu"
+ */
public String metricsDatadogSite;
+
+ /**
+ * host to send UDP DD metrics to
+ */
public String metricsDatadogHost;
+
+ /**
+ * port to send UDP DD metrics to
+ */
public int metricsDatadogPort;
+
+ /**
+ * time in seconds between datadog metrics pushes
+ */
public Long metricsDatadogInterval;
+
+ /**
+ * whether to report JVM metrics
+ */
public boolean metricsJvm;
+
+ /**
+ * time in seconds before incrementing the "slo_violation" metric
+ */
public int metricsAgeSlo;
+ /**
+ * configuration for maxwell http diagnostic endpoint
+ */
public MaxwellDiagnosticContext.Config diagnosticConfig;
+ /**
+ * whether to enable reconfiguration via http endpoint
+ *
+ * For the moment this endpoint only allows changing of filters in runtime
+ *
+ */
public boolean enableHttpConfig;
+ /**
+ * String that uniquely identifies this instance of maxwell
+ */
public String clientID;
+
+ /**
+ * integer that maxwell will report to the server as its "server_id".
+ *
+ * Must be unique within the cluster.
+ *
+ */
public Long replicaServerID;
+ /**
+ * Override Maxwell's stored starting position
+ */
public Position initPosition;
+
+ /**
+ * If true, Maxwell plays events but otherwise stores no schema changes or position changes
+ */
public boolean replayMode;
+
+ /**
+ * Enable non-GTID master recovery code
+ */
public boolean masterRecovery;
+
+ /**
+ * If true, continue on certain producer errors. Otherwise crash.
+ */
public boolean ignoreProducerError;
+
+ /**
+ * Force a new schema capture upon startup. dangerous.
+ */
public boolean recaptureSchema;
+
+ /**
+ * float between 0 and 1, defines percentage of JVM memory to use buffering rows.
+ *
+ * actual formula is given as bufferMemoryUsage * Runtime.getRuntime().maxMemory().
+ *
+ */
public float bufferMemoryUsage;
+
+ /**
+ * How many schema "deltas" are kept live before a schema compaction is triggered.
+ * @see com.zendesk.maxwell.schema.MysqlSchemaCompactor
+ */
public Integer maxSchemaDeltas;
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} username
+ */
public String rabbitmqUser;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} password
+ */
public String rabbitmqPass;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} host
+ */
public String rabbitmqHost;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} port
+ */
public Integer rabbitmqPort;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} virtual host
+ */
public String rabbitmqVirtualHost;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} url (alternative to other configuration settings)
+ */
public String rabbitmqURI;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} handshake timeout
+ */
+ public Integer rabbitmqHandshakeTimeout;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} exchange
+ */
public String rabbitmqExchange;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} exchange type
+ */
public String rabbitmqExchangeType;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} exchange durability
+ */
public boolean rabbitMqExchangeDurable;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} exchange audo deletion
+ */
public boolean rabbitMqExchangeAutoDelete;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} routing key template
+ */
public String rabbitmqRoutingKeyTemplate;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} message persistence
+ */
public boolean rabbitmqMessagePersistent;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} declare exchange
+ */
public boolean rabbitmqDeclareExchange;
+ /**
+ * {@link com.zendesk.maxwell.producer.RabbitmqProducer} use SSL
+ */
+ public boolean rabbitmqUseSSL;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.NatsProducer} URL
+ */
public String natsUrl;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.NatsProducer} Message Subject
+ */
public String natsSubject;
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} host
+ */
public String redisHost;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} port
+ */
public int redisPort;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} password
+ */
public String redisAuth;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} database
+ */
public int redisDatabase;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} key
+ */
public String redisKey;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} JSON key for XADD
+ *
+ * when XADD is used, the event is embedded as a JSON string
+ * inside a field named this. defaults to 'message'
+ *
+ */
public String redisStreamJsonKey;
+
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} comma seperated list of redis sentials
+ */
public String redisSentinels;
- public String redisSentinelMasterName;
- public String redisPubChannel;
- public String redisListKey;
- public String redisStreamKey;
+ /**
+ * {@link com.zendesk.maxwell.producer.MaxwellRedisProducer} name of master redis sentinel
+ */
+ public String redisSentinelMasterName;
+ /**
+ * type of redis operation to perform: XADD, LPUSH, RPUSH, PUBSUB
+ */
public String redisType;
+
+ /**
+ * path to file containing javascript filtering functions
+ */
public String javascriptFile;
+
+ /**
+ * Instantiated by {@link #validate()}. Should be moved to MaxwellContext.
+ */
public Scripting scripting;
+ /**
+ * Enable high available support (via jgroups-raft)
+ */
public boolean haMode;
+
+ /**
+ * Path to raft.xml file that configures high availability support
+ */
public String jgroupsConf;
+
+ /**
+ * Defines membership within a HA cluster
+ */
public String raftMemberID;
+ /**
+ * The size for the queue used to buffer events parsed off binlog in
+ * {@link com.zendesk.maxwell.replication.BinlogConnectorReplicator}
+ */
+ public int binlogEventQueueSize;
+
+ /**
+ * Build a default configuration object.
+ */
public MaxwellConfig() { // argv is only null in tests
this.customProducerProperties = new Properties();
this.kafkaProperties = new Properties();
@@ -175,6 +657,10 @@ public MaxwellConfig() { // argv is only null in tests
setup(null, null); // setup defaults
}
+ /**
+ * build a configuration instance from command line arguments
+ * @param argv command line arguments
+ */
public MaxwellConfig(String argv[]) {
this();
this.parse(argv);
@@ -184,6 +670,8 @@ protected MaxwellOptionParser buildOptionParser() {
final MaxwellOptionParser parser = new MaxwellOptionParser();
parser.accepts( "config", "location of config.properties file" )
.withRequiredArg();
+ parser.accepts( "env_config", "json object encoded config in an environment variable" )
+ .withRequiredArg();
parser.separator();
@@ -206,7 +694,7 @@ protected MaxwellOptionParser buildOptionParser() {
parser.section("mysql");
parser.accepts( "binlog_heartbeat", "enable binlog replication heartbeats, default false" )
- .withOptionalArg().ofType(Boolean.class);
+ .withOptionalArg().ofType(Boolean.class);
parser.accepts( "jdbc_options", "additional jdbc connection options: key1=val1&key2=val2" )
.withRequiredArg();
@@ -222,6 +710,8 @@ protected MaxwellOptionParser buildOptionParser() {
.withRequiredArg();
parser.accepts( "replica_server_id", "server_id that maxwell reports to the master. See docs for full explanation. ")
.withRequiredArg().ofType(Long.class);
+ parser.accepts( "replication_reconnection_retries", "define how many time should replicator try reconnect, default 1, 0 = unlimited" )
+ .withOptionalArg().ofType(Integer.class);
parser.separator();
@@ -252,6 +742,57 @@ protected MaxwellOptionParser buildOptionParser() {
parser.separator();
parser.accepts( "max_schemas", "Maximum schema-updates to keep before triggering a compaction operation. Default: unlimited" )
.withRequiredArg();
+
+ parser.section( "output" );
+
+ parser.accepts( "output_binlog_position", "include 'position' (binlog position) field. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_gtid_position", "include 'gtid' (gtid position) field. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_commit_info", "include 'commit' and 'xid' field. default: true" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_xoffset", "include 'xoffset' (row offset inside transaction) field. depends on '--output_commit_info'. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_nulls", "include data fields with NULL values. default: true" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_server_id", "include 'server_id' field. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_thread_id", "include 'thread_id' (client thread_id) field. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_schema_id", "include 'schema_id' (unique ID for this DDL). default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_row_query", "include 'query' field (original SQL DML query). depends on server option 'binlog_rows_query_log_events'. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_primary_keys", "include 'primary_key' field (array of PK values). default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_primary_key_columns", "include 'primary_key_columns' field (array of PK column names). default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_null_zerodates", "convert '0000-00-00' dates/datetimes to null default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_ddl", "produce DDL records. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_push_timestamp", "include a microsecond timestamp representing when Maxwell sent a record. default: false" )
+ .withOptionalArg().ofType(Boolean.class);
+ parser.accepts( "output_naming_strategy", "optionally use an alternate name for fields: underscore_to_camelcase" )
+ .withOptionalArg().ofType(String.class);
+ parser.accepts( "exclude_columns", "suppress these comma-separated columns from output" )
+ .withRequiredArg();
+ parser.accepts("secret_key", "The secret key for the AES encryption" )
+ .withRequiredArg();
+ parser.accepts("encrypt", "encryption mode: [none|data|all]. default: none" )
+ .withRequiredArg();
+ parser.accepts( "row_query_max_length", "truncates the 'query' field if it is above this length. default: 0 (disabled)" )
+ .withOptionalArg().ofType(Integer.class);
+
+ parser.section( "filtering" );
+
+ parser.accepts( "filter", "filter specs. specify like \"include:db.*, exclude:*.tbl, include: foo./.*bar$/, exclude:foo.bar.baz=reject\"").withRequiredArg();
+
+ parser.accepts( "ignore_missing_schema", "Ignore missing database and table schemas. Only use running with limited permissions." )
+ .withOptionalArg().ofType(Boolean.class);
+
+ parser.accepts( "javascript", "file containing per-row javascript to execute" ).withRequiredArg();
+
parser.section("operation");
parser.accepts( "daemon", "run maxwell in the background" )
@@ -290,6 +831,8 @@ protected MaxwellOptionParser buildOptionParser() {
.withOptionalArg().ofType(Boolean.class);
parser.accepts( "buffer_memory_usage", "Percentage of JVM memory available for transaction buffer. Floating point between 0 and 1." )
.withRequiredArg().ofType(Float.class);
+ parser.accepts("binlog_event_queue_size", "Size of queue to buffer events parsed from binlog.")
+ .withOptionalArg().ofType(Integer.class);
parser.section( "custom_producer" );
parser.accepts( "custom_producer.factory", "fully qualified custom producer factory class" )
@@ -319,7 +862,7 @@ protected MaxwellOptionParser buildOptionParser() {
parser.separator();
- parser.accepts( "kafka_version", "kafka client library version: 0.8.2.2|0.9.0.1|0.10.0.1|0.10.2.1|0.11.0.1|1.0.0")
+ parser.accepts( "kafka_version", "kafka client library version: 0.8.2.2|0.9.0.1|0.10.0.1|0.10.2.1|0.11.0.1|1.0.0|2.7.0|3.4.0")
.withRequiredArg();
parser.accepts( "kafka_key_format", "how to format the kafka key; array|hash" )
.withRequiredArg();
@@ -328,23 +871,28 @@ protected MaxwellOptionParser buildOptionParser() {
parser.accepts( "dead_letter_topic", "write to this topic when unable to publish a row for known reasons (eg message is too big)" )
.withRequiredArg();
- parser.accepts( "kafka_partition_by", "[deprecated]").withRequiredArg();
- parser.accepts( "kafka_partition_columns", "[deprecated]").withRequiredArg();
- parser.accepts( "kafka_partition_by_fallback", "[deprecated]").withRequiredArg();
-
parser.accepts( "ddl_kafka_topic", "public DDL (schema change) events to this topic. default: kafka_topic ( see also --output_ddl )" )
.withRequiredArg();
parser.section( "kinesis" );
parser.accepts( "kinesis_stream", "kinesis stream name" )
.withOptionalArg();
+
+ parser.section("sqs");
parser.accepts( "sqs_queue_uri", "SQS Queue uri" )
.withRequiredArg();
+ parser.accepts( "sqs_service_endpoint", "SQS Service Endpoint" )
+ .withRequiredArg();
+ parser.accepts( "sqs_signing_region", "SQS Signing region" )
+ .withRequiredArg();
+
+ parser.section("sns");
parser.accepts("sns_topic", "SNS Topic ARN")
.withRequiredArg();
- parser.accepts("sns_attrs", "Fields to add as message attributes")
+ parser.accepts("sns_attrs", "Comma separated fields to add as message attributes: \"database, table\"")
.withOptionalArg();
parser.separator();
+
parser.addToSection("producer_partition_by");
parser.addToSection("producer_partition_columns");
parser.addToSection("producer_partition_by_fallback");
@@ -355,6 +903,14 @@ protected MaxwellOptionParser buildOptionParser() {
parser.accepts( "nats_url", "Url(s) of Nats connection (comma separated). Default is localhost:4222" ).withRequiredArg();
parser.accepts( "nats_subject", "Subject Hierarchies of Nats. Default is '%{database}.%{table}'" ).withRequiredArg();
+ parser.section( "bigquery" );
+ parser.accepts( "bigquery_project_id", "provide a google cloud platform project id associated with the bigquery table" )
+ .withRequiredArg();
+ parser.accepts( "bigquery_dataset", "provide a google cloud platform dataset id associated with the bigquery table" )
+ .withRequiredArg();
+ parser.accepts( "bigquery_table", "provide a google cloud platform table id associated with the bigquery table" )
+ .withRequiredArg();
+
parser.section( "pubsub" );
parser.accepts( "pubsub_project_id", "provide a google cloud platform project id associated with the pubsub topic" )
.withRequiredArg();
@@ -366,73 +922,26 @@ protected MaxwellOptionParser buildOptionParser() {
.withRequiredArg().ofType(Long.class);
parser.accepts( "pubsub_message_count_batch_size", "threshold in message count that triggers a batch to be sent. default: 1 message" )
.withRequiredArg().ofType(Long.class);
+ parser.accepts( "pubsub_message_ordering_key", "message ordering key template (will enable message ordering if specified). default: null" )
+ .withOptionalArg();
parser.accepts( "pubsub_publish_delay_threshold", "threshold in delay time (milliseconds) before batch is sent. default: 1 ms" )
.withRequiredArg().ofType(Long.class);
parser.accepts( "pubsub_retry_delay", "delay in millis before sending the first retry message. default: 100 ms" )
.withRequiredArg().ofType(Long.class);
parser.accepts( "pubsub_retry_delay_multiplier", "multiply by this ratio to increase delay time each retry. default: 1.3" )
- .withRequiredArg();
+ .withRequiredArg().ofType(Float.class);
parser.accepts( "pubsub_max_retry_delay", "maximum retry delay time in seconds. default: 60 seconds" )
.withRequiredArg().ofType(Long.class);
parser.accepts( "pubsub_initial_rpc_timeout", "timeout for initial rpc call. default: 5 seconds" )
.withRequiredArg();
parser.accepts( "pubsub_rpc_timeout_multiplier", "backoff delay ratio for rpc timeout. default: 1.0" )
- .withRequiredArg().ofType(Long.class);
+ .withRequiredArg().ofType(Float.class);
parser.accepts( "pubsub_max_rpc_timeout", "max delay in seconds for rpc timeout. default: 600 seconds" )
.withRequiredArg().ofType(Long.class);
parser.accepts( "pubsub_total_timeout", "maximum timeout in seconds (clamps exponential backoff)" )
.withRequiredArg().ofType(Long.class);
-
- parser.section( "output" );
-
- parser.accepts( "output_binlog_position", "include 'position' (binlog position) field. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_gtid_position", "include 'gtid' (gtid position) field. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_commit_info", "include 'commit' and 'xid' field. default: true" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_xoffset", "include 'xoffset' (row offset inside transaction) field. depends on '--output_commit_info'. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_nulls", "include data fields with NULL values. default: true" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_server_id", "include 'server_id' field. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_thread_id", "include 'thread_id' (client thread_id) field. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_schema_id", "include 'schema_id' (unique ID for this DDL). default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_row_query", "include 'query' field (original SQL DML query). depends on server option 'binlog_rows_query_log_events'. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_primary_keys", "include 'primary_key' field (array of PK values). default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_primary_key_columns", "include 'primary_key_columns' field (array of PK column names). default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_null_zerodates", "convert '0000-00-00' dates/datetimes to null default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_ddl", "produce DDL records. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "output_push_timestamp", "include a microsecond timestamp representing when Maxwell sent a record. default: false" )
- .withOptionalArg().ofType(Boolean.class);
- parser.accepts( "exclude_columns", "suppress these comma-separated columns from output" )
- .withRequiredArg();
- parser.accepts("secret_key", "The secret key for the AES encryption" )
- .withRequiredArg();
- parser.accepts("encrypt", "encryption mode: [none|data|all]. default: none" )
- .withRequiredArg();
-
- parser.section( "filtering" );
-
- parser.accepts( "include_dbs", "[deprecated]" ).withRequiredArg();
- parser.accepts( "exclude_dbs", "[deprecated]" ).withRequiredArg();
- parser.accepts( "include_tables", "[deprecated]" ).withRequiredArg();
- parser.accepts( "exclude_tables", "[deprecated]" ).withRequiredArg();
- parser.accepts( "blacklist_dbs", "[deprecated]" ).withRequiredArg();
- parser.accepts( "blacklist_tables", "[deprecated]" ).withRequiredArg();
-
- parser.accepts( "filter", "filter specs. specify like \"include:db.*, exclude:*.tbl, include: foo./.*bar$/, exclude:foo.bar.baz=reject\"").withRequiredArg();
-
- parser.accepts( "include_column_values", "[deprecated]" ).withRequiredArg();
- parser.accepts( "javascript", "file containing per-row javascript to execute" ).withRequiredArg();
+ parser.accepts( "pubsub_emulator", "pubsub emulator host to use. default: null" )
+ .withOptionalArg();
parser.section( "rabbitmq" );
@@ -441,6 +950,7 @@ protected MaxwellOptionParser buildOptionParser() {
parser.accepts( "rabbitmq_host", "Host of Rabbitmq machine" ).withRequiredArg();
parser.accepts( "rabbitmq_port", "Port of Rabbitmq machine" ).withRequiredArg().ofType(Integer.class);
parser.accepts( "rabbitmq_uri", "URI to rabbit server, eg amqp://, amqps://. other rabbitmq options take precendence over uri." ).withRequiredArg();
+ parser.accepts( "rabbitmq_handshake_timeout", "Handshake timeout of Rabbitmq connection in milliseconds" ).withOptionalArg().ofType(Integer.class);;
parser.accepts( "rabbitmq_virtual_host", "Virtual Host of Rabbitmq" ).withRequiredArg();
parser.accepts( "rabbitmq_exchange", "Name of exchange for rabbitmq publisher" ).withRequiredArg();
parser.accepts( "rabbitmq_exchange_type", "Exchange type for rabbitmq" ).withRequiredArg();
@@ -449,6 +959,7 @@ protected MaxwellOptionParser buildOptionParser() {
parser.accepts( "rabbitmq_routing_key_template", "A string template for the routing key, '%db%' and '%table%' will be substituted. Default is '%db%.%table%'." ).withRequiredArg();
parser.accepts( "rabbitmq_message_persistent", "Message persistence. Defaults to false" ).withOptionalArg();
parser.accepts( "rabbitmq_declare_exchange", "Should declare the exchange for rabbitmq publisher. Defaults to true" ).withOptionalArg();
+ parser.accepts( "rabbitmq_use_ssl", "If true, will connect to the server using SSL. Defaults to false" ).withOptionalArg();
parser.section( "redis" );
@@ -462,11 +973,7 @@ protected MaxwellOptionParser buildOptionParser() {
parser.accepts("redis_sentinels", "List of Redis sentinels in format host1:port1,host2:port2,host3:port3. It can be used instead of redis_host and redis_port" ).withRequiredArg();
parser.accepts("redis_sentinel_master_name", "Redis sentinel master name. It is used with redis_sentinels" ).withRequiredArg();
- parser.accepts( "redis_pub_channel", "[deprecated]" ).withRequiredArg();
- parser.accepts( "redis_stream_key", "[deprecated]" ).withRequiredArg();
- parser.accepts( "redis_list_key", "[deprecated]" ).withRequiredArg();
-
- parser.section("metrics");
+ parser.section("monitoring");
parser.accepts( "metrics_prefix", "the prefix maxwell will apply to all metrics" ).withRequiredArg();
parser.accepts( "metrics_type", "how maxwell metrics will be reported, at least one of slf4j|jmx|http|datadog|stackdriver" ).withRequiredArg();
@@ -480,9 +987,8 @@ protected MaxwellOptionParser buildOptionParser() {
parser.accepts( "metrics_datadog_site", "the site to publish metrics to when metrics_datadog_type = http, one of us|eu, default us" ).withRequiredArg();
parser.accepts( "metrics_datadog_host", "the host to publish metrics to when metrics_datadog_type = udp" ).withRequiredArg();
parser.accepts( "metrics_datadog_port", "the port to publish metrics to when metrics_datadog_type = udp" ).withRequiredArg().ofType(Integer.class);
+ parser.accepts( "custom_health.factory", "fully qualified custom maxwell health check").withRequiredArg();
-
- parser.section("http");
parser.accepts( "http_port", "the port the server will bind to when http reporting is configured" ).withRequiredArg().ofType(Integer.class);
parser.accepts( "http_path_prefix", "the http path prefix when metrics_type includes http or diagnostic is enabled, default /" ).withRequiredArg();
parser.accepts( "http_bind_address", "the ip address the server will bind to when http reporting is configured" ).withRequiredArg();
@@ -500,7 +1006,7 @@ private void parse(String [] argv) {
MaxwellOptionParser parser = buildOptionParser();
OptionSet options = parser.parse(argv);
- Properties properties;
+ final Properties properties;
if (options.has("config")) {
properties = parseFile((String) options.valueOf("config"), true);
@@ -508,13 +1014,31 @@ private void parse(String [] argv) {
properties = parseFile(DEFAULT_CONFIG_FILE, false);
}
+ if (options.has("env_config")) {
+ Properties envConfigProperties = readPropertiesEnv((String) options.valueOf("env_config"));
+ for (Map.Entry entry : envConfigProperties.entrySet()) {
+ Object key = entry.getKey();
+ if (properties.put(key, entry.getValue()) != null) {
+ LOGGER.debug("Replaced config key {} with value from env_config", key);
+ }
+ }
+ }
+
String envConfigPrefix = fetchStringOption("env_config_prefix", options, properties, null);
if (envConfigPrefix != null) {
String prefix = envConfigPrefix.toLowerCase();
System.getenv().entrySet().stream()
.filter(map -> map.getKey().toLowerCase().startsWith(prefix))
- .forEach(config -> properties.put(config.getKey().toLowerCase().replaceFirst(prefix, ""), config.getValue()));
+ .forEach(config -> {
+ String rawKey = config.getKey();
+ String newKey = rawKey.toLowerCase().replaceFirst(prefix, "");
+ if (properties.put(newKey, config.getValue()) != null) {
+ LOGGER.debug("Got env variable {} and replacing config key {}", rawKey, newKey);
+ } else {
+ LOGGER.debug("Got env variable {} as config key {}", rawKey, newKey);
+ }
+ });
}
if (options.has("help"))
@@ -550,26 +1074,29 @@ private void setup(OptionSet options, Properties properties) {
this.kafkaTopic = fetchStringOption("kafka_topic", options, properties, "maxwell");
this.deadLetterTopic = fetchStringOption("dead_letter_topic", options, properties, null);
this.kafkaKeyFormat = fetchStringOption("kafka_key_format", options, properties, "hash");
- this.kafkaPartitionKey = fetchStringOption("kafka_partition_by", options, properties, null);
- this.kafkaPartitionColumns = fetchStringOption("kafka_partition_columns", options, properties, null);
- this.kafkaPartitionFallback = fetchStringOption("kafka_partition_by_fallback", options, properties, null);
this.kafkaPartitionHash = fetchStringOption("kafka_partition_hash", options, properties, "default");
this.ddlKafkaTopic = fetchStringOption("ddl_kafka_topic", options, properties, this.kafkaTopic);
+ this.bigQueryProjectId = fetchStringOption("bigquery_project_id", options, properties, null);
+ this.bigQueryDataset = fetchStringOption("bigquery_dataset", options, properties, null);
+ this.bigQueryTable = fetchStringOption("bigquery_table", options, properties, null);
+
this.pubsubProjectId = fetchStringOption("pubsub_project_id", options, properties, null);
this.pubsubTopic = fetchStringOption("pubsub_topic", options, properties, "maxwell");
this.ddlPubsubTopic = fetchStringOption("ddl_pubsub_topic", options, properties, this.pubsubTopic);
this.pubsubRequestBytesThreshold = fetchLongOption("pubsub_request_bytes_threshold", options, properties, 1L);
this.pubsubMessageCountBatchSize = fetchLongOption("pubsub_message_count_batch_size", options, properties, 1L);
+ this.pubsubMessageOrderingKey = fetchStringOption("pubsub_message_ordering_key", options, properties, null);
this.pubsubPublishDelayThreshold = Duration.ofMillis(fetchLongOption("pubsub_publish_delay_threshold", options, properties, 1L));
this.pubsubRetryDelay = Duration.ofMillis(fetchLongOption("pubsub_retry_delay", options, properties, 100L));
- this.pubsubRetryDelayMultiplier = Double.parseDouble(fetchStringOption("pubsub_retry_delay_multiplier", options, properties, "1.3"));
+ this.pubsubRetryDelayMultiplier = fetchFloatOption("pubsub_retry_delay_multiplier", options, properties, 1.3f);
this.pubsubMaxRetryDelay = Duration.ofSeconds(fetchLongOption("pubsub_max_retry_delay", options, properties, 60L));
this.pubsubInitialRpcTimeout = Duration.ofSeconds(fetchLongOption("pubsub_initial_rpc_timeout", options, properties, 5L));
- this.pubsubRpcTimeoutMultiplier = Double.parseDouble(fetchStringOption("pubsub_rpc_timeout_multiplier", options, properties, "1.0"));
+ this.pubsubRpcTimeoutMultiplier = fetchFloatOption("pubsub_rpc_timeout_multiplier", options, properties, 1.0f);
this.pubsubMaxRpcTimeout = Duration.ofSeconds(fetchLongOption("pubsub_max_rpc_timeout", options, properties, 600L));
this.pubsubTotalTimeout = Duration.ofSeconds(fetchLongOption("pubsub_total_timeout", options, properties, 600L));
+ this.pubsubEmulator = fetchStringOption("pubsub_emulator", options, properties, null);
this.rabbitmqHost = fetchStringOption("rabbitmq_host", options, properties, null);
this.rabbitmqPort = fetchIntegerOption("rabbitmq_port", options, properties, null);
@@ -577,6 +1104,7 @@ private void setup(OptionSet options, Properties properties) {
this.rabbitmqPass = fetchStringOption("rabbitmq_pass", options, properties, "guest");
this.rabbitmqVirtualHost = fetchStringOption("rabbitmq_virtual_host", options, properties, "/");
this.rabbitmqURI = fetchStringOption("rabbitmq_uri", options, properties, null);
+ this.rabbitmqHandshakeTimeout = fetchIntegerOption("rabbitmq_handshake_timeout", options, properties, null);
this.rabbitmqExchange = fetchStringOption("rabbitmq_exchange", options, properties, "maxwell");
this.rabbitmqExchangeType = fetchStringOption("rabbitmq_exchange_type", options, properties, "fanout");
this.rabbitMqExchangeDurable = fetchBooleanOption("rabbitmq_exchange_durable", options, properties, false);
@@ -584,6 +1112,7 @@ private void setup(OptionSet options, Properties properties) {
this.rabbitmqRoutingKeyTemplate = fetchStringOption("rabbitmq_routing_key_template", options, properties, "%db%.%table%");
this.rabbitmqMessagePersistent = fetchBooleanOption("rabbitmq_message_persistent", options, properties, false);
this.rabbitmqDeclareExchange = fetchBooleanOption("rabbitmq_declare_exchange", options, properties, true);
+ this.rabbitmqUseSSL = fetchBooleanOption("rabbitmq_use_ssl", options, properties, false);
this.natsUrl = fetchStringOption("nats_url", options, properties, "nats://localhost:4222");
this.natsSubject = fetchStringOption("nats_subject", options, properties, "%{database}.%{table}");
@@ -599,11 +1128,6 @@ private void setup(OptionSet options, Properties properties) {
this.redisSentinels = fetchStringOption("redis_sentinels", options, properties, null);
this.redisSentinelMasterName = fetchStringOption("redis_sentinel_master_name", options, properties, null);
- // deprecated options
- this.redisPubChannel = fetchStringOption("redis_pub_channel", options, properties, null);
- this.redisListKey = fetchStringOption("redis_list_key", options, properties, null);
- this.redisStreamKey = fetchStringOption("redis_stream_key", options, properties, null);
-
this.redisType = fetchStringOption("redis_type", options, properties, "pubsub");
String kafkaBootstrapServers = fetchStringOption("kafka.bootstrap.servers", options, properties, null);
@@ -615,6 +1139,8 @@ private void setup(OptionSet options, Properties properties) {
String k = (String) e.nextElement();
if (k.startsWith("custom_producer.")) {
this.customProducerProperties.setProperty(k.replace("custom_producer.", ""), properties.getProperty(k));
+ } else if (k.startsWith("custom_producer_")) {
+ this.customProducerProperties.setProperty(k.replace("custom_producer_", ""), properties.getProperty(k));
} else if (k.startsWith("kafka.")) {
if (k.equals("kafka.bootstrap.servers") && kafkaBootstrapServers != null)
continue; // don't override command line bootstrap servers with config files'
@@ -632,6 +1158,8 @@ private void setup(OptionSet options, Properties properties) {
this.kinesisMd5Keys = fetchBooleanOption("kinesis_md5_keys", options, properties, false);
this.sqsQueueUri = fetchStringOption("sqs_queue_uri", options, properties, null);
+ this.sqsServiceEndpoint = fetchStringOption("sqs_service_endpoint", options, properties, null);
+ this.sqsSigningRegion = fetchStringOption("sqs_signing_region", options, properties, null);
this.snsTopic = fetchStringOption("sns_topic", options, properties, null);
this.snsAttrs = fetchStringOption("sns_attrs", options, properties, null);
@@ -641,6 +1169,7 @@ private void setup(OptionSet options, Properties properties) {
this.metricsReportingType = fetchStringOption("metrics_type", options, properties, null);
this.metricsSlf4jInterval = fetchLongOption("metrics_slf4j_interval", options, properties, 60L);
+ this.customHealthFactory = fetchHealthCheckFactory(options, properties);
this.httpPort = fetchIntegerOption("http_port", options, properties, 8080);
this.httpBindAddress = fetchStringOption("http_bind_address", options, properties, null);
this.httpPathPrefix = fetchStringOption("http_path_prefix", options, properties, "/");
@@ -664,14 +1193,8 @@ private void setup(OptionSet options, Properties properties) {
this.enableHttpConfig = fetchBooleanOption("http_config", options, properties, false);
- this.includeDatabases = fetchStringOption("include_dbs", options, properties, null);
- this.excludeDatabases = fetchStringOption("exclude_dbs", options, properties, null);
- this.includeTables = fetchStringOption("include_tables", options, properties, null);
- this.excludeTables = fetchStringOption("exclude_tables", options, properties, null);
- this.blacklistDatabases = fetchStringOption("blacklist_dbs", options, properties, null);
- this.blacklistTables = fetchStringOption("blacklist_tables", options, properties, null);
this.filterList = fetchStringOption("filter", options, properties, null);
- this.includeColumnValues = fetchStringOption("include_column_values", options, properties, null);
+ this.ignoreMissingSchema = fetchBooleanOption("ignore_missing_schema", options, properties, false);
setupInitPosition(options);
@@ -691,6 +1214,7 @@ private void setup(OptionSet options, Properties properties) {
outputConfig.includesThreadId = fetchBooleanOption("output_thread_id", options, properties, false);
outputConfig.includesSchemaId = fetchBooleanOption("output_schema_id", options, properties, false);
outputConfig.includesRowQuery = fetchBooleanOption("output_row_query", options, properties, false);
+ outputConfig.rowQueryMaxLength = fetchIntegerOption("row_query_max_length", options, properties, 0);
outputConfig.includesPrimaryKeys = fetchBooleanOption("output_primary_keys", options, properties, false);
outputConfig.includesPrimaryKeyColumns = fetchBooleanOption("output_primary_key_columns", options, properties, false);
outputConfig.includesPushTimestamp = fetchBooleanOption("output_push_timestamp", options, properties, false);
@@ -704,6 +1228,9 @@ private void setup(OptionSet options, Properties properties) {
this.haMode = fetchBooleanOption("ha", options, properties, false);
this.jgroupsConf = fetchStringOption("jgroups_config", options, properties, "raft.xml");
this.raftMemberID = fetchStringOption("raft_member_id", options, properties, null);
+ this.replicationReconnectionRetries = fetchIntegerOption("replication_reconnection_retries", options, properties, 1);
+
+ this.binlogEventQueueSize = fetchIntegerOption("binlog_event_queue_size", options, properties, BinlogConnectorReplicator.BINLOG_QUEUE_SIZE);
}
private void setupEncryptionOptions(OptionSet options, Properties properties) {
@@ -766,21 +1293,6 @@ private Properties parseFile(String filename, Boolean abortOnMissing) {
}
private void validatePartitionBy() {
- if ( this.producerPartitionKey == null && this.kafkaPartitionKey != null ) {
- LOGGER.warn("kafka_partition_by is deprecated, please use producer_partition_by");
- this.producerPartitionKey = this.kafkaPartitionKey;
- }
-
- if ( this.producerPartitionColumns == null && this.kafkaPartitionColumns != null) {
- LOGGER.warn("kafka_partition_columns is deprecated, please use producer_partition_columns");
- this.producerPartitionColumns = this.kafkaPartitionColumns;
- }
-
- if ( this.producerPartitionFallback == null && this.kafkaPartitionFallback != null ) {
- LOGGER.warn("kafka_partition_by_fallback is deprecated, please use producer_partition_by_fallback");
- this.producerPartitionFallback = this.kafkaPartitionFallback;
- }
-
String[] validPartitionBy = {"database", "table", "primary_key", "transaction_id", "thread_id", "column", "random"};
if ( this.producerPartitionKey == null ) {
this.producerPartitionKey = "database";
@@ -801,35 +1313,16 @@ private void validateFilter() {
if ( this.filterList != null ) {
this.filter = new Filter(this.databaseName, filterList);
} else {
- boolean hasOldStyleFilters =
- includeDatabases != null ||
- excludeDatabases != null ||
- includeTables != null ||
- excludeTables != null ||
- blacklistDatabases != null ||
- blacklistTables != null ||
- includeColumnValues != null;
-
- if ( hasOldStyleFilters ) {
- this.filter = Filter.fromOldFormat(
- this.databaseName,
- includeDatabases,
- excludeDatabases,
- includeTables,
- excludeTables,
- blacklistDatabases,
- blacklistTables,
- includeColumnValues
- );
- } else {
- this.filter = new Filter(this.databaseName, "");
- }
+ this.filter = new Filter(this.databaseName, "");
}
} catch (InvalidFilterException e) {
usageForOptions("Invalid filter options: " + e.getLocalizedMessage(), "filter");
}
}
+ /**
+ * Validate the maxwell configuration, exiting with an error message if invalid.
+ */
public void validate() {
validatePartitionBy();
validateFilter();
@@ -856,9 +1349,16 @@ public void validate() {
usageForOptions("please specify a stream name for kinesis", "kinesis_stream");
} else if (this.producerType.equals("sqs") && this.sqsQueueUri == null) {
usageForOptions("please specify a queue uri for sqs", "sqs_queue_uri");
+ } else if (this.producerType.equals("sqs") && this.sqsServiceEndpoint == null) {
+ usageForOptions("please specify a service endpoint for sqs", "sqs_service_endpoint");
+ } else if (this.producerType.equals("sqs") && this.sqsSigningRegion == null) {
+ usageForOptions("please specify a signing region for sqs", "sqs_signing_region");
} else if (this.producerType.equals("sns") && this.snsTopic == null) {
usageForOptions("please specify a topic ARN for SNS", "sns_topic");
} else if (this.producerType.equals("pubsub")) {
+ if (this.pubsubProjectId == null)
+ usageForOptions("please specify --pubsub_project_id.", "--pubsub_project_id");
+
if (this.pubsubRequestBytesThreshold <= 0L)
usage("--pubsub_request_bytes_threshold must be > 0");
if (this.pubsubMessageCountBatchSize <= 0L)
@@ -880,17 +1380,6 @@ public void validate() {
if (this.pubsubTotalTimeout.isNegative() || this.pubsubTotalTimeout.isZero())
usage("--pubsub_total_timeout must be > 0");
} else if (this.producerType.equals("redis")) {
- if ( this.redisPubChannel != null ) {
- LOGGER.warn("--redis_pub_channel is deprecated, please use redis_key");
- this.redisKey = this.redisPubChannel;
- } else if ( this.redisListKey != null ) {
- LOGGER.warn("--redis_list_key is deprecated, please use redis_key");
- this.redisKey = this.redisListKey;
- } else if ( this.redisStreamKey != null ) {
- LOGGER.warn("--redis_stream_key is deprecated, please use redis_key");
- this.redisKey = this.redisStreamKey;
- }
-
if ( this.redisKey == null ) {
usage("please specify --redis_key=KEY");
}
@@ -1001,11 +1490,15 @@ public void validate() {
}
}
+ /**
+ * return a filtered list of properties for the Kafka producer
+ * @return Properties object containing all kafka properties found in config.properties
+ */
public Properties getKafkaProperties() {
return this.kafkaProperties;
}
- public static Pattern compileStringToPattern(String name) throws InvalidFilterException {
+ private static Pattern compileStringToPattern(String name) throws InvalidFilterException {
name = name.trim();
if ( name.startsWith("/") ) {
if ( !name.endsWith("/") ) {
@@ -1017,21 +1510,58 @@ public static Pattern compileStringToPattern(String name) throws InvalidFilterEx
}
}
- protected ProducerFactory fetchProducerFactory(OptionSet options, Properties properties) {
- String name = "custom_producer.factory";
+ private T fetchFactory(OptionSet options, Properties properties, String name) {
String strOption = fetchStringOption(name, options, properties, null);
if ( strOption != null ) {
try {
Class clazz = Class.forName(strOption);
- return ProducerFactory.class.cast(clazz.newInstance());
+ Class[] carg = new Class[0];
+ Constructor ct = clazz.getDeclaredConstructor(carg);
+ return (T) ct.newInstance();
} catch ( ClassNotFoundException e ) {
usageForOptions("Invalid value for " + name + ", class '" + strOption + "' not found", "--" + name);
} catch ( IllegalAccessException | InstantiationException | ClassCastException e) {
usageForOptions("Invalid value for " + name + ", class instantiation error", "--" + name);
+ } catch (NoSuchMethodException e) {
+ usageForOptions("No valid constructor found for " + strOption, "--" + name);
+ } catch (InvocationTargetException e) {
+ String msg = String.format("Unable to construct customer producer '%s'", strOption);
+ usageForOptions(msg, "--" + name);
+ e.printStackTrace();
}
return null; // unreached
} else {
return null;
}
+
+ }
+ /**
+ * If present in the configuration, build an instance of a custom producer factor
+ * @param options command line arguments
+ * @param properties properties from config.properties
+ * @return NULL or ProducerFactory instance
+ */
+ protected ProducerFactory fetchProducerFactory(OptionSet options, Properties properties) {
+ return fetchFactory(options, properties, "custom_producer.factory");
+ }
+
+
+ /**
+ * If present in the configuration, build an instance of a custom health factory
+ * @param options command line arguments
+ * @param properties properties from config.properties
+ * @return NULL or MaxwellHealthCheckFactory instance
+ */
+ protected MaxwellHealthCheckFactory fetchHealthCheckFactory(OptionSet options, Properties properties) {
+ return fetchFactory(options, properties, "custom_health.factory");
+ }
+
+
+ public Boolean getIgnoreMissingSchema() {
+ return ignoreMissingSchema;
+ }
+
+ public void setIgnoreMissingSchema(Boolean ignoreMissingSchema) {
+ this.ignoreMissingSchema = ignoreMissingSchema;
}
}
diff --git a/src/main/java/com/zendesk/maxwell/MaxwellContext.java b/src/main/java/com/zendesk/maxwell/MaxwellContext.java
index ca2d7882a..1da30273f 100644
--- a/src/main/java/com/zendesk/maxwell/MaxwellContext.java
+++ b/src/main/java/com/zendesk/maxwell/MaxwellContext.java
@@ -27,11 +27,15 @@
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+/**
+ * Class containing runtime state
+ */
public class MaxwellContext {
static final Logger LOGGER = LoggerFactory.getLogger(MaxwellContext.class);
@@ -60,9 +64,24 @@ public class MaxwellContext {
private BootstrapController bootstrapController;
private Thread bootstrapControllerThread;
+ private Boolean isMariaDB;
+
+ /**
+ * Contains various Maxwell metrics
+ */
public MetricRegistry metricRegistry;
+
+ /**
+ * Contains Maxwell health checks
+ */
public HealthCheckRegistry healthCheckRegistry;
+ /**
+ * Create a runtime context from a configuration object
+ * @param config Maxwell configuration
+ * @throws SQLException if there's issues connecting to the database
+ * @throws URISyntaxException if there's issues building database URIs
+ */
public MaxwellContext(MaxwellConfig config) throws SQLException, URISyntaxException {
this.config = config;
this.config.validate();
@@ -123,17 +142,39 @@ public MaxwellContext(MaxwellConfig config) throws SQLException, URISyntaxExcept
this.healthCheckRegistry = new HealthCheckRegistry();
}
+ /**
+ * Get Maxwell configuration used in this context
+ * @return the Maxwell configuration
+ */
public MaxwellConfig getConfig() {
return this.config;
}
+ /**
+ * Get the a connection from the replication pool
+ * @return a connection to the replication pool
+ * @throws SQLException if we can't connect
+ */
public Connection getReplicationConnection() throws SQLException {
return this.replicationConnectionPool.getConnection();
}
+ /**
+ * Get the replication pool
+ * @return the replication (connection to replicate from) connection pool
+ */
public ConnectionPool getReplicationConnectionPool() { return replicationConnectionPool; }
+
+ /**
+ * Get a connection from the maxwell (metadata) pool
+ * @return the maxwell (connection to store metadata in) connection pool
+ */
public ConnectionPool getMaxwellConnectionPool() { return maxwellConnectionPool; }
+ /**
+ * Get a connection from the schema pool
+ * @return the schema (connection to capture from) connection pool
+ */
public ConnectionPool getSchemaConnectionPool() {
if (this.schemaConnectionPool != null) {
return schemaConnectionPool;
@@ -142,31 +183,87 @@ public ConnectionPool getSchemaConnectionPool() {
return replicationConnectionPool;
}
+ /**
+ * get a connection from the maxwell pool
+ * @return a connection from the maxwell pool
+ * @throws SQLException if we can't connect
+ */
public Connection getMaxwellConnection() throws SQLException {
return this.maxwellConnectionPool.getConnection();
}
+ /**
+ * get a database-less connection from the maxwell pool
+ *
+ * Used to create the maxwell schema.
+ * @return a connection to the maxwell pool, without a database name specific
+ * @throws SQLException if we can't connect
+ */
public Connection getRawMaxwellConnection() throws SQLException {
return rawMaxwellConnectionPool.getConnection();
}
+ /**
+ * Start the HTTP server and position store thread
+ * @throws IOException if the HTTP server can't be started
+ */
public void start() throws IOException {
MaxwellHTTPServer.startIfRequired(this);
- getPositionStoreThread(); // boot up thread explicitly.
+ getPositionStoreThread();
}
+ /**
+ * Manually trigger a heartbeat to be sent
+ * @return Timestamp of the heartbeeat
+ * @throws Exception If we can't send a heartbeat
+ */
public long heartbeat() throws Exception {
return this.positionStore.heartbeat();
}
+ /**
+ * Add a task (usually a thread) that will be stopped upon shutdown
+ * @param task The task
+ */
public void addTask(StoppableTask task) {
this.taskManager.add(task);
}
+ /**
+ * Begin the maxwell shutdown process.
+ *
+ *
Shuts down the {@link #replicator}
+ *
Calls {@link TaskManager#stop}
+ *
Stops metrics collection
+ *
Destroys all database pools
+ *
+ * @return A thread that will complete shutdown.
+ */
public Thread terminate() {
return terminate(null);
}
+ /**
+ * Begin the Maxwell shutdown process
+ * @param error An exception that caused the shutdown, or null
+ * @return A thread that will complete shutdown.
+ * @see #terminate()
+ */
+ public Thread terminate(Exception error) {
+ if (this.error == null) {
+ this.error = error;
+ }
+
+ if (taskManager.requestStop()) {
+ if (this.error == null && this.replicator != null) {
+ sendFinalHeartbeat();
+ }
+ this.terminationThread = spawnTerminateThread();
+ }
+ return this.terminationThread;
+ }
+
+
private void sendFinalHeartbeat() {
long heartbeat = System.currentTimeMillis();
LOGGER.info("Sending final heartbeat: " + heartbeat);
@@ -186,7 +283,7 @@ private void sendFinalHeartbeat() {
}
}
- public void shutdown(AtomicBoolean complete) {
+ private void shutdown(AtomicBoolean complete) {
try {
taskManager.stop(this.error);
this.metrics.stop();
@@ -231,8 +328,9 @@ public void run() {
// ignore
}
- LOGGER.debug("Shutdown complete: " + shutdownComplete.get());
- if (!shutdownComplete.get()) {
+ final boolean isShutdownComplete = shutdownComplete.get();
+ LOGGER.debug("Shutdown complete: {}", isShutdownComplete);
+ if (!isShutdownComplete) {
LOGGER.error("Shutdown stalled - forcefully killing maxwell process");
if (self.error != null) {
LOGGER.error("Termination reason:", self.error);
@@ -246,21 +344,7 @@ public void run() {
return thread;
}
- public Thread terminate(Exception error) {
- if (this.error == null) {
- this.error = error;
- }
-
- if (taskManager.requestStop()) {
- if (this.error == null && this.replicator != null) {
- sendFinalHeartbeat();
- }
- this.terminationThread = spawnTerminateThread();
- }
- return this.terminationThread;
- }
-
- public Thread startTask(RunLoopProcess task, String name) {
+ private Thread startTask(RunLoopProcess task, String name) {
Thread t = new Thread(() -> {
try {
task.runLoop();
@@ -279,10 +363,18 @@ public Thread startTask(RunLoopProcess task, String name) {
return t;
}
+ /**
+ * Get the Exception that triggered shutdown
+ * @return An error that caused maxwell to shutdown
+ */
public Exception getError() {
return error;
}
+ /**
+ * Get or spawn a thread that persists the current position into the metadata database.
+ * @return Position store thread
+ */
public PositionStoreThread getPositionStoreThread() {
if ( this.positionStoreThread == null ) {
this.positionStoreThread = new PositionStoreThread(this.positionStore, this);
@@ -293,6 +385,11 @@ public PositionStoreThread getPositionStoreThread() {
}
+ /**
+ * Retrieve Maxwell's starting position from the metadata database
+ * @return The initial binlog position
+ * @throws SQLException If the position can't be retrieved from the database
+ */
public Position getInitialPosition() throws SQLException {
if ( this.initialPosition != null )
return this.initialPosition;
@@ -301,19 +398,39 @@ public Position getInitialPosition() throws SQLException {
return this.initialPosition;
}
+ /**
+ * Finds the most recent position any client has reached on the server
+ * @return A binlog position or NULL
+ * @throws SQLException If an error is encountered fetching the position
+ * @see MysqlPositionStore#getLatestFromAnyClient()
+ */
public Position getOtherClientPosition() throws SQLException {
return this.positionStore.getLatestFromAnyClient();
}
+ /**
+ * Build a {@link RecoveryInfo} object, used in non-GTID master failover
+ * @return Information used to recover a master position, or NULL
+ * @throws SQLException If we have database issues
+ * @see MysqlPositionStore#getRecoveryInfo(MaxwellConfig)
+ */
public RecoveryInfo getRecoveryInfo() throws SQLException {
return this.positionStore.getRecoveryInfo(config);
}
+ /**
+ * If the passed {@link RowMap} is a transaction-commit, update maxwell's position
+ * @param r A processed Rowmap
+ */
public void setPosition(RowMap r) {
if ( r.isTXCommit() )
this.setPosition(r.getNextPosition());
}
+ /**
+ * Set Maxwell's next binlog position
+ * @param position The new position
+ */
public void setPosition(Position position) {
if ( position == null )
return;
@@ -321,20 +438,35 @@ public void setPosition(Position position) {
this.getPositionStoreThread().setPosition(position);
}
+ /**
+ * Get the last stored binlog position
+ * @return The last binlog position set
+ * @throws SQLException If we have database issues
+ */
public Position getPosition() throws SQLException {
return this.getPositionStoreThread().getPosition();
}
+ /**
+ * Get the position store service object
+ * @return The mysql position store
+ */
public MysqlPositionStore getPositionStore() {
return this.positionStore;
}
+ /**
+ * Get the replication connection's server id
+ * @return a server id
+ * @throws SQLException if we have connection issues
+ */
public Long getServerID() throws SQLException {
if ( this.serverID != null)
return this.serverID;
- try ( Connection c = getReplicationConnection() ) {
- ResultSet rs = c.createStatement().executeQuery("SELECT @@server_id as server_id");
+ try ( Connection c = getReplicationConnection();
+ Statement s = c.createStatement();
+ ResultSet rs = s.executeQuery("SELECT @@server_id as server_id") ) {
if ( !rs.next() ) {
throw new RuntimeException("Could not retrieve server_id!");
}
@@ -343,6 +475,11 @@ public Long getServerID() throws SQLException {
}
}
+ /**
+ * Get the replication connection's mysql version
+ * @return The mysql version
+ * @throws SQLException if we have connection issues
+ */
public MysqlVersion getMysqlVersion() throws SQLException {
if ( mysqlVersion == null ) {
try ( Connection c = getReplicationConnection() ) {
@@ -352,10 +489,11 @@ public MysqlVersion getMysqlVersion() throws SQLException {
return mysqlVersion;
}
- public boolean shouldHeartbeat() throws SQLException {
- return getMysqlVersion().atLeast(5,5);
- }
-
+ /**
+ * Get the replication connection's case sensitivity settings
+ * @return case sensitivity settings
+ * @throws SQLException if we have connection issues
+ */
public CaseSensitivity getCaseSensitivity() throws SQLException {
if ( this.caseSensitivity == null ) {
try (Connection c = getReplicationConnection()) {
@@ -365,6 +503,11 @@ public CaseSensitivity getCaseSensitivity() throws SQLException {
return this.caseSensitivity;
}
+ /**
+ * get or build an {@link AbstractProducer} based on settings in {@link #config}
+ * @return A producer
+ * @throws IOException if there's trouble instantiating the producer
+ */
public AbstractProducer getProducer() throws IOException {
if ( this.producer != null )
return this.producer;
@@ -383,7 +526,7 @@ public AbstractProducer getProducer() throws IOException {
this.producer = new MaxwellKinesisProducer(this, this.config.kinesisStream);
break;
case "sqs":
- this.producer = new MaxwellSQSProducer(this, this.config.sqsQueueUri);
+ this.producer = new MaxwellSQSProducer(this, this.config.sqsQueueUri, this.config.sqsServiceEndpoint, this.config.sqsSigningRegion);
break;
case "sns":
this.producer = new MaxwellSNSProducer(this, this.config.snsTopic);
@@ -392,7 +535,7 @@ public AbstractProducer getProducer() throws IOException {
this.producer = new NatsProducer(this);
break;
case "pubsub":
- this.producer = new MaxwellPubsubProducer(this, this.config.pubsubProjectId, this.config.pubsubTopic, this.config.ddlPubsubTopic);
+ this.producer = new MaxwellPubsubProducer(this, this.config.pubsubProjectId, this.config.pubsubTopic, this.config.ddlPubsubTopic, this.config.pubsubMessageOrderingKey, this.config.pubsubEmulator);
break;
case "profiler":
this.producer = new ProfilerProducer(this);
@@ -409,6 +552,9 @@ public AbstractProducer getProducer() throws IOException {
case "redis":
this.producer = new MaxwellRedisProducer(this);
break;
+ case "bigquery":
+ this.producer = new MaxwellBigQueryProducer(this, this.config.bigQueryProjectId, this.config.bigQueryDataset, this.config.bigQueryTable);
+ break;
case "none":
this.producer = new NoneProducer(this);
break;
@@ -434,12 +580,21 @@ public AbstractProducer getProducer() throws IOException {
return this.producer;
}
+ /**
+ * only used in test code. interrupt the bootstrap thread to quicken tests.
+ */
public void runBootstrapNow() {
if ( this.bootstrapControllerThread != null ) {
this.bootstrapControllerThread.interrupt();
}
}
+ /**
+ * get or start a {@link BootstrapController}
+ * @param currentSchemaID the currently active mysql schema
+ * @return a bootstrap controller
+ * @throws IOException if the bootstrap thread can't be started
+ */
public synchronized BootstrapController getBootstrapController(Long currentSchemaID) throws IOException {
if ( this.bootstrapController != null ) {
return this.bootstrapController;
@@ -463,6 +618,10 @@ public synchronized BootstrapController getBootstrapController(Long currentSchem
return this.bootstrapController;
}
+ /**
+ * get or start a {@link MysqlSchemaCompactor}
+ * @throws SQLException if we have connection issues
+ */
public void startSchemaCompactor() throws SQLException {
if ( this.config.maxSchemaDeltas == null )
return;
@@ -478,28 +637,68 @@ public void startSchemaCompactor() throws SQLException {
this.startTask(compactor, "maxwell-schema-compactor");
}
+ /**
+ * get the current active filter
+ * @return the currently active Filter
+ */
public Filter getFilter() {
return config.filter;
}
+ /**
+ * Get the replayMode flag
+ * @return whether we are in "replay mode" (--replay)
+ */
public boolean getReplayMode() {
return this.config.replayMode;
}
+ /**
+ * Set the current binlog replicator
+ * @param replicator the replicator
+ */
public void setReplicator(Replicator replicator) {
this.addTask(replicator);
this.replicator = replicator;
}
+ /**
+ * Get the current metrics registry
+ * @return the current metrics registry
+ */
public Metrics getMetrics() {
return metrics;
}
+ /**
+ * Get the heartbeat notifier object, which can be asked to broadcast heartbeats
+ * @return a heartbeat notifier
+ */
public HeartbeatNotifier getHeartbeatNotifier() {
return heartbeatNotifier;
}
+ /**
+ * Get the context for maxwell diagnostics
+ * @return the maxwell diagnostic context
+ */
public MaxwellDiagnosticContext getDiagnosticContext() {
return this.diagnosticContext;
}
+
+ /**
+ * Is the replication host running MariaDB?
+ * @return mariadbornot
+ */
+ public boolean isMariaDB() {
+ if ( this.isMariaDB == null ) {
+ try ( Connection c = this.getReplicationConnection() ) {
+ this.isMariaDB = MaxwellMysqlStatus.isMaria(c);
+ } catch ( SQLException e ) {
+ return false;
+ }
+ }
+
+ return this.isMariaDB;
+ }
}
diff --git a/src/main/java/com/zendesk/maxwell/MaxwellHA.java b/src/main/java/com/zendesk/maxwell/MaxwellHA.java
index 1bfdfb41d..e9caf138c 100644
--- a/src/main/java/com/zendesk/maxwell/MaxwellHA.java
+++ b/src/main/java/com/zendesk/maxwell/MaxwellHA.java
@@ -1,19 +1,16 @@
package com.zendesk.maxwell;
import org.jgroups.JChannel;
-import org.jgroups.protocols.raft.RaftLeaderException;
import org.jgroups.protocols.raft.Role;
-import org.jgroups.protocols.raft.StateMachine;
import org.jgroups.raft.RaftHandle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+/**
+ * Class that joins a jgroups-raft cluster of servers
+ */
public class MaxwellHA {
static final Logger LOGGER = LoggerFactory.getLogger(MaxwellHA.class);
@@ -22,6 +19,13 @@ public class MaxwellHA {
private boolean hasRun = false;
private AtomicBoolean isRaftLeader = new AtomicBoolean(false);
+ /**
+ * Build a MaxwellHA object
+ * @param maxwell The Maxwell instance that will be run when an election is won
+ * @param jgroupsConf Path to an xml file that will configure the RAFT cluster
+ * @param raftMemberID unique ID identifying the raft member in the cluster
+ * @param clientID The maxwell clientID. Used to create a unique "channel" for the election
+ */
public MaxwellHA(Maxwell maxwell, String jgroupsConf, String raftMemberID, String clientID) {
this.maxwell = maxwell;
this.jgroupsConf = jgroupsConf;
@@ -43,6 +47,12 @@ private void run() {
}
}
+ /**
+ * Join the raft cluster, starting and stopping Maxwell on elections.
+ *
+ * Does not return.
+ * @throws Exception if there's any issues
+ */
public void startHA() throws Exception {
JChannel ch=new JChannel(jgroupsConf);
RaftHandle handle=new RaftHandle(ch, null);
diff --git a/src/main/java/com/zendesk/maxwell/MaxwellMysqlConfig.java b/src/main/java/com/zendesk/maxwell/MaxwellMysqlConfig.java
index c3a275cc7..d00a46f8e 100644
--- a/src/main/java/com/zendesk/maxwell/MaxwellMysqlConfig.java
+++ b/src/main/java/com/zendesk/maxwell/MaxwellMysqlConfig.java
@@ -1,19 +1,16 @@
package com.zendesk.maxwell;
import java.net.URISyntaxException;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import com.github.shyiko.mysql.binlog.network.SSLMode;
-import org.apache.commons.lang3.StringUtils;
-import joptsimple.OptionSet;
import org.apache.http.client.utils.URIBuilder;
/**
- * Created by kaufmannkr on 12/23/15.
+ * Configuration object describing a mysql connection
*/
public class MaxwellMysqlConfig {
@@ -23,10 +20,22 @@ public class MaxwellMysqlConfig {
public String user;
public String password;
public SSLMode sslMode;
+ /**
+ * determines whether we enable replication heartbeats. Different from maxwell heartbeats
+ */
public boolean enableHeartbeat;
- public Map jdbcOptions = new HashMap<>();
+ /**
+ * Hashmap of key/value JDBC driver options. Not used for replication connections, generally.
+ */
+ public Map jdbcOptions;
+ /**
+ * Connection timeout for JDBC connections
+ */
public Integer connectTimeoutMS = 5000;
+ /**
+ * Instantiate a default connection config
+ */
public MaxwellMysqlConfig() {
this.host = null;
this.port = null;
@@ -42,6 +51,16 @@ public MaxwellMysqlConfig() {
this.jdbcOptions.put("allowPublicKeyRetrieval", "true");
}
+ /**
+ * Instantiate a mysql connection config
+ * @param host Mysql Host
+ * @param port Mysql port
+ * @param database Database name
+ * @param user User
+ * @param password Password
+ * @param sslMode SSL connection mode
+ * @param enableHeartbeat Replication heartbeats
+ */
public MaxwellMysqlConfig(String host, Integer port, String database, String user, String password,
SSLMode sslMode, boolean enableHeartbeat) {
this();
@@ -54,6 +73,10 @@ public MaxwellMysqlConfig(String host, Integer port, String database, String use
this.enableHeartbeat = enableHeartbeat;
}
+ /**
+ * Clone a mysql config
+ * @param c the config to clone
+ */
public MaxwellMysqlConfig(MaxwellMysqlConfig c) {
this();
this.host = c.host;
@@ -76,6 +99,10 @@ private void verifyServerCertificate(boolean should) {
this.jdbcOptions.put("verifyServerCertificate", String.valueOf(should));
}
+ /**
+ * Parse JDBC options from a key=val&key2=val2 string
+ * @param opts string to parse
+ */
public void setJDBCOptions(String opts) {
if (opts == null)
return;
@@ -105,6 +132,12 @@ private void setSSLOptions() {
}
}
+ /**
+ * Build a connection URI from the config
+ * @param includeDatabase whether to include the database name in th euri
+ * @return a connection URI string
+ * @throws URISyntaxException if we have problems building the URI
+ */
public String getConnectionURI(boolean includeDatabase) throws URISyntaxException {
this.setSSLOptions();
@@ -129,6 +162,11 @@ public String getConnectionURI(boolean includeDatabase) throws URISyntaxExceptio
return uriBuilder.build().toString();
}
+ /**
+ * Build a connection URI from the config
+ * @return a connection URI
+ * @throws URISyntaxException if we have problems
+ */
public String getConnectionURI() throws URISyntaxException { return getConnectionURI(true); }
@Override
diff --git a/src/main/java/com/zendesk/maxwell/MaxwellMysqlStatus.java b/src/main/java/com/zendesk/maxwell/MaxwellMysqlStatus.java
index 538fc5b8d..4c2fd6839 100644
--- a/src/main/java/com/zendesk/maxwell/MaxwellMysqlStatus.java
+++ b/src/main/java/com/zendesk/maxwell/MaxwellMysqlStatus.java
@@ -4,9 +4,16 @@
import org.slf4j.LoggerFactory;
import java.sql.Connection;
+import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
+import java.util.Properties;
+/**
+ * Class with some utility functions for querying mysql server state
+ */
public class MaxwellMysqlStatus {
static final Logger LOGGER = LoggerFactory.getLogger(MaxwellMysqlStatus.class);
private Connection connection;
@@ -18,22 +25,82 @@ public MaxwellMysqlStatus(Connection c) {
private String sqlStatement(String variableName) {
return "SHOW VARIABLES LIKE '" + variableName + "'";
}
+ public boolean isMaria() {
+ try {
+ DatabaseMetaData md = connection.getMetaData();
+ return md.getDatabaseProductVersion().toLowerCase().contains("mariadb");
+ } catch ( SQLException e ) {
+ return false;
+ }
+ }
- private String getVariableState(String variableName, boolean throwOnMissing) throws SQLException, MaxwellCompatibilityError {
- ResultSet rs;
-
- rs = connection.createStatement().executeQuery(sqlStatement(variableName));
- String status;
- if(!rs.next()) {
- if ( throwOnMissing ) {
- throw new MaxwellCompatibilityError("Could not check state for Mysql variable: " + variableName);
- } else {
- return null;
+ /**
+ * Generates the appropriate SQL command to retrieve binary log status based on
+ * the database type and version.
+ *
+ * This method checks the database product name and version to determine
+ * the most suitable SQL command for retrieving binary log status information.
+ * It supports MySQL and MariaDB, with compatibility for recent version
+ * requirements:
+ *
+ *
MySQL 8.2 and above: uses "SHOW BINARY LOG STATUS"
+ *
MariaDB 10.5 and above: uses "SHOW BINLOG STATUS"
+ *
All other versions default to "SHOW MASTER STATUS"
");
+ writer.println("");
+ }
+
+ public void addLink(String endpoint, String description) {
+ endpoints.add(Pair.of(endpoint, description));
+
+ }
+}
diff --git a/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHTTPServer.java b/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHTTPServer.java
index 9b589daea..ce57bdec6 100644
--- a/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHTTPServer.java
+++ b/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHTTPServer.java
@@ -6,6 +6,7 @@
import com.zendesk.maxwell.MaxwellConfig;
import com.zendesk.maxwell.MaxwellContext;
import com.zendesk.maxwell.util.StoppableTask;
+import jnr.ffi.annotations.In;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
@@ -50,7 +51,11 @@ private static MaxwellMetrics.Registries getMetricsRegistries(MaxwellContext con
MaxwellConfig config = context.getConfig();
String reportingType = config.metricsReportingType;
if (reportingType != null && reportingType.contains(reportingTypeHttp)) {
- context.healthCheckRegistry.register("MaxwellHealth", new MaxwellHealthCheck(context.getProducer()));
+ if (config.customHealthFactory != null) {
+ context.healthCheckRegistry.register("MaxwellHealth", config.customHealthFactory.createHealthCheck(context.getProducer()));
+ } else {
+ context.healthCheckRegistry.register("MaxwellHealth", new MaxwellHealthCheck(context.getProducer()));
+ }
return new MaxwellMetrics.Registries(context.metricRegistry, context.healthCheckRegistry);
} else {
return null;
@@ -87,6 +92,8 @@ public MaxwellHTTPServerWorker(String bindAddress, int port, String pathPrefix,
}
public void startServer() throws Exception {
+ IndexListServlet indexList = new IndexListServlet();
+
if (this.bindAddress != null) {
this.server = new Server(new InetSocketAddress(InetAddress.getByName(this.bindAddress), port));
}
@@ -95,20 +102,29 @@ public void startServer() throws Exception {
}
ServletContextHandler handler = new ServletContextHandler(this.server, pathPrefix);
+ handler.addServlet(new ServletHolder(indexList), "/");
+
if (metricsRegistries != null) {
// TODO: there is a way to wire these up automagically via the AdminServlet, but it escapes me right now
handler.addServlet(new ServletHolder(new MetricsServlet(metricsRegistries.metricRegistry)), "/metrics");
handler.addServlet(new ServletHolder(new io.prometheus.client.exporter.MetricsServlet()), "/prometheus");
handler.addServlet(new ServletHolder(new HealthCheckServlet(metricsRegistries.healthCheckRegistry)), "/healthcheck");
handler.addServlet(new ServletHolder(new PingServlet()), "/ping");
+
+ indexList.addLink("/metrics", "codahale metrics");
+ indexList.addLink("/prometheus", "prometheus metrics");
+ indexList.addLink("/healthcheck", "healthcheck endpoint");
+ indexList.addLink("/ping", "ping me");
}
if (this.context.getConfig().enableHttpConfig) {
handler.addServlet(new ServletHolder(new MaxwellConfigServlet(this.context)), "/config");
+ indexList.addLink("/config", "POST endpoing to update maxwell config.");
}
if (diagnosticContext != null) {
handler.addServlet(new ServletHolder(new DiagnosticHealthCheck(diagnosticContext)), "/diagnostic");
+ indexList.addLink("/diagnostic", "deeper diagnostic health checks");
}
this.server.start();
diff --git a/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHealthCheckFactory.java b/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHealthCheckFactory.java
new file mode 100644
index 000000000..cca234409
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/monitoring/MaxwellHealthCheckFactory.java
@@ -0,0 +1,8 @@
+package com.zendesk.maxwell.monitoring;
+
+import com.zendesk.maxwell.monitoring.MaxwellHealthCheck;
+import com.zendesk.maxwell.producer.AbstractProducer;
+
+public interface MaxwellHealthCheckFactory {
+ MaxwellHealthCheck createHealthCheck(AbstractProducer producer);
+}
diff --git a/src/main/java/com/zendesk/maxwell/monitoring/MaxwellMetrics.java b/src/main/java/com/zendesk/maxwell/monitoring/MaxwellMetrics.java
index adf8c7099..65c04c90d 100644
--- a/src/main/java/com/zendesk/maxwell/monitoring/MaxwellMetrics.java
+++ b/src/main/java/com/zendesk/maxwell/monitoring/MaxwellMetrics.java
@@ -48,9 +48,12 @@ private void setup(MaxwellConfig config) {
metricsPrefix = config.metricsPrefix;
if (config.metricsReportingType == null) {
- if ( hasMetricsConfig(config) )
- LOGGER.warn("Metrics will not be exposed: --metrics_type not set");
- return;
+ if ( hasMetricsConfig(config) ) {
+ LOGGER.info("Found HTTP server configuration, enabling HTTP-based metrics");
+ config.metricsReportingType = "http";
+ } else {
+ return;
+ }
}
if (config.metricsJvm) {
diff --git a/src/main/java/com/zendesk/maxwell/monitoring/package-info.java b/src/main/java/com/zendesk/maxwell/monitoring/package-info.java
new file mode 100644
index 000000000..9238b54b1
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/monitoring/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * monitoring, diagnostics, and the HTTP server
+ */
+package com.zendesk.maxwell.monitoring;
diff --git a/src/main/java/com/zendesk/maxwell/package-info.java b/src/main/java/com/zendesk/maxwell/package-info.java
new file mode 100644
index 000000000..38c07b473
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * top level classes concerned with instantiating and running a Maxwell
+ * replicator.
+ */
+
+package com.zendesk.maxwell;
diff --git a/src/main/java/com/zendesk/maxwell/producer/AbstractAsyncProducer.java b/src/main/java/com/zendesk/maxwell/producer/AbstractAsyncProducer.java
index 22fc194d1..97262b537 100644
--- a/src/main/java/com/zendesk/maxwell/producer/AbstractAsyncProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/AbstractAsyncProducer.java
@@ -66,8 +66,7 @@ public AbstractAsyncProducer(MaxwellContext context) {
@Override
public final void push(RowMap r) throws Exception {
Position position = r.getNextPosition();
- // Rows that do not get sent to a target will be automatically marked as complete.
- // We will attempt to commit a checkpoint up to the current row.
+ // Rows that do not get sent to the prodcuer will be automatically marked as complete.
if(!r.shouldOutput(outputConfig)) {
if ( position != null ) {
inflightMessages.addMessage(position, r.getTimestampMillis(), 0L);
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellBigQueryProducer.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellBigQueryProducer.java
new file mode 100644
index 000000000..6629ddbe9
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellBigQueryProducer.java
@@ -0,0 +1,282 @@
+package com.zendesk.maxwell.producer;
+
+import com.google.api.core.ApiFuture;
+import com.google.api.core.ApiFutureCallback;
+import com.google.api.core.ApiFutures;
+import com.google.api.services.bigquery.model.JsonObject;
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.Schema;
+import com.google.cloud.bigquery.Table;
+import com.google.cloud.bigquery.storage.v1.AppendRowsResponse;
+import com.google.cloud.bigquery.storage.v1.Exceptions;
+import com.google.cloud.bigquery.storage.v1.Exceptions.StorageException;
+import com.google.cloud.bigquery.storage.v1.JsonStreamWriter;
+import com.google.cloud.bigquery.storage.v1.TableName;
+import com.google.cloud.bigquery.storage.v1.TableSchema;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.protobuf.Descriptors.DescriptorValidationException;
+import com.zendesk.maxwell.MaxwellContext;
+import com.zendesk.maxwell.monitoring.Metrics;
+import com.zendesk.maxwell.replication.Position;
+import com.zendesk.maxwell.row.RowMap;
+import com.zendesk.maxwell.schema.BqToBqStorageSchemaConverter;
+import com.zendesk.maxwell.util.StoppableTask;
+import com.zendesk.maxwell.util.StoppableTaskState;
+
+import io.grpc.Status;
+import io.grpc.Status.Code;
+import java.io.IOException;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.TimeoutException;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Meter;
+
+import javax.annotation.concurrent.GuardedBy;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class BigQueryCallback implements ApiFutureCallback {
+ public final Logger LOGGER = LoggerFactory.getLogger(BigQueryCallback.class);
+
+ private final MaxwellBigQueryProducerWorker parent;
+ private final AbstractAsyncProducer.CallbackCompleter cc;
+ private final Position position;
+ private MaxwellContext context;
+ AppendContext appendContext;
+
+ private Counter succeededMessageCount;
+ private Counter failedMessageCount;
+ private Meter succeededMessageMeter;
+ private Meter failedMessageMeter;
+
+ private static final int MAX_RETRY_COUNT = 2;
+ private final ImmutableList RETRIABLE_ERROR_CODES = ImmutableList.of(Code.INTERNAL, Code.ABORTED,
+ Code.CANCELLED);
+
+ public BigQueryCallback(MaxwellBigQueryProducerWorker parent,
+ AppendContext appendContext,
+ AbstractAsyncProducer.CallbackCompleter cc,
+ Position position,
+ Counter producedMessageCount, Counter failedMessageCount,
+ Meter succeededMessageMeter, Meter failedMessageMeter,
+ MaxwellContext context) {
+ this.parent = parent;
+ this.appendContext = appendContext;
+ this.cc = cc;
+ this.position = position;
+ this.succeededMessageCount = producedMessageCount;
+ this.failedMessageCount = failedMessageCount;
+ this.succeededMessageMeter = succeededMessageMeter;
+ this.failedMessageMeter = failedMessageMeter;
+ this.context = context;
+ }
+
+ @Override
+ public void onSuccess(AppendRowsResponse response) {
+ this.succeededMessageCount.inc();
+ this.succeededMessageMeter.mark();
+
+ if (LOGGER.isDebugEnabled()) {
+ try {
+ LOGGER.debug("-> {}\n" +
+ " {}\n",
+ this.appendContext.r.toJSON(), this.position);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ cc.markCompleted();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ this.failedMessageCount.inc();
+ this.failedMessageMeter.mark();
+
+ LOGGER.error(t.getClass().getSimpleName() + " @ " + position);
+ LOGGER.error(t.getLocalizedMessage());
+
+ Status status = Status.fromThrowable(t);
+ if (appendContext.retryCount < MAX_RETRY_COUNT
+ && RETRIABLE_ERROR_CODES.contains(status.getCode())) {
+ appendContext.retryCount++;
+ try {
+ this.parent.sendAsync(appendContext.r, this.cc);
+ return;
+ } catch (Exception e) {
+ System.out.format("Failed to retry append: %s\n", e);
+ }
+ }
+
+ synchronized (this.parent.getLock()) {
+ if (this.parent.getError() == null && !this.context.getConfig().ignoreProducerError) {
+ StorageException storageException = Exceptions.toStorageException(t);
+ this.parent.setError((storageException != null) ? storageException : new RuntimeException(t));
+ context.terminate();
+ return;
+ }
+ }
+ cc.markCompleted();
+ }
+}
+
+public class MaxwellBigQueryProducer extends AbstractProducer {
+
+ private final ArrayBlockingQueue queue;
+ private final MaxwellBigQueryProducerWorker worker;
+
+ public MaxwellBigQueryProducer(MaxwellContext context, String bigQueryProjectId,
+ String bigQueryDataset, String bigQueryTable)
+ throws IOException {
+ super(context);
+ this.queue = new ArrayBlockingQueue<>(100);
+ this.worker = new MaxwellBigQueryProducerWorker(context, this.queue, bigQueryProjectId, bigQueryDataset,
+ bigQueryTable);
+
+ TableName table = TableName.of(bigQueryProjectId, bigQueryDataset, bigQueryTable);
+ try {
+ this.worker.initialize(table);
+ } catch (DescriptorValidationException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ Thread thread = new Thread(this.worker, "maxwell-bigquery-worker");
+ thread.setDaemon(true);
+ thread.start();
+ }
+
+ @Override
+ public void push(RowMap r) throws Exception {
+ this.queue.put(r);
+ }
+}
+
+class AppendContext {
+ JSONArray data;
+ int retryCount = 0;
+ RowMap r = null;
+
+ AppendContext(JSONArray data, int retryCount, RowMap r) {
+ this.data = data;
+ this.retryCount = retryCount;
+ this.r = r;
+ }
+}
+
+class MaxwellBigQueryProducerWorker extends AbstractAsyncProducer implements Runnable, StoppableTask {
+ static final Logger LOGGER = LoggerFactory.getLogger(MaxwellBigQueryProducerWorker.class);
+
+ private final ArrayBlockingQueue queue;
+ private StoppableTaskState taskState;
+ private Thread thread;
+ private final Object lock = new Object();
+
+ @GuardedBy("lock")
+ private RuntimeException error = null;
+ private JsonStreamWriter streamWriter;
+
+ public MaxwellBigQueryProducerWorker(MaxwellContext context,
+ ArrayBlockingQueue queue, String bigQueryProjectId,
+ String bigQueryDataset, String bigQueryTable) throws IOException {
+ super(context);
+ this.queue = queue;
+ Metrics metrics = context.getMetrics();
+ this.taskState = new StoppableTaskState("MaxwellBigQueryProducerWorker");
+ }
+
+ public Object getLock() {
+ return lock;
+ }
+
+ public RuntimeException getError() {
+ return error;
+ }
+
+ public void setError(RuntimeException error) {
+ this.error = error;
+ }
+
+ private void covertJSONObjectFieldsToString(JSONObject record) {
+ if (this.context.getConfig().outputConfig.includesPrimaryKeys) {
+ record.put("primary_key", record.get("primary_key").toString());
+ }
+ String data = record.has("data") == true ? record.get("data").toString() : null;
+ record.put("data", data);
+ String old = record.has("old") == true ? record.get("old").toString() : null;
+ record.put("old", old);
+ }
+
+ public void initialize(TableName tName)
+ throws DescriptorValidationException, IOException, InterruptedException {
+
+ BigQuery bigquery = BigQueryOptions.newBuilder().setProjectId(tName.getProject()).build().getService();
+ Table table = bigquery.getTable(tName.getDataset(), tName.getTable());
+ Schema schema = table.getDefinition().getSchema();
+ TableSchema tableSchema = BqToBqStorageSchemaConverter.convertTableSchema(schema);
+ streamWriter = JsonStreamWriter.newBuilder(tName.toString(), tableSchema).build();
+ }
+
+ @Override
+ public void requestStop() throws Exception {
+ taskState.requestStop();
+ streamWriter.close();
+ synchronized (this.lock) {
+ if (this.error != null) {
+ throw this.error;
+ }
+ }
+ }
+
+ @Override
+ public void awaitStop(Long timeout) throws TimeoutException {
+ taskState.awaitStop(thread, timeout);
+ }
+
+ @Override
+ public void run() {
+ this.thread = Thread.currentThread();
+ while (true) {
+ try {
+ RowMap row = queue.take();
+ if (!taskState.isRunning()) {
+ taskState.stopped();
+ return;
+ }
+ this.push(row);
+ } catch (Exception e) {
+ taskState.stopped();
+ context.terminate(e);
+ return;
+ }
+ }
+ }
+
+ @Override
+ public void sendAsync(RowMap r, CallbackCompleter cc) throws Exception {
+ synchronized (this.lock) {
+ if (this.error != null) {
+ throw this.error;
+ }
+ }
+ JSONArray jsonArr = new JSONArray();
+ JSONObject record = new JSONObject(r.toJSON(outputConfig));
+ //convert json and array fields to String
+ covertJSONObjectFieldsToString(record);
+ jsonArr.put(record);
+ AppendContext appendContext = new AppendContext(jsonArr, 0, r);
+
+ ApiFuture future = streamWriter.append(appendContext.data);
+ ApiFutures.addCallback(
+ future, new BigQueryCallback(this, appendContext, cc, r.getNextPosition(),
+ this.succeededMessageCount, this.failedMessageCount, this.succeededMessageMeter, this.failedMessageMeter,
+ this.context),
+ MoreExecutors.directExecutor());
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellKafkaProducer.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellKafkaProducer.java
index e6fc9e5fd..470626c61 100644
--- a/src/main/java/com/zendesk/maxwell/producer/MaxwellKafkaProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellKafkaProducer.java
@@ -83,10 +83,10 @@ public void onCompletion(RecordMetadata md, Exception e) {
this.succeededMessageMeter.mark();
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("-> key:" + key + ", partition:" + md.partition() + ", offset:" + md.offset());
- LOGGER.debug(" " + this.json);
- LOGGER.debug(" " + position);
- LOGGER.debug("");
+ LOGGER.debug("-> key:{}, partition:{}, offset:{}\n" +
+ " {}\n" +
+ " {}\n",
+ key, md.partition(), md.offset(), this.json, position);
}
cc.markCompleted();
}
@@ -265,7 +265,6 @@ void sendAsync(ProducerRecord record, Callback callback) {
}
ProducerRecord makeProducerRecord(final RowMap r) throws Exception {
- RowIdentity pk = r.getRowIdentity();
String key = r.pkToJson(keyFormat);
String value = r.toJSON(outputConfig);
ProducerRecord record;
@@ -279,7 +278,7 @@ record = new ProducerRecord<>(this.ddlTopic, this.ddlPartitioner.kafkaPartition(
if (topic == null) {
topic = this.topicInterpolator.generateFromRowMap(r);
}
- LOGGER.debug("context.getConfig().producerPartitionKey = " + context.getConfig().producerPartitionKey);
+ LOGGER.debug("context.getConfig().producerPartitionKey = {}", context.getConfig().producerPartitionKey);
record = new ProducerRecord<>(topic, this.partitioner.kafkaPartition(r, getNumPartitions(topic)), key, value);
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellKinesisProducer.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellKinesisProducer.java
index 5ba0e26da..07489cf17 100644
--- a/src/main/java/com/zendesk/maxwell/producer/MaxwellKinesisProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellKinesisProducer.java
@@ -80,10 +80,10 @@ public void onSuccess(UserRecordResult result) {
this.succeededMessageCount.inc();
this.succeededMessageMeter.mark();
if(logger.isDebugEnabled()) {
- logger.debug("-> key:" + key + ", shard id:" + result.getShardId() + ", sequence number:" + result.getSequenceNumber());
- logger.debug(" " + json);
- logger.debug(" " + position);
- logger.debug("");
+ logger.debug("-> key:{}, shard id:{}, sequence number:{}\n" +
+ " {}\n" +
+ " {}\n",
+ key, result.getShardId(), result.getSequenceNumber(), json, position);
}
cc.markCompleted();
@@ -112,7 +112,12 @@ public MaxwellKinesisProducer(MaxwellContext context, String kinesisStream) {
KinesisProducerConfiguration config = KinesisProducerConfiguration.fromPropertiesFile(path.toString());
this.kinesisProducer = new KinesisProducer(config);
} else {
- this.kinesisProducer = new KinesisProducer();
+ // The default 30 second record Ttl is too aggressive and prevents our own back-pressure
+ // logic from backing as needed off before the producer fails. Setting it to 1 hour
+ // instead.
+ KinesisProducerConfiguration config = new KinesisProducerConfiguration();
+ config.setRecordTtl(3600000);
+ this.kinesisProducer = new KinesisProducer(config);
}
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellOutputConfig.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellOutputConfig.java
index 79a5067ae..ec40788bc 100644
--- a/src/main/java/com/zendesk/maxwell/producer/MaxwellOutputConfig.java
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellOutputConfig.java
@@ -24,6 +24,7 @@ public class MaxwellOutputConfig {
public String secretKey;
public boolean zeroDatesAsNull;
public String namingStrategy;
+ public int rowQueryMaxLength;
public MaxwellOutputConfig() {
this.includesBinlogPosition = false;
@@ -43,6 +44,7 @@ public MaxwellOutputConfig() {
this.encryptionMode = EncryptionMode.ENCRYPT_NONE;
this.secretKey = null;
this.namingStrategy = null;
+ this.rowQueryMaxLength = 0;
}
public boolean encryptionEnabled() {
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellPubsubProducer.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellPubsubProducer.java
index 36dca9794..6f0040531 100644
--- a/src/main/java/com/zendesk/maxwell/producer/MaxwellPubsubProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellPubsubProducer.java
@@ -6,8 +6,14 @@
import com.google.api.core.ApiFutureCallback;
import com.google.api.core.ApiFutures;
import com.google.api.gax.batching.BatchingSettings;
+import com.google.api.gax.core.CredentialsProvider;
+import com.google.api.gax.core.NoCredentialsProvider;
+import com.google.api.gax.grpc.GrpcTransportChannel;
import com.google.api.gax.retrying.RetrySettings;
+import com.google.api.gax.rpc.FixedTransportChannelProvider;
+import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.cloud.pubsub.v1.Publisher;
+import com.google.cloud.pubsub.v1.Publisher.Builder;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import com.google.pubsub.v1.ProjectTopicName;
@@ -19,9 +25,13 @@
import com.zendesk.maxwell.schema.ddl.DDLMap;
import com.zendesk.maxwell.util.StoppableTask;
import com.zendesk.maxwell.util.StoppableTaskState;
+import com.zendesk.maxwell.util.TopicInterpolator;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.threeten.bp.Duration;
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
@@ -61,9 +71,9 @@ public void onSuccess(String messageId) {
this.succeededMessageMeter.mark();
if ( LOGGER.isDebugEnabled() ) {
- LOGGER.debug("-> " + this.json);
- LOGGER.debug(" " + this.position);
- LOGGER.debug("");
+ LOGGER.debug("-> {}\n" +
+ " {}\n",
+ this.json, this.position);
}
cc.markCompleted();
@@ -93,12 +103,14 @@ public class MaxwellPubsubProducer extends AbstractProducer {
private final MaxwellPubsubProducerWorker worker;
public MaxwellPubsubProducer(MaxwellContext context, String pubsubProjectId,
- String pubsubTopic, String ddlPubsubTopic)
+ String pubsubTopic, String ddlPubsubTopic,
+ String pubsubMessageOrderingKey, String pubsubEmulator)
throws IOException {
super(context);
this.queue = new ArrayBlockingQueue<>(100);
this.worker = new MaxwellPubsubProducerWorker(context, pubsubProjectId,
pubsubTopic, ddlPubsubTopic,
+ pubsubMessageOrderingKey, pubsubEmulator,
this.queue);
Thread thread = new Thread(this.worker, "maxwell-pubsub-worker");
@@ -126,6 +138,7 @@ class MaxwellPubsubProducerWorker
private final ProjectTopicName topic;
private final ProjectTopicName ddlTopic;
private Publisher ddlPubsub;
+ private final TopicInterpolator messageOrderingKeyInterpolator;
private final ArrayBlockingQueue queue;
private Thread thread;
private StoppableTaskState taskState;
@@ -133,6 +146,8 @@ class MaxwellPubsubProducerWorker
public MaxwellPubsubProducerWorker(MaxwellContext context,
String pubsubProjectId, String pubsubTopic,
String ddlPubsubTopic,
+ String pubsubMessageOrderingKey,
+ String pubsubEmulator,
ArrayBlockingQueue queue)
throws IOException {
super(context);
@@ -159,8 +174,23 @@ public MaxwellPubsubProducerWorker(MaxwellContext context,
this.projectId = pubsubProjectId;
this.topic = ProjectTopicName.of(pubsubProjectId, pubsubTopic);
- this.pubsub = Publisher.newBuilder(this.topic).setBatchingSettings(batchingSettings).setRetrySettings(retrySettings).build();
+ Builder pubsubBuilder = Publisher
+ .newBuilder(this.topic)
+ .setEnableMessageOrdering(pubsubMessageOrderingKey != null)
+ .setBatchingSettings(batchingSettings)
+ .setRetrySettings(retrySettings);
+
+ if (pubsubEmulator != null) {
+ ManagedChannel channel = ManagedChannelBuilder.forTarget(pubsubEmulator).usePlaintext().build();
+ TransportChannelProvider channelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel));
+ CredentialsProvider credentialsProvider = NoCredentialsProvider.create();
+ pubsubBuilder
+ .setCredentialsProvider(credentialsProvider)
+ .setChannelProvider(channelProvider);
+ }
+ this.pubsub = pubsubBuilder.build();
+
if ( context.getConfig().outputConfig.outputDDL == true &&
ddlPubsubTopic != pubsubTopic ) {
this.ddlTopic = ProjectTopicName.of(pubsubProjectId, ddlPubsubTopic);
@@ -170,6 +200,11 @@ public MaxwellPubsubProducerWorker(MaxwellContext context,
this.ddlPubsub = this.pubsub;
}
+
+ this.messageOrderingKeyInterpolator = pubsubMessageOrderingKey == null
+ ? null
+ : new TopicInterpolator(pubsubMessageOrderingKey);
+
Metrics metrics = context.getMetrics();
this.queue = queue;
@@ -200,7 +235,15 @@ public void sendAsync(RowMap r, AbstractAsyncProducer.CallbackCompleter cc)
throws Exception {
String message = r.toJSON(outputConfig);
ByteString data = ByteString.copyFromUtf8(message);
- PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build();
+ PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder().setData(data);
+ if (this.messageOrderingKeyInterpolator != null) {
+ String orderingKey = this.messageOrderingKeyInterpolator.generateFromRowMapAndCleanUpIllegalCharacters(r);
+ pubsubMessageBuilder = pubsubMessageBuilder.setOrderingKey(orderingKey);
+ LOGGER.debug("using message ordering key {}", orderingKey);
+ } else {
+ LOGGER.debug("using no message ordering key");
+ }
+ PubsubMessage pubsubMessage = pubsubMessageBuilder.build();
if ( r instanceof DDLMap ) {
ApiFuture apiFuture = ddlPubsub.publish(pubsubMessage);
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellRedisProducer.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellRedisProducer.java
index d26ff9975..f46605a64 100644
--- a/src/main/java/com/zendesk/maxwell/producer/MaxwellRedisProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellRedisProducer.java
@@ -125,17 +125,17 @@ private void sendToRedis(RowMap msg) throws Exception {
if (logger.isDebugEnabled()) {
switch (redisType) {
case "lpush":
- logger.debug("-> queue (left):" + channel + ", msg:" + msg);
+ logger.debug("-> queue (left):{}, msg:{}", channel, msg);
break;
case "rpush":
- logger.debug("-> queue (right):" + channel + ", msg:" + msg);
+ logger.debug("-> queue (right):{}, msg:{}", channel, msg);
break;
case "xadd":
- logger.debug("-> stream:" + channel + ", msg:" + msg);
+ logger.debug("-> stream:{}, msg:{}", channel, msg);
break;
case "pubsub":
default:
- logger.debug("-> channel:" + channel + ", msg:" + msg);
+ logger.debug("-> channel:{}, msg:{}", channel, msg);
break;
}
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellSNSProducer.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellSNSProducer.java
index 54eba0578..e051cda13 100644
--- a/src/main/java/com/zendesk/maxwell/producer/MaxwellSNSProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellSNSProducer.java
@@ -7,6 +7,7 @@
import com.amazonaws.services.sns.model.PublishRequest;
import com.amazonaws.services.sns.model.PublishResult;
import com.zendesk.maxwell.MaxwellContext;
+import com.zendesk.maxwell.producer.partitioners.MaxwellSNSPartitioner;
import com.zendesk.maxwell.replication.Position;
import com.zendesk.maxwell.row.RowMap;
import org.apache.commons.lang3.ArrayUtils;
@@ -22,11 +23,16 @@ public class MaxwellSNSProducer extends AbstractAsyncProducer {
private String topic;
private String[] stringFelds = {"database", "table"};
private String[] numberFields = {"ts", "xid"};
+ private MaxwellSNSPartitioner partitioner;
public MaxwellSNSProducer(MaxwellContext context, String topic) {
super(context);
this.topic = topic;
this.client = AmazonSNSAsyncClientBuilder.defaultClient();
+ String partitionKey = context.getConfig().producerPartitionKey;
+ String partitionColumns = context.getConfig().producerPartitionColumns;
+ String partitionFallback = context.getConfig().producerPartitionFallback;
+ this.partitioner = new MaxwellSNSPartitioner(partitionKey, partitionColumns, partitionFallback);
}
public void setClient(AmazonSNSAsync client) {
@@ -35,7 +41,7 @@ public void setClient(AmazonSNSAsync client) {
@Override
public void sendAsync(RowMap r, CallbackCompleter cc) throws Exception {
- String value = r.toJSON();
+ String value = r.toJSON(outputConfig);
// Publish a message to an Amazon SNS topic.
final PublishRequest publishRequest = new PublishRequest(topic, value);
Map messageAttributes = new HashMap<>();
@@ -61,7 +67,8 @@ public void sendAsync(RowMap r, CallbackCompleter cc) throws Exception {
}
if ( topic.endsWith(".fifo")) {
- publishRequest.setMessageGroupId(r.getDatabase());
+ String key = this.partitioner.getSNSKey(r);
+ publishRequest.setMessageGroupId(key);
}
publishRequest.setMessageAttributes(messageAttributes);
@@ -103,7 +110,7 @@ public void onError(Exception t) {
@Override
public void onSuccess(PublishRequest request, PublishResult result) {
if (logger.isDebugEnabled()) {
- logger.debug("-> MessageId: " + result.getMessageId());
+ logger.debug("-> MessageId: {}", result.getMessageId());
}
cc.markCompleted();
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/MaxwellSQSProducer.java b/src/main/java/com/zendesk/maxwell/producer/MaxwellSQSProducer.java
index ca35093b6..027afe5d6 100644
--- a/src/main/java/com/zendesk/maxwell/producer/MaxwellSQSProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/MaxwellSQSProducer.java
@@ -1,15 +1,16 @@
package com.zendesk.maxwell.producer;
+import com.amazonaws.client.builder.AwsClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.handlers.AsyncHandler;
import com.amazonaws.services.sqs.AmazonSQSAsync;
-import com.amazonaws.services.sqs.AmazonSQSAsyncClient;
import com.amazonaws.services.sqs.AmazonSQSAsyncClientBuilder;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.amazonaws.services.sqs.model.SendMessageResult;
import com.zendesk.maxwell.MaxwellContext;
+import com.zendesk.maxwell.producer.partitioners.MaxwellSQSPartitioner;
import com.zendesk.maxwell.replication.Position;
import com.zendesk.maxwell.row.RowMap;
@@ -17,19 +18,27 @@ public class MaxwellSQSProducer extends AbstractAsyncProducer {
private AmazonSQSAsync client;
private String queueUri;
+ private MaxwellSQSPartitioner partitioner;
- public MaxwellSQSProducer(MaxwellContext context, String queueUri) {
+ public MaxwellSQSProducer(MaxwellContext context, String queueUri, String serviceEndpoint, String signingRegion) {
super(context);
this.queueUri = queueUri;
- this.client = AmazonSQSAsyncClientBuilder.defaultClient();
+ this.client = AmazonSQSAsyncClientBuilder.standard()
+ .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(serviceEndpoint, signingRegion))
+ .build();
+ String partitionKey = context.getConfig().producerPartitionKey;
+ String partitionColumns = context.getConfig().producerPartitionColumns;
+ String partitionFallback = context.getConfig().producerPartitionFallback;
+ this.partitioner = new MaxwellSQSPartitioner(partitionKey, partitionColumns, partitionFallback);
}
@Override
public void sendAsync(RowMap r, CallbackCompleter cc) throws Exception {
- String value = r.toJSON();
+ String value = r.toJSON(outputConfig);
SendMessageRequest messageRequest = new SendMessageRequest(queueUri, value);
if ( queueUri.endsWith(".fifo")) {
- messageRequest.setMessageGroupId(r.getDatabase());
+ String key = this.partitioner.getSQSKey(r);
+ messageRequest.setMessageGroupId(key);
}
SQSCallback callback = new SQSCallback(cc, r.getNextPosition(), value, context);
client.sendMessageAsync(messageRequest, callback);
@@ -69,7 +78,8 @@ public void onError(Exception t) {
@Override
public void onSuccess(SendMessageRequest request, SendMessageResult result) {
if (logger.isDebugEnabled()) {
- logger.debug("-> Message id:" + result.getMessageId() + ", sequence number:" + result.getSequenceNumber()+" "+json+" "+position);
+ logger.debug("-> Message id:{}, sequence number:{} {} {}",
+ result.getMessageId(), result.getSequenceNumber(), json, position);
}
cc.markCompleted();
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/NatsProducer.java b/src/main/java/com/zendesk/maxwell/producer/NatsProducer.java
index 264c99d9a..dc41f3d73 100644
--- a/src/main/java/com/zendesk/maxwell/producer/NatsProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/NatsProducer.java
@@ -59,7 +59,7 @@ public void push(RowMap r) throws Exception {
context.setPosition(r.getNextPosition());
}
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("-> nats subject:" + natsSubject + ", message:" + value);
+ LOGGER.debug("-> nats subject:{}, message:{}", natsSubject, value);
}
}
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/RabbitmqProducer.java b/src/main/java/com/zendesk/maxwell/producer/RabbitmqProducer.java
index 9fefe78a8..eb038fa85 100644
--- a/src/main/java/com/zendesk/maxwell/producer/RabbitmqProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/RabbitmqProducer.java
@@ -10,6 +10,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.net.URISyntaxException;
import java.security.KeyManagementException;
@@ -57,6 +58,14 @@ public RabbitmqProducer(MaxwellContext context) {
factory.setVirtualHost(config.rabbitmqVirtualHost);
}
+ if ( config.rabbitmqHandshakeTimeout != null ) {
+ factory.setHandshakeTimeout(config.rabbitmqHandshakeTimeout);
+ }
+
+ if ( config.rabbitmqUseSSL ) {
+ factory.useSslProtocol(SSLContext.getDefault());
+ }
+
this.channel = factory.newConnection().createChannel();
if(context.getConfig().rabbitmqDeclareExchange) {
this.channel.exchangeDeclare(exchangeName, context.getConfig().rabbitmqExchangeType, context.getConfig().rabbitMqExchangeDurable, context.getConfig().rabbitMqExchangeAutoDelete, null);
@@ -84,7 +93,7 @@ public void push(RowMap r) throws Exception {
context.setPosition(r.getNextPosition());
}
if ( LOGGER.isDebugEnabled()) {
- LOGGER.debug("-> routing key:" + routingKey + ", partition:" + value);
+ LOGGER.debug("-> routing key:{}, partition:{}", routingKey, value);
}
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/StdoutProducer.java b/src/main/java/com/zendesk/maxwell/producer/StdoutProducer.java
index 4f73c83fc..c9a45a278 100644
--- a/src/main/java/com/zendesk/maxwell/producer/StdoutProducer.java
+++ b/src/main/java/com/zendesk/maxwell/producer/StdoutProducer.java
@@ -3,6 +3,8 @@
import com.zendesk.maxwell.MaxwellContext;
import com.zendesk.maxwell.row.RowMap;
+import java.util.concurrent.TimeUnit;
+
public class StdoutProducer extends AbstractProducer {
public StdoutProducer(MaxwellContext context) {
super(context);
@@ -15,6 +17,10 @@ public void push(RowMap r) throws Exception {
if ( output != null && r.shouldOutput(outputConfig) )
System.out.println(output);
+ this.messageLatencyTimer.update(
+ System.currentTimeMillis() - r.getTimestampMillis(),
+ TimeUnit.MILLISECONDS
+ );
this.context.setPosition(r);
}
}
diff --git a/src/main/java/com/zendesk/maxwell/producer/package-info.java b/src/main/java/com/zendesk/maxwell/producer/package-info.java
new file mode 100644
index 000000000..2db31a302
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/producer/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * producers write to streams and sinks of various types.
+ */
+package com.zendesk.maxwell.producer;
diff --git a/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSNSPartitioner.java b/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSNSPartitioner.java
new file mode 100644
index 000000000..fc832df29
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSNSPartitioner.java
@@ -0,0 +1,15 @@
+package com.zendesk.maxwell.producer.partitioners;
+
+import com.zendesk.maxwell.row.RowMap;
+
+public class MaxwellSNSPartitioner extends AbstractMaxwellPartitioner {
+
+ public MaxwellSNSPartitioner(String partitionKey, String csvPartitionColumns, String partitionKeyFallback) {
+ super(partitionKey, csvPartitionColumns, partitionKeyFallback);
+ }
+
+ public String getSNSKey(RowMap r) {
+ String key = this.getHashString(r);
+ return key;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSQSPartitioner.java b/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSQSPartitioner.java
new file mode 100644
index 000000000..a9cdab907
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/producer/partitioners/MaxwellSQSPartitioner.java
@@ -0,0 +1,15 @@
+package com.zendesk.maxwell.producer.partitioners;
+
+import com.zendesk.maxwell.row.RowMap;
+
+public class MaxwellSQSPartitioner extends AbstractMaxwellPartitioner {
+
+ public MaxwellSQSPartitioner(String partitionKey, String csvPartitionColumns, String partitionKeyFallback) {
+ super(partitionKey, csvPartitionColumns, partitionKeyFallback);
+ }
+
+ public String getSQSKey(RowMap r) {
+ String key = this.getHashString(r);
+ return key;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/zendesk/maxwell/producer/partitioners/package-info.java b/src/main/java/com/zendesk/maxwell/producer/partitioners/package-info.java
new file mode 100644
index 000000000..c5f31598b
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/producer/partitioners/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * partitioners deal with subdividing streams by data key
+ */
+package com.zendesk.maxwell.producer.partitioners;
diff --git a/src/main/java/com/zendesk/maxwell/recovery/Recovery.java b/src/main/java/com/zendesk/maxwell/recovery/Recovery.java
index 0206bcc86..16dc03a63 100644
--- a/src/main/java/com/zendesk/maxwell/recovery/Recovery.java
+++ b/src/main/java/com/zendesk/maxwell/recovery/Recovery.java
@@ -19,6 +19,7 @@
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
@@ -58,7 +59,7 @@ public HeartbeatRowMap recover() throws Exception {
Position position = Position.valueOf(binlogPosition, recoveryInfo.getHeartbeat());
Metrics metrics = new NoOpMetrics();
- LOGGER.debug("scanning binlog: " + binlogPosition);
+ LOGGER.debug("scanning binlog: {}", binlogPosition);
Replicator replicator = new BinlogConnectorReplicator(
this.schemaStore,
null,
@@ -74,7 +75,8 @@ public HeartbeatRowMap recover() throws Exception {
null,
new RecoveryFilter(this.maxwellDatabaseName),
new MaxwellOutputConfig(),
- 0.25f // Default memory usage size, not used
+ 0.25f, // Default memory usage size, not used
+ 1
);
HeartbeatRowMap h = findHeartbeat(replicator);
@@ -113,8 +115,9 @@ private HeartbeatRowMap findHeartbeat(Replicator r) throws Exception {
private List getBinlogInfo() throws SQLException {
ArrayList list = new ArrayList<>();
- try ( Connection c = replicationConnectionPool.getConnection() ) {
- ResultSet rs = c.createStatement().executeQuery("SHOW BINARY LOGS");
+ try ( Connection c = replicationConnectionPool.getConnection() ;
+ Statement s = c.createStatement();
+ ResultSet rs = s.executeQuery("SHOW BINARY LOGS") ) {
while ( rs.next() ) {
list.add(BinlogPosition.at(4, rs.getString("Log_name")));
}
diff --git a/src/main/java/com/zendesk/maxwell/recovery/package-info.java b/src/main/java/com/zendesk/maxwell/recovery/package-info.java
new file mode 100644
index 000000000..4ccc82771
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/recovery/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * allows users to continue a stream after changing masters
+ */
+package com.zendesk.maxwell.recovery;
diff --git a/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEvent.java b/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEvent.java
index e8f5e1391..f068396ad 100644
--- a/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEvent.java
+++ b/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEvent.java
@@ -59,6 +59,10 @@ public TableMapEventData tableMapData() {
return (TableMapEventData) event.getData();
}
+ public MariadbGtidEventData mariaGtidData() {
+ return (MariadbGtidEventData) event.getData();
+ }
+
public BinlogPosition getPosition() {
return position;
}
diff --git a/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEventListener.java b/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEventListener.java
index 0d892db89..06f5b5670 100644
--- a/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEventListener.java
+++ b/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorEventListener.java
@@ -6,6 +6,7 @@
import com.github.shyiko.mysql.binlog.event.Event;
import com.github.shyiko.mysql.binlog.event.EventType;
import com.github.shyiko.mysql.binlog.event.GtidEventData;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
import com.zendesk.maxwell.monitoring.Metrics;
import com.zendesk.maxwell.producer.MaxwellOutputConfig;
import org.slf4j.Logger;
@@ -50,8 +51,12 @@ public void onEvent(Event event) {
long eventSeenAt = 0;
boolean trackMetrics = false;
- if (event.getHeader().getEventType() == EventType.GTID) {
+ EventType eventType = event.getHeader().getEventType();
+
+ if ( eventType == EventType.GTID) {
gtid = ((GtidEventData)event.getData()).getGtid();
+ } else if ( eventType == EventType.MARIADB_GTID) {
+ gtid = ((MariadbGtidEventData)event.getData()).toString();
}
BinlogConnectorEvent ep = new BinlogConnectorEvent(event, client.getBinlogFilename(), client.getGtidSet(), gtid, outputConfig);
diff --git a/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorReplicator.java b/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorReplicator.java
index 47ff2539c..fe24f17ba 100644
--- a/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorReplicator.java
+++ b/src/main/java/com/zendesk/maxwell/replication/BinlogConnectorReplicator.java
@@ -4,7 +4,9 @@
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.github.shyiko.mysql.binlog.BinaryLogClient;
+import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
import com.github.shyiko.mysql.binlog.event.EventType;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
import com.github.shyiko.mysql.binlog.event.QueryEventData;
import com.github.shyiko.mysql.binlog.event.RowsQueryEventData;
import com.github.shyiko.mysql.binlog.event.TableMapEventData;
@@ -34,11 +36,13 @@
import java.util.Set;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
public class BinlogConnectorReplicator extends RunLoopProcess implements Replicator, BinaryLogClient.LifecycleListener {
static final Logger LOGGER = LoggerFactory.getLogger(BinlogConnectorReplicator.class);
private static final long MAX_TX_ELEMENTS = 10000;
+ public static int BINLOG_QUEUE_SIZE = 5000;
public static final int BAD_BINLOG_ERROR_CODE = 1236;
public static final int ACCESS_DENIED_ERROR_CODE = 1227;
@@ -46,9 +50,10 @@ public class BinlogConnectorReplicator extends RunLoopProcess implements Replica
private final String maxwellSchemaDatabaseName;
protected final BinaryLogClient client;
+ private final int replicationReconnectionRetries;
private BinlogConnectorEventListener binlogEventListener;
private BinlogConnectorLivenessMonitor binlogLivenessMonitor;
- private final LinkedBlockingDeque queue = new LinkedBlockingDeque<>(20);
+ private final LinkedBlockingDeque queue;
private final TableCache tableCache;
private final Scripting scripting;
private ServerException lastCommError;
@@ -60,6 +65,7 @@ public class BinlogConnectorReplicator extends RunLoopProcess implements Replica
private final HeartbeatNotifier heartbeatNotifier;
private Long stopAtHeartbeat;
private Filter filter;
+ private Boolean ignoreMissingSchema;
private final BootstrapController bootstrapper;
private final AbstractProducer producer;
@@ -82,6 +88,46 @@ public class BinlogConnectorReplicator extends RunLoopProcess implements Replica
private class ClientReconnectedException extends Exception {}
+ public BinlogConnectorReplicator(
+ SchemaStore schemaStore,
+ AbstractProducer producer,
+ BootstrapController bootstrapper,
+ MaxwellMysqlConfig mysqlConfig,
+ Long replicaServerID,
+ String maxwellSchemaDatabaseName,
+ Metrics metrics,
+ Position start,
+ boolean stopOnEOF,
+ String clientID,
+ HeartbeatNotifier heartbeatNotifier,
+ Scripting scripting,
+ Filter filter,
+ MaxwellOutputConfig outputConfig,
+ float bufferMemoryUsage,
+ int replicationReconnectionRetries
+ ) {
+ this(
+ schemaStore,
+ producer,
+ bootstrapper,
+ mysqlConfig,
+ replicaServerID,
+ maxwellSchemaDatabaseName,
+ metrics,
+ start,
+ stopOnEOF,
+ clientID,
+ heartbeatNotifier,
+ scripting,
+ filter,
+ false,
+ outputConfig,
+ bufferMemoryUsage,
+ replicationReconnectionRetries,
+ BINLOG_QUEUE_SIZE
+ );
+ }
+
public BinlogConnectorReplicator(
SchemaStore schemaStore,
AbstractProducer producer,
@@ -96,8 +142,11 @@ public BinlogConnectorReplicator(
HeartbeatNotifier heartbeatNotifier,
Scripting scripting,
Filter filter,
+ boolean ignoreMissingSchema,
MaxwellOutputConfig outputConfig,
- float bufferMemoryUsage
+ float bufferMemoryUsage,
+ int replicationReconnectionRetries,
+ int binlogEventQueueSize
) {
this.clientID = clientID;
this.bootstrapper = bootstrapper;
@@ -110,8 +159,10 @@ public BinlogConnectorReplicator(
this.schemaStore = schemaStore;
this.tableCache = new TableCache(maxwellSchemaDatabaseName);
this.filter = filter;
+ this.ignoreMissingSchema = ignoreMissingSchema;
this.lastCommError = null;
this.bufferMemoryUsage = bufferMemoryUsage;
+ this.queue = new LinkedBlockingDeque<>(binlogEventQueueSize);
/* setup metrics */
rowCounter = metrics.getRegistry().counter(
@@ -128,6 +179,8 @@ public BinlogConnectorReplicator(
/* setup binlog client */
this.client = new BinaryLogClient(mysqlConfig.host, mysqlConfig.port, mysqlConfig.user, mysqlConfig.password);
this.client.setSSLMode(mysqlConfig.sslMode);
+ this.client.setUseSendAnnotateRowsEvent(true);
+
BinlogPosition startBinlog = start.getBinlogPosition();
if (startBinlog.getGtidSetStr() != null) {
@@ -167,6 +220,8 @@ public BinlogConnectorReplicator(
this.client.registerEventListener(binlogEventListener);
this.client.registerLifecycleListener(this);
this.client.setServerId(replicaServerID.intValue());
+
+ this.replicationReconnectionRetries = replicationReconnectionRetries;
}
/**
@@ -311,7 +366,7 @@ private RowMap processHeartbeats(RowMap row) {
return row; // plain row -- do not process.
long lastHeartbeatRead = (Long) row.getData("heartbeat");
- LOGGER.debug("replicator picked up heartbeat: " + lastHeartbeatRead);
+ LOGGER.debug("replicator picked up heartbeat: {}", lastHeartbeatRead);
this.lastHeartbeatPosition = row.getPosition().withHeartbeat(lastHeartbeatRead);
heartbeatNotifier.heartbeat(lastHeartbeatRead);
return HeartbeatRowMap.valueOf(row.getDatabase(), this.lastHeartbeatPosition, row.getNextPosition().withHeartbeat(lastHeartbeatRead));
@@ -436,8 +491,7 @@ private void ensureReplicatorThread() throws Exception {
client.setBinlogFilename("");
client.setBinlogPosition(4L);
-
- client.connect(5000);
+ tryReconnect();
throw new ClientReconnectedException();
} else {
@@ -445,11 +499,29 @@ private void ensureReplicatorThread() throws Exception {
// we like, so we don't have to bail out of the middle of an event.
LOGGER.warn("replicator stopped at position: {} -- restarting", client.getBinlogFilename() + ":" + client.getBinlogPosition());
- client.connect(5000);
+ Long oldMasterId = client.getMasterServerId();
+ tryReconnect();
+ if (client.getMasterServerId() != oldMasterId) {
+ throw new Exception("Master id changed from " + oldMasterId + " to " + client.getMasterServerId()
+ + " while using binlog coordinate positioning. Cannot continue with the info that we have");
+ }
}
}
}
+ private void tryReconnect() throws TimeoutException {
+ int reconnectionAttempts = 0;
+
+ while ((reconnectionAttempts += 1) <= this.replicationReconnectionRetries || this.replicationReconnectionRetries == 0) {
+ try {
+ LOGGER.info(String.format("Reconnection attempt: %s of %s", reconnectionAttempts, replicationReconnectionRetries > 0 ? this.replicationReconnectionRetries : "unlimited"));
+ client.connect(5000);
+ return;
+ } catch (IOException | TimeoutException ignored) { }
+ }
+ throw new TimeoutException("Maximum reconnection attempts reached.");
+ }
+
/**
* Get a batch of rows for the current transaction.
*
@@ -514,23 +586,26 @@ private RowMapBuffer getTransactionRows(BinlogConnectorEvent beginEvent) throws
buffer.add(r);
}
}
- currentQuery = null;
break;
case TABLE_MAP:
TableMapEventData data = event.tableMapData();
- tableCache.processEvent(getSchema(), this.filter, data.getTableId(), data.getDatabase(), data.getTable());
+ tableCache.processEvent(getSchema(), this.filter, this.ignoreMissingSchema, data.getTableId(), data.getDatabase(), data.getTable());
break;
case ROWS_QUERY:
RowsQueryEventData rqed = event.getEvent().getData();
currentQuery = rqed.getQuery();
break;
+ case ANNOTATE_ROWS:
+ AnnotateRowsEventData ared = event.getEvent().getData();
+ currentQuery = ared.getRowsQuery();
+ break;
case QUERY:
QueryEventData qe = event.queryData();
String sql = qe.getSql();
String upperCaseSql = sql.toUpperCase();
if ( upperCaseSql.startsWith(BinlogConnectorEvent.SAVEPOINT)) {
- LOGGER.debug("Ignoring SAVEPOINT in transaction: " + qe);
+ LOGGER.debug("Ignoring SAVEPOINT in transaction: {}", qe);
} else if ( createTablePattern.matcher(sql).find() ) {
// CREATE TABLE `foo` SELECT * FROM `bar` will put a CREATE TABLE
// inside a transaction. Note that this could, in rare cases, lead
@@ -548,10 +623,17 @@ private RowMapBuffer getTransactionRows(BinlogConnectorEvent beginEvent) throws
// We don't need to process them, just ignore
} else if (upperCaseSql.startsWith("DROP TEMPORARY TABLE")) {
// Ignore temporary table drop statements inside transactions
+ } else if ( upperCaseSql.startsWith("# DUMMY EVENT")) {
+ // MariaDB injected event
+ } else if ( upperCaseSql.equals("ROLLBACK") ) {
+ LOGGER.debug("rolling back transaction inside binlog.");
+ return new RowMapBuffer(0);
} else {
LOGGER.warn("Unhandled QueryEvent @ {} inside transaction: {}", event.getPosition().fullPosition(), qe);
}
break;
+ default:
+ break;
}
}
}
@@ -574,13 +656,13 @@ private void logColumnDefCastException(Table table, ColumnDefCastException e) {
/**
* The main entry point into the event reading loop.
- *
+ *
* We maintain a buffer of events in a transaction,
* and each subsequent call to `getRow` can grab one from
* the buffer. If that buffer is empty, we'll go check
* the open-replicator buffer for rows to process. If that
* buffer is empty, we return null.
- *
+ *
* @return either a RowMap or null
*/
public RowMap getRow() throws Exception {
@@ -635,7 +717,7 @@ public RowMap getRow() throws Exception {
break;
case TABLE_MAP:
TableMapEventData data = event.tableMapData();
- tableCache.processEvent(getSchema(), this.filter, data.getTableId(), data.getDatabase(), data.getTable());
+ tableCache.processEvent(getSchema(), this.filter,this.ignoreMissingSchema, data.getTableId(), data.getDatabase(), data.getTable());
break;
case QUERY:
QueryEventData qe = event.queryData();
@@ -656,6 +738,22 @@ public RowMap getRow() throws Exception {
processQueryEvent(event);
}
break;
+ case MARIADB_GTID:
+ // in mariaDB the GTID event supplants the normal BEGIN
+ MariadbGtidEventData g = event.mariaGtidData();
+ if ( (g.getFlags() & MariadbGtidEventData.FL_STANDALONE) == 0 ) {
+ try {
+ rowBuffer = getTransactionRows(event);
+ } catch ( ClientReconnectedException e ) {
+ // rowBuffer should already be empty by the time we get to this switch
+ // statement, but we null it for clarity
+ rowBuffer = null;
+ break;
+ }
+ rowBuffer.setServerId(event.getEvent().getHeader().getServerId());
+ rowBuffer.setSchemaId(getSchemaId());
+ }
+ break;
case ROTATE:
tableCache.clear();
if ( stopOnEOF && event.getPosition().getOffset() > 0 ) {
diff --git a/src/main/java/com/zendesk/maxwell/replication/BinlogPosition.java b/src/main/java/com/zendesk/maxwell/replication/BinlogPosition.java
index 6023439f8..85112b6ce 100644
--- a/src/main/java/com/zendesk/maxwell/replication/BinlogPosition.java
+++ b/src/main/java/com/zendesk/maxwell/replication/BinlogPosition.java
@@ -2,6 +2,9 @@
import com.github.shyiko.mysql.binlog.GtidSet;
+import com.github.shyiko.mysql.binlog.MariadbGtidSet;
+import com.zendesk.maxwell.MaxwellCompatibilityError;
+import com.zendesk.maxwell.MaxwellMysqlStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -9,6 +12,7 @@
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
public class BinlogPosition implements Serializable {
static final Logger LOGGER = LoggerFactory.getLogger(BinlogPosition.class);
@@ -17,9 +21,10 @@ public class BinlogPosition implements Serializable {
private static final String POSITION_COLUMN = "Position";
private static final String GTID_COLUMN = "Executed_Gtid_Set";
- private final String gtidSetStr;
+ private String gtidSetStr;
private final String gtid;
private final long offset;
+ private final Long fileNumber;
private final String file;
public BinlogPosition(String gtidSetStr, String gtid, long l, String file) {
@@ -27,6 +32,7 @@ public BinlogPosition(String gtidSetStr, String gtid, long l, String file) {
this.gtid = gtid;
this.offset = l;
this.file = file;
+ this.fileNumber = parseFileNumber(file);
}
public BinlogPosition(long l, String file) {
@@ -34,16 +40,26 @@ public BinlogPosition(long l, String file) {
}
public static BinlogPosition capture(Connection c, boolean gtidMode) throws SQLException {
- ResultSet rs;
- rs = c.createStatement().executeQuery("SHOW MASTER STATUS");
- rs.next();
- long l = rs.getInt(POSITION_COLUMN);
- String file = rs.getString(FILE_COLUMN);
- String gtidSetStr = null;
- if (gtidMode) {
- gtidSetStr = rs.getString(GTID_COLUMN);
+ MaxwellMysqlStatus m = new MaxwellMysqlStatus(c);
+
+
+
+
+ try ( Statement stmt = c.createStatement();
+ ResultSet rs = stmt.executeQuery(m.getShowBinlogSQL())) {
+ rs.next();
+ long l = rs.getInt(POSITION_COLUMN);
+ String file = rs.getString(FILE_COLUMN);
+ String gtidSetStr = null;
+ if (gtidMode) {
+ if ( m.isMaria() ) {
+ gtidSetStr = m.getVariableState("gtid_binlog_state");
+ } else {
+ gtidSetStr = rs.getString(GTID_COLUMN);
+ }
+ }
+ return new BinlogPosition(gtidSetStr, null, l, file);
}
- return new BinlogPosition(gtidSetStr, null, l, file);
}
public static BinlogPosition at(BinlogPosition position) {
@@ -74,8 +90,32 @@ public String getGtidSetStr() {
return gtidSetStr;
}
+ public BinlogPosition addGtid(String gtid, long offset, String file) {
+ GtidSet set = this.getGtidSet();
+ if ( set == null )
+ return new BinlogPosition(offset, file);
+
+ set.add(gtid);
+ return new BinlogPosition(set.toSeenString(), gtid, offset, file);
+ }
+
+ public void mergeGtids(GtidSet seenSet) {
+ if ( seenSet == null ) {
+ this.gtidSetStr = this.gtid;
+ } else {
+ seenSet.add(this.getGtid());
+ this.gtidSetStr = seenSet.toSeenString();
+ }
+ }
+
public GtidSet getGtidSet() {
- return new GtidSet(gtidSetStr);
+ if ( gtidSetStr == null )
+ return null;
+
+ if ( MariadbGtidSet.isMariaGtidSet(gtidSetStr) )
+ return new MariadbGtidSet((gtidSetStr));
+ else
+ return new GtidSet(gtidSetStr);
}
@Override
@@ -92,6 +132,20 @@ public String fullPosition() {
return pos;
}
+ private Long parseFileNumber(String filename) {
+ String[] split = filename.split("\\.");
+ if ( split.length < 2 ) {
+ return null;
+ } else {
+ return Long.valueOf(split[split.length - 1]);
+ }
+ }
+
+ public Long getFileNumber() {
+ return this.fileNumber;
+ }
+
+
public boolean newerThan(BinlogPosition other) {
if ( other == null )
return true;
@@ -100,7 +154,13 @@ public boolean newerThan(BinlogPosition other) {
return !getGtidSet().isContainedWithin(other.getGtidSet());
}
- int cmp = this.file.compareTo(other.file);
+ long cmp;
+ if ( this.fileNumber == null || other.getFileNumber() == null ) {
+ cmp = this.file.compareTo(other.file);
+ } else {
+ cmp = this.fileNumber - other.getFileNumber();
+ }
+
if ( cmp > 0 ) {
return true;
} else if ( cmp == 0 ) {
@@ -132,4 +192,5 @@ public int hashCode() {
return Long.valueOf(offset).hashCode();
}
}
+
}
diff --git a/src/main/java/com/zendesk/maxwell/replication/MysqlVersion.java b/src/main/java/com/zendesk/maxwell/replication/MysqlVersion.java
index 82feb0709..f2e520182 100644
--- a/src/main/java/com/zendesk/maxwell/replication/MysqlVersion.java
+++ b/src/main/java/com/zendesk/maxwell/replication/MysqlVersion.java
@@ -1,5 +1,7 @@
package com.zendesk.maxwell.replication;
+import org.apache.arrow.flatbuf.Int;
+
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.SQLException;
@@ -7,6 +9,7 @@
public class MysqlVersion {
private final int major;
private final int minor;
+ public boolean isMariaDB = false;
public MysqlVersion(int major, int minor) {
this.major = major;
@@ -37,4 +40,20 @@ public int getMajor() {
public int getMinor() {
return this.minor;
}
+
+ public static MysqlVersion parse(String versionString) {
+ if ( versionString.equals("mariadb") ) {
+ MysqlVersion v = new MysqlVersion(0, 0);
+ v.isMariaDB = true;
+ return v;
+ }
+
+
+ String[] split = versionString.split("\\.");
+ if ( split.length < 2 ) {
+ throw new IllegalArgumentException("Invalid version string: " + versionString + ". Expected at least major and minor versions separated by a dot.");
+ }
+ return new MysqlVersion(Integer.parseInt(split[0]), Integer.parseInt(split[1]));
+ }
+
}
diff --git a/src/main/java/com/zendesk/maxwell/replication/Position.java b/src/main/java/com/zendesk/maxwell/replication/Position.java
index 2eb0a97bf..3ed0a4c78 100644
--- a/src/main/java/com/zendesk/maxwell/replication/Position.java
+++ b/src/main/java/com/zendesk/maxwell/replication/Position.java
@@ -35,6 +35,10 @@ public BinlogPosition getBinlogPosition() {
return binlogPosition;
}
+ public Position addGtid(String gtid, long offset, String file) {
+ return new Position(binlogPosition.addGtid(gtid, offset, file), lastHeartbeatRead);
+ }
+
@Override
public String toString() {
return "Position[" + binlogPosition + ", lastHeartbeat=" + lastHeartbeatRead + "]";
diff --git a/src/main/java/com/zendesk/maxwell/replication/Replicator.java b/src/main/java/com/zendesk/maxwell/replication/Replicator.java
index 553e7a821..6e9468266 100644
--- a/src/main/java/com/zendesk/maxwell/replication/Replicator.java
+++ b/src/main/java/com/zendesk/maxwell/replication/Replicator.java
@@ -1,6 +1,5 @@
package com.zendesk.maxwell.replication;
-import com.zendesk.maxwell.filtering.Filter;
import com.zendesk.maxwell.row.RowMap;
import com.zendesk.maxwell.schema.SchemaStoreException;
import com.zendesk.maxwell.schema.Schema;
diff --git a/src/main/java/com/zendesk/maxwell/replication/TableCache.java b/src/main/java/com/zendesk/maxwell/replication/TableCache.java
index fafd6fce9..07d4a5dc4 100644
--- a/src/main/java/com/zendesk/maxwell/replication/TableCache.java
+++ b/src/main/java/com/zendesk/maxwell/replication/TableCache.java
@@ -15,22 +15,28 @@ public TableCache(String maxwellDB) {
}
private final HashMap tableMapCache = new HashMap<>();
- public void processEvent(Schema schema, Filter filter, Long tableId, String dbName, String tblName) {
+ public void processEvent(Schema schema, Filter filter, Boolean ignoreMissingSchema, Long tableId, String dbName, String tblName) {
if ( !tableMapCache.containsKey(tableId)) {
if ( filter.isTableBlacklisted(dbName, tblName) ) {
return;
}
+
Database db = schema.findDatabase(dbName);
- if ( db == null )
- throw new RuntimeException("Couldn't find database " + dbName);
- else {
+ if ( db == null ) {
+ if ( !ignoreMissingSchema || filter.includes(dbName, tblName) )
+ throw new RuntimeException("Couldn't find database " + dbName);
+
+ } else {
Table tbl = db.findTable(tblName);
- if (tbl == null)
- throw new RuntimeException("Couldn't find table " + tblName + " in database " + dbName);
- else
+ if (tbl == null) {
+ if ( !ignoreMissingSchema || filter.includes(dbName, tblName) )
+ throw new RuntimeException("Couldn't find table " + tblName + " in database " + dbName);
+
+ } else {
tableMapCache.put(tableId, tbl);
+ }
}
}
diff --git a/src/main/java/com/zendesk/maxwell/replication/package-info.java b/src/main/java/com/zendesk/maxwell/replication/package-info.java
new file mode 100644
index 000000000..16b9f6638
--- /dev/null
+++ b/src/main/java/com/zendesk/maxwell/replication/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * concerns the operation of the binlog replicator
+ */
+package com.zendesk.maxwell.replication;
diff --git a/src/main/java/com/zendesk/maxwell/row/MaxwellJson.java b/src/main/java/com/zendesk/maxwell/row/MaxwellJson.java
index 3990dff1d..a14cfc0e4 100644
--- a/src/main/java/com/zendesk/maxwell/row/MaxwellJson.java
+++ b/src/main/java/com/zendesk/maxwell/row/MaxwellJson.java
@@ -3,7 +3,7 @@
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.zendesk.maxwell.scripting.Scripting;
-import jdk.nashorn.api.scripting.ScriptObjectMirror;
+import org.openjdk.nashorn.api.scripting.ScriptObjectMirror;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/main/java/com/zendesk/maxwell/row/RowMap.java b/src/main/java/com/zendesk/maxwell/row/RowMap.java
index 158bbb1be..37b3343b1 100644
--- a/src/main/java/com/zendesk/maxwell/row/RowMap.java
+++ b/src/main/java/com/zendesk/maxwell/row/RowMap.java
@@ -18,6 +18,7 @@
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import com.fasterxml.jackson.core.JsonGenerator;
@@ -149,7 +150,13 @@ public String toJSON(MaxwellOutputConfig outputConfig) throws Exception {
g.writeStringField(fieldNameStrategy.apply(FieldNames.TABLE), this.table);
if ( outputConfig.includesRowQuery && this.rowQuery != null) {
- g.writeStringField(fieldNameStrategy.apply(FieldNames.QUERY), this.rowQuery);
+ String outputQuery;
+ if(outputConfig.rowQueryMaxLength > 0 && this.rowQuery.length() > outputConfig.rowQueryMaxLength){
+ outputQuery = this.rowQuery.substring(0,outputConfig.rowQueryMaxLength);
+ }else{
+ outputQuery = this.rowQuery;
+ }
+ g.writeStringField(fieldNameStrategy.apply(FieldNames.QUERY), outputQuery);
}
g.writeStringField(fieldNameStrategy.apply(FieldNames.TYPE), this.rowType);
@@ -447,4 +454,20 @@ public String getComment() {
public void setComment(String comment) {
this.comment = comment;
}
+
+ public List getPrimaryKeyColumns() {
+ return Collections.unmodifiableList(this.pkColumns);
+ }
+
+ public List