-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
AuroraClusterMonitor reliability changes
This updates the transitive dependencies and attempts to change AuroraClusterMonitor logic for improved reliability. There are several changes involved in this: * First the connection is checked for isValid before it used to check master / slave status. This hopefully prevents errors from invalid connections which might remove the server from selection * Errors during connection are now delayed. This allows the monitor to be constructed while some of the cluster is unhealthy. The delayed error will still be found and logged quickly, it just wont prevent the monitor from being constructed (which may otherwise prevent an application from being usable / startup) * Errors during regular scheduled state check are handled differently. If an error occurs from an expedited state check the health status will still transition immediately. This is desired because a connection has already witnessed an error so we should treat that seriously. However if an error occurs from a regular check it will be ignored initially, with a new check scheduled quickly. It's possible that new connections can't be established but existing ones are working, so we are attempting to delay impacting those existing connections use. In the future we may want to inspect the error type to make even more decisions here. * Added the ability to get notified on every cluster state check error, but only log when the error state changes
- Loading branch information
Showing
14 changed files
with
314 additions
and
92 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
arcCommon/src/main/java/org/threadly/db/ErrorInvalidSqlConnection.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package org.threadly.db; | ||
|
|
||
| import java.sql.Connection; | ||
| import java.sql.SQLException; | ||
|
|
||
| /** | ||
| * Implementation of {@link Connection} which is perpetually in a state of error. Any operation on | ||
| * this connection will result in an exception being thrown. | ||
| * <p> | ||
| * The connection will appear invalid when checked by {@link #isValid(int)}, always returning | ||
| * {@code false}. | ||
| * | ||
| * @since 0.10 | ||
| */ | ||
| public class ErrorInvalidSqlConnection extends AbstractErrorSqlConnection { | ||
| /** | ||
| * Construct a new {@link ErrorInvalidSqlConnection}. | ||
| * | ||
| * @param errorThrownListener Listener to be invoked when error is realized (ie thrown) | ||
| * @param error Error to throw once Connection is attempted to be used | ||
| */ | ||
| public ErrorInvalidSqlConnection(Runnable errorThrownListener, SQLException error) { | ||
| super(errorThrownListener, error); | ||
| } | ||
|
|
||
| /** | ||
| * Construct a new {@link ErrorInvalidSqlConnection}. | ||
| * | ||
| * @param errorThrownListener Listener to be invoked when error is realized (ie thrown) | ||
| * @param error Error to throw once Connection is attempted to be used | ||
| */ | ||
| public ErrorInvalidSqlConnection(Runnable errorThrownListener, RuntimeException error) { | ||
| super(errorThrownListener, error); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isValid(int timeout) { | ||
| return false; | ||
| } | ||
| } |
40 changes: 40 additions & 0 deletions
40
arcCommon/src/main/java/org/threadly/db/ErrorValidSqlConnection.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package org.threadly.db; | ||
|
|
||
| import java.sql.Connection; | ||
| import java.sql.SQLException; | ||
|
|
||
| /** | ||
| * Implementation of {@link Connection} which is perpetually in a state of error. Any operation on | ||
| * this connection will result in an exception being thrown. | ||
| * <p> | ||
| * The connection will appear valid (from {@link #isValid(int)} and non-closed UNTIL the exception | ||
| * is thrown. After that point the connection will appear as if it was closed. | ||
| * | ||
| * @since 0.10 | ||
| */ | ||
| public class ErrorValidSqlConnection extends AbstractErrorSqlConnection { | ||
| /** | ||
| * Construct a new {@link ErrorValidSqlConnection}. | ||
| * | ||
| * @param errorThrownListener Listener to be invoked when error is realized (ie thrown) | ||
| * @param error Error to throw once Connection is attempted to be used | ||
| */ | ||
| public ErrorValidSqlConnection(Runnable errorThrownListener, SQLException error) { | ||
| super(errorThrownListener, error); | ||
| } | ||
|
|
||
| /** | ||
| * Construct a new {@link ErrorValidSqlConnection}. | ||
| * | ||
| * @param errorThrownListener Listener to be invoked when error is realized (ie thrown) | ||
| * @param error Error to throw once Connection is attempted to be used | ||
| */ | ||
| public ErrorValidSqlConnection(Runnable errorThrownListener, RuntimeException error) { | ||
| super(errorThrownListener, error); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isValid(int timeout) { | ||
| return ! isClosed(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.