Introduce FaultTolerantRedisClient

service/config/sample.yml

@@ -163,9 +163,8 @@ cacheCluster: # Redis server configuration for cache cluster
 clientPresenceCluster: # Redis server configuration for client presence cluster
   configurationUri: redis://redis.example.com:6379/
 
-provisioning:
-  pubsub: # Redis server configuration for pubsub cluster
-    uri: redis://redis.example.com:6379/
+pubsub: # Redis server configuration for pubsub cluster
+  uri: redis://redis.example.com:6379/
 
 pushSchedulerCluster: # Redis server configuration for push scheduler cluster
   configurationUri: redis://redis.example.com:6379/

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java

@@ -23,7 +23,6 @@
 import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration;
 import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration;
 import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
-import org.whispersystems.textsecuregcm.configuration.ClientCdnConfiguration;
 import org.whispersystems.textsecuregcm.configuration.ClientReleaseConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DefaultAwsCredentialsFactory;
@@ -34,6 +33,7 @@
 import org.whispersystems.textsecuregcm.configuration.DynamoDbTables;
 import org.whispersystems.textsecuregcm.configuration.ExternalRequestFilterConfiguration;
 import org.whispersystems.textsecuregcm.configuration.FaultTolerantRedisClusterFactory;
+import org.whispersystems.textsecuregcm.configuration.FaultTolerantRedisClientFactory;
 import org.whispersystems.textsecuregcm.configuration.FcmConfiguration;
 import org.whispersystems.textsecuregcm.configuration.GcpAttachmentsConfiguration;
 import org.whispersystems.textsecuregcm.configuration.GenericZkConfig;
@@ -47,7 +47,6 @@
 import org.whispersystems.textsecuregcm.configuration.NoiseWebSocketTunnelConfiguration;
 import org.whispersystems.textsecuregcm.configuration.OneTimeDonationConfiguration;
 import org.whispersystems.textsecuregcm.configuration.PaymentsServiceConfiguration;
-import org.whispersystems.textsecuregcm.configuration.ProvisioningConfiguration;
 import org.whispersystems.textsecuregcm.configuration.RegistrationServiceClientFactory;
 import org.whispersystems.textsecuregcm.configuration.RemoteConfigConfiguration;
 import org.whispersystems.textsecuregcm.configuration.ReportMessageConfiguration;
@@ -143,7 +142,7 @@ public class WhisperServerConfiguration extends Configuration {
   @NotNull
   @Valid
   @JsonProperty
-  private ProvisioningConfiguration provisioning;
+  private FaultTolerantRedisClientFactory pubsub;
 
   @NotNull
   @Valid
@@ -410,8 +409,8 @@ public FaultTolerantRedisClusterFactory getCacheClusterConfiguration() {
     return cacheCluster;
   }
 
-  public ProvisioningConfiguration getProvisioningConfiguration() {
-    return provisioning;
+  public FaultTolerantRedisClientFactory getRedisPubSubConfiguration() {
+    return pubsub;
   }
 
   public SecureValueRecovery2Configuration getSvr2Configuration() {

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java

@@ -200,7 +200,8 @@
 import org.whispersystems.textsecuregcm.push.PushNotificationScheduler;
 import org.whispersystems.textsecuregcm.push.ReceiptSender;
 import org.whispersystems.textsecuregcm.redis.ConnectionEventLogger;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.registration.RegistrationServiceClient;
 import org.whispersystems.textsecuregcm.s3.PolicySigner;
 import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
@@ -447,18 +448,21 @@ public void run(WhisperServerConfiguration config, Environment environment) thro
         .build();
     ConnectionEventLogger.logConnectionEvents(sharedClientResources);
 
-    FaultTolerantRedisCluster cacheCluster = config.getCacheClusterConfiguration()
+    FaultTolerantRedisClusterClient cacheCluster = config.getCacheClusterConfiguration()
         .build("main_cache", sharedClientResources.mutate());
-    FaultTolerantRedisCluster messagesCluster =
+    FaultTolerantRedisClusterClient messagesCluster =
         config.getMessageCacheConfiguration().getRedisClusterConfiguration()
             .build("messages", sharedClientResources.mutate());
-    FaultTolerantRedisCluster clientPresenceCluster = config.getClientPresenceClusterConfiguration()
+    FaultTolerantRedisClusterClient clientPresenceCluster = config.getClientPresenceClusterConfiguration()
         .build("client_presence", sharedClientResources.mutate());
-    FaultTolerantRedisCluster pushSchedulerCluster = config.getPushSchedulerCluster().build("push_scheduler",
+    FaultTolerantRedisClusterClient pushSchedulerCluster = config.getPushSchedulerCluster().build("push_scheduler",
         sharedClientResources.mutate());
-    FaultTolerantRedisCluster rateLimitersCluster = config.getRateLimitersCluster().build("rate_limiters",
+    FaultTolerantRedisClusterClient rateLimitersCluster = config.getRateLimitersCluster().build("rate_limiters",
         sharedClientResources.mutate());
 
+    FaultTolerantRedisClient pubsubClient =
+        config.getRedisPubSubConfiguration().build("pubsub", sharedClientResources);
+
     final BlockingQueue<Runnable> keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(100_000);
     Metrics.gaugeCollectionSize(name(getClass(), "keyspaceNotificationDispatchQueueSize"), Collections.emptyList(),
         keyspaceNotificationDispatchQueue);
@@ -652,9 +656,7 @@ public void run(WhisperServerConfiguration config, Environment environment) thro
         new PushNotificationManager(accountsManager, apnSender, fcmSender, pushNotificationScheduler);
     RateLimiters rateLimiters = RateLimiters.createAndValidate(config.getLimitsConfiguration(),
         dynamicConfigurationManager, rateLimitersCluster);
-    ProvisioningManager provisioningManager = new ProvisioningManager(
-        config.getProvisioningConfiguration().pubsub().build(sharedClientResources),
-        config.getProvisioningConfiguration().circuitBreaker());
+    ProvisioningManager provisioningManager = new ProvisioningManager(pubsubClient);
     IssuedReceiptsManager issuedReceiptsManager = new IssuedReceiptsManager(
         config.getDynamoDbTables().getIssuedReceipts().getTableName(),
         config.getDynamoDbTables().getIssuedReceipts().getExpiration(),

service/src/main/java/org/whispersystems/textsecuregcm/configuration/SingletonRedisClientFactory.java → service/src/main/java/org/whispersystems/textsecuregcm/configuration/FaultTolerantRedisClientFactory.java

@@ -7,11 +7,11 @@
 
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import io.dropwizard.jackson.Discoverable;
-import io.lettuce.core.RedisClient;
 import io.lettuce.core.resource.ClientResources;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = RedisConfiguration.class)
-public interface SingletonRedisClientFactory extends Discoverable {
+public interface FaultTolerantRedisClientFactory extends Discoverable {
 
-  RedisClient build(ClientResources clientResources);
+  FaultTolerantRedisClient build(String name, ClientResources clientResources);
 }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/FaultTolerantRedisClusterFactory.java

@@ -8,10 +8,10 @@
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import io.dropwizard.jackson.Discoverable;
 import io.lettuce.core.resource.ClientResources;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = RedisClusterConfiguration.class)
 public interface FaultTolerantRedisClusterFactory extends Discoverable {
 
-  FaultTolerantRedisCluster build(String name, ClientResources.Builder clientResourcesBuilder);
+  FaultTolerantRedisClusterClient build(String name, ClientResources.Builder clientResourcesBuilder);
 }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/RedisClusterConfiguration.java

@@ -13,7 +13,7 @@
 import javax.validation.Valid;
 import javax.validation.constraints.NotEmpty;
 import javax.validation.constraints.NotNull;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 
 @JsonTypeName("default")
 public class RedisClusterConfiguration implements FaultTolerantRedisClusterFactory {
@@ -58,7 +58,7 @@ public RetryConfiguration getRetryConfiguration() {
   }
 
   @Override
-  public FaultTolerantRedisCluster build(final String name, final ClientResources.Builder clientResourcesBuilder) {
-    return new FaultTolerantRedisCluster(name, this, clientResourcesBuilder);
+  public FaultTolerantRedisClusterClient build(final String name, final ClientResources.Builder clientResourcesBuilder) {
+    return new FaultTolerantRedisClusterClient(name, this, clientResourcesBuilder);
   }
 }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/RedisConfiguration.java

@@ -7,15 +7,16 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
-import io.lettuce.core.RedisClient;
+import com.google.common.annotations.VisibleForTesting;
 import io.lettuce.core.resource.ClientResources;
 import java.time.Duration;
+import javax.validation.Valid;
 import javax.validation.constraints.NotEmpty;
 import javax.validation.constraints.NotNull;
-import org.whispersystems.textsecuregcm.redis.RedisUriUtil;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 
 @JsonTypeName("default")
-public class RedisConfiguration implements SingletonRedisClientFactory {
+public class RedisConfiguration implements FaultTolerantRedisClientFactory {
 
   @JsonProperty
   @NotEmpty
@@ -25,20 +26,39 @@ public class RedisConfiguration implements SingletonRedisClientFactory {
   @NotNull
   private Duration timeout = Duration.ofSeconds(1);
 
+  @JsonProperty
+  @NotNull
+  @Valid
+  private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
+
+  @JsonProperty
+  @NotNull
+  @Valid
+  private RetryConfiguration retry = new RetryConfiguration();
+
   public String getUri() {
     return uri;
   }
 
+  @VisibleForTesting
+  public void setUri(String uri) {
+    this.uri = uri;
+  }
+
   public Duration getTimeout() {
     return timeout;
   }
 
-  @Override
-  public RedisClient build(final ClientResources clientResources) {
-    final RedisClient redisClient = RedisClient.create(clientResources,
-        RedisUriUtil.createRedisUriWithTimeout(uri, timeout));
-    redisClient.setDefaultTimeout(timeout);
+  public @NotNull @Valid CircuitBreakerConfiguration getCircuitBreakerConfiguration() {
+    return circuitBreaker;
+  }
 
-    return redisClient;
+  public @NotNull @Valid RetryConfiguration getRetryConfiguration() {
+    return retry;
+  }
+
+  @Override
+  public FaultTolerantRedisClient build(final String name, final ClientResources clientResources) {
+    return new FaultTolerantRedisClient(name, this, clientResources.mutate());
   }
 }

service/src/main/java/org/whispersystems/textsecuregcm/currency/CurrencyConversionManager.java

@@ -27,7 +27,7 @@
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.entities.CurrencyConversionEntity;
 import org.whispersystems.textsecuregcm.entities.CurrencyConversionEntityList;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 
 public class CurrencyConversionManager implements Managed {
 
@@ -47,7 +47,7 @@ public class CurrencyConversionManager implements Managed {
 
   private final CoinMarketCapClient coinMarketCapClient;
 
-  private final FaultTolerantRedisCluster cacheCluster;
+  private final FaultTolerantRedisClusterClient cacheCluster;
 
   private final Clock clock;
 
@@ -67,7 +67,7 @@ public class CurrencyConversionManager implements Managed {
   public CurrencyConversionManager(
       final FixerClient fixerClient,
       final CoinMarketCapClient coinMarketCapClient,
-      final FaultTolerantRedisCluster cacheCluster,
+      final FaultTolerantRedisClusterClient cacheCluster,
       final List<String> currencies,
       final ScheduledExecutorService executor,
       final Clock clock) {

service/src/main/java/org/whispersystems/textsecuregcm/limits/BaseRateLimiters.java

@@ -22,7 +22,7 @@
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
 import org.whispersystems.textsecuregcm.redis.ClusterLuaScript;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
 
 public abstract class BaseRateLimiters<T extends RateLimiterDescriptor> {
@@ -39,7 +39,7 @@ protected BaseRateLimiters(
       final Map<String, RateLimiterConfig> configs,
       final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
       final ClusterLuaScript validateScript,
-      final FaultTolerantRedisCluster cacheCluster,
+      final FaultTolerantRedisClusterClient cacheCluster,
       final Clock clock) {
     this.configs = configs;
     this.rateLimiterByDescriptor = Arrays.stream(values)
@@ -69,7 +69,7 @@ public void validateValuesAndConfigs() {
     }
   }
 
-  protected static ClusterLuaScript defaultScript(final FaultTolerantRedisCluster cacheCluster) {
+  protected static ClusterLuaScript defaultScript(final FaultTolerantRedisClusterClient cacheCluster) {
     try {
       return ClusterLuaScript.fromResource(
           cacheCluster, "lua/validate_rate_limit.lua", ScriptOutputType.INTEGER);
@@ -83,7 +83,7 @@ private static RateLimiter createForDescriptor(
       final Map<String, RateLimiterConfig> configs,
       final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
       final ClusterLuaScript validateScript,
-      final FaultTolerantRedisCluster cacheCluster,
+      final FaultTolerantRedisClusterClient cacheCluster,
       final Clock clock) {
     if (descriptor.isDynamic()) {
       final Supplier<RateLimiterConfig> configResolver = () -> {

service/src/main/java/org/whispersystems/textsecuregcm/limits/CardinalityEstimator.java

@@ -12,7 +12,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.util.Util;
 
 /**
@@ -21,11 +21,11 @@
 public class CardinalityEstimator {
 
   private volatile double uniqueElementCount;
-  private final FaultTolerantRedisCluster redisCluster;
+  private final FaultTolerantRedisClusterClient redisCluster;
   private final String hllName;
   private final Duration period;
 
-  public CardinalityEstimator(final FaultTolerantRedisCluster redisCluster, final String name, final Duration period) {
+  public CardinalityEstimator(final FaultTolerantRedisClusterClient redisCluster, final String name, final Duration period) {
     this.redisCluster = redisCluster;
     this.hllName = "cardinality_estimator::" + name;
     this.period = period;

service/src/main/java/org/whispersystems/textsecuregcm/limits/DynamicRateLimiter.java

@@ -15,7 +15,7 @@
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
 import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
 import org.whispersystems.textsecuregcm.redis.ClusterLuaScript;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
 
 public class DynamicRateLimiter implements RateLimiter {
@@ -26,7 +26,7 @@ public class DynamicRateLimiter implements RateLimiter {
 
   private final ClusterLuaScript validateScript;
 
-  private final FaultTolerantRedisCluster cluster;
+  private final FaultTolerantRedisClusterClient cluster;
 
   private final Clock clock;
 
@@ -38,7 +38,7 @@ public DynamicRateLimiter(
       final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
       final Supplier<RateLimiterConfig> configResolver,
       final ClusterLuaScript validateScript,
-      final FaultTolerantRedisCluster cluster,
+      final FaultTolerantRedisClusterClient cluster,
       final Clock clock) {
     this.name = requireNonNull(name);
     this.dynamicConfigurationManager = dynamicConfigurationManager;

service/src/main/java/org/whispersystems/textsecuregcm/limits/MessageDeliveryLoopMonitor.java

@@ -11,7 +11,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.redis.ClusterLuaScript;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 
 public class MessageDeliveryLoopMonitor {
 
@@ -22,7 +22,7 @@ public class MessageDeliveryLoopMonitor {
 
   private static final Logger logger = LoggerFactory.getLogger(MessageDeliveryLoopMonitor.class);
 
-  public MessageDeliveryLoopMonitor(final FaultTolerantRedisCluster rateLimitCluster) {
+  public MessageDeliveryLoopMonitor(final FaultTolerantRedisClusterClient rateLimitCluster) {
     try {
       getDeliveryAttemptsScript =
           ClusterLuaScript.fromResource(rateLimitCluster, "lua/get_delivery_attempt_count.lua", ScriptOutputType.INTEGER);

service/src/main/java/org/whispersystems/textsecuregcm/limits/RateLimiters.java

@@ -11,7 +11,7 @@
 import java.util.Map;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
 import org.whispersystems.textsecuregcm.redis.ClusterLuaScript;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
 
 public class RateLimiters extends BaseRateLimiters<RateLimiters.For> {
@@ -81,7 +81,7 @@ public RateLimiterConfig defaultConfig() {
   public static RateLimiters createAndValidate(
       final Map<String, RateLimiterConfig> configs,
       final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
-      final FaultTolerantRedisCluster cacheCluster) {
+      final FaultTolerantRedisClusterClient cacheCluster) {
     final RateLimiters rateLimiters = new RateLimiters(
         configs, dynamicConfigurationManager, defaultScript(cacheCluster), cacheCluster, Clock.systemUTC());
     rateLimiters.validateValuesAndConfigs();
@@ -93,7 +93,7 @@ public static RateLimiters createAndValidate(
       final Map<String, RateLimiterConfig> configs,
       final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
       final ClusterLuaScript validateScript,
-      final FaultTolerantRedisCluster cacheCluster,
+      final FaultTolerantRedisClusterClient cacheCluster,
       final Clock clock) {
     super(For.values(), configs, dynamicConfigurationManager, validateScript, cacheCluster, clock);
   }