Use central registries for Retry and CircuitBreaker instances

integration-tests/src/main/java/org/signal/integration/Operations.java

@@ -41,7 +41,6 @@ import org.signal.libsignal.protocol.kem.KEMKeyType;
 import org.signal.libsignal.protocol.kem.KEMPublicKey;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
 import org.whispersystems.textsecuregcm.entities.AccountAttributes;
 import org.whispersystems.textsecuregcm.entities.AccountIdentityResponse;
 import org.whispersystems.textsecuregcm.entities.DeviceActivationRequest;
@@ -306,11 +305,7 @@ public final class Operations {
 
   private static FaultTolerantHttpClient buildClient() {
     try {
-      return FaultTolerantHttpClient.newBuilder()
+      return FaultTolerantHttpClient.newBuilder("integration-test", Executors.newFixedThreadPool(16))
-          .withName("integration-test")
-          .withExecutor(Executors.newFixedThreadPool(16))
-          .withRetryExecutor(Executors.newSingleThreadScheduledExecutor())
-          .withCircuitBreaker(new CircuitBreakerConfiguration())
           .withTrustedServerCertificates(CONFIG.rootCert())
           .build();
     } catch (final CertificateException e) {

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java

@@ -9,6 +9,7 @@ import io.dropwizard.core.Configuration;
 import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotNull;
 import java.time.Duration;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -22,6 +23,7 @@ import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration;
 import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration;
 import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration;
 import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
+import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
 import org.whispersystems.textsecuregcm.configuration.ClientReleaseConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DefaultAwsCredentialsFactory;
@@ -51,6 +53,7 @@ import org.whispersystems.textsecuregcm.configuration.PaymentsServiceConfigurati
 import org.whispersystems.textsecuregcm.configuration.RegistrationServiceClientFactory;
 import org.whispersystems.textsecuregcm.configuration.RemoteConfigConfiguration;
 import org.whispersystems.textsecuregcm.configuration.ReportMessageConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 import org.whispersystems.textsecuregcm.configuration.S3ObjectMonitorFactory;
 import org.whispersystems.textsecuregcm.configuration.SecureStorageServiceConfiguration;
 import org.whispersystems.textsecuregcm.configuration.SecureValueRecoveryConfiguration;
@@ -340,6 +343,12 @@ public class WhisperServerConfiguration extends Configuration {
   private IdlePrimaryDeviceReminderConfiguration idlePrimaryDeviceReminder =
       new IdlePrimaryDeviceReminderConfiguration(Duration.ofDays(30));
 
+  @JsonProperty
+  private Map<String, CircuitBreakerConfiguration> circuitBreakers = Collections.emptyMap();
+
+  @JsonProperty
+  private Map<String, RetryConfiguration> retries = Collections.emptyMap();
+
   public TlsKeyStoreConfiguration getTlsKeyStoreConfiguration() {
     return tlsKeyStore;
   }
@@ -562,4 +571,12 @@ public class WhisperServerConfiguration extends Configuration {
   public IdlePrimaryDeviceReminderConfiguration idlePrimaryDeviceReminderConfiguration() {
     return idlePrimaryDeviceReminder;
   }
+
+  public Map<String, CircuitBreakerConfiguration> getCircuitBreakerConfigurations() {
+    return circuitBreakers;
+  }
+
+  public Map<String, RetryConfiguration> getRetryConfigurations() {
+    return retries;
+  }
 }

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java

@@ -257,6 +257,7 @@ import org.whispersystems.textsecuregcm.subscriptions.BraintreeManager;
 import org.whispersystems.textsecuregcm.subscriptions.GooglePlayBillingManager;
 import org.whispersystems.textsecuregcm.subscriptions.StripeManager;
 import org.whispersystems.textsecuregcm.util.BufferingInterceptor;
+import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
 import org.whispersystems.textsecuregcm.util.ManagedAwsCrt;
 import org.whispersystems.textsecuregcm.util.ManagedExecutors;
 import org.whispersystems.textsecuregcm.util.SystemMapper;
@@ -363,6 +364,12 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
 
     UncaughtExceptionHandler.register();
 
+    config.getCircuitBreakerConfigurations().forEach((name, configuration) ->
+        CircuitBreakerUtil.getCircuitBreakerRegistry().addConfiguration(name, configuration.toCircuitBreakerConfig()));
+
+    config.getRetryConfigurations().forEach((name, configuration) ->
+        CircuitBreakerUtil.getRetryRegistry().addConfiguration(name, configuration.toRetryConfigBuilder().build()));
+
     ScheduledExecutorService dynamicConfigurationExecutor = ScheduledExecutorServiceBuilder.of(environment, "dynamicConfiguration")
         .threads(1).build();
 
@@ -706,9 +713,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
         config.getTurnConfiguration().cloudflare().urlsWithIps(),
         config.getTurnConfiguration().cloudflare().hostname(),
         config.getTurnConfiguration().cloudflare().numHttpClients(),
-        config.getTurnConfiguration().cloudflare().circuitBreaker(),
+        config.getTurnConfiguration().cloudflare().circuitBreakerConfigurationName(),
         cloudflareTurnHttpExecutor,
-        config.getTurnConfiguration().cloudflare().retry(),
+        config.getTurnConfiguration().cloudflare().retryConfigurationName(),
         cloudflareTurnRetryExecutor,
         cloudflareDnsResolver
         );
@@ -741,7 +748,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
         config.getBraintree().environment(),
         config.getBraintree().supportedCurrenciesByPaymentMethod(), config.getBraintree().merchantAccounts(),
         config.getBraintree().graphqlUrl(), currencyManager, config.getBraintree().pubSubPublisher().build(),
-        config.getBraintree().circuitBreaker(), subscriptionProcessorExecutor,
+        config.getBraintree().circuitBreakerConfigurationName(), subscriptionProcessorExecutor,
         subscriptionProcessorRetryExecutor);
     GooglePlayBillingManager googlePlayBillingManager = new GooglePlayBillingManager(
         new ByteArrayInputStream(config.getGooglePlayBilling().credentialsJson().value().getBytes(StandardCharsets.UTF_8)),
@@ -755,7 +762,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
         config.getAppleAppStore().encodedKey().value(), config.getAppleAppStore().subscriptionGroupId(),
         config.getAppleAppStore().productIdToLevel(),
         config.getAppleAppStore().appleRootCerts(),
-        config.getAppleAppStore().retry(), appleAppStoreExecutor, appleAppStoreRetryExecutor);
+        config.getAppleAppStore().retryConfigurationName(), appleAppStoreExecutor, appleAppStoreRetryExecutor);
 
     environment.lifecycle().manage(apnSender);
     environment.lifecycle().manage(pushNotificationScheduler);

service/src/main/java/org/whispersystems/textsecuregcm/auth/CloudflareTurnCredentialsManager.java

@@ -19,10 +19,9 @@ import java.util.List;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
+import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
 import org.whispersystems.textsecuregcm.util.ExceptionUtils;
 import org.whispersystems.textsecuregcm.util.SystemMapper;
@@ -60,18 +59,15 @@ public class CloudflareTurnCredentialsManager {
       final List<String> cloudflareTurnUrlsWithIps,
       final String cloudflareTurnHostname,
       final int cloudflareTurnNumHttpClients,
-      final CircuitBreakerConfiguration circuitBreaker,
+      @Nullable final String circuitBreakerConfigurationName,
       final ExecutorService executor,
-      final RetryConfiguration retry,
+      @Nullable final String retryConfigurationName,
       final ScheduledExecutorService retryExecutor,
       final DnsNameResolver dnsNameResolver) {
 
-    this.cloudflareTurnClient = FaultTolerantHttpClient.newBuilder()
+    this.cloudflareTurnClient = FaultTolerantHttpClient.newBuilder("cloudflare-turn", executor)
-        .withName("cloudflare-turn")
+        .withCircuitBreaker(circuitBreakerConfigurationName)
-        .withCircuitBreaker(circuitBreaker)
+        .withRetry(retryConfigurationName, retryExecutor)
-        .withExecutor(executor)
-        .withRetry(retry)
-        .withRetryExecutor(retryExecutor)
         .withNumClients(cloudflareTurnNumHttpClients)
         .build();
     this.cloudflareTurnUrls = cloudflareTurnUrls;

service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java

@@ -67,12 +67,9 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
     this.clientSecret = configuration.clientSecret().value();
 
     // Client used for calls to storage-manager
-    this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder()
+    this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder("cdn3-storage-manager", httpExecutor)
-        .withName("cdn3-storage-manager")
+        .withCircuitBreaker(configuration.circuitBreakerConfigurationName())
-        .withCircuitBreaker(configuration.circuitBreaker())
+        .withRetry(configuration.retryConfigurationName(), retryExecutor)
-        .withExecutor(httpExecutor)
-        .withRetryExecutor(retryExecutor)
-        .withRetry(configuration.retry())
         .withConnectTimeout(Duration.ofSeconds(10))
         .withVersion(HttpClient.Version.HTTP_2)
         .withNumClients(configuration.numHttpClients())

service/src/main/java/org/whispersystems/textsecuregcm/configuration/AppleAppStoreConfiguration.java

@@ -12,6 +12,7 @@ import jakarta.validation.constraints.NotNull;
 import java.util.List;
 import java.util.Map;
 import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
+import javax.annotation.Nullable;
 
 /**
  * @param env                 The ios environment to use, typically SANDBOX or PRODUCTION
@@ -26,6 +27,8 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
  *                            subscription levels
  * @param appleRootCerts      Apple root certificates to verify signed API responses, encoded as base64 strings:
  *                            https://www.apple.com/certificateauthority/
+ * @param retryConfigurationName The name of the retry configuration to use in the App Store client; if `null`, uses the
+ *                               global default configuration.
  */
 public record AppleAppStoreConfiguration(
     @NotNull Environment env,
@@ -37,11 +40,5 @@ public record AppleAppStoreConfiguration(
     @NotBlank String subscriptionGroupId,
     @NotNull Map<String, Long> productIdToLevel,
     @NotNull List<@NotBlank String> appleRootCerts,
-    @NotNull @Valid RetryConfiguration retry) {
+    @Nullable String retryConfigurationName) {
-
-  public AppleAppStoreConfiguration {
-    if (retry == null) {
-      retry = new RetryConfiguration();
-    }
-  }
 }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/BraintreeConfiguration.java

@@ -11,6 +11,7 @@ import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
 import java.util.Map;
 import java.util.Set;
+import javax.annotation.Nullable;
 import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
 import org.whispersystems.textsecuregcm.subscriptions.PaymentMethod;
 
@@ -22,7 +23,8 @@ import org.whispersystems.textsecuregcm.subscriptions.PaymentMethod;
  * @param supportedCurrenciesByPaymentMethod the set of supported currencies
  * @param graphqlUrl          the Braintree GraphQL URl to use (this must match the environment)
  * @param merchantAccounts    merchant account within the merchant for processing individual currencies
- * @param circuitBreaker      configuration for the circuit breaker used by the GraphQL HTTP client
+ * @param circuitBreakerConfigurationName the name of the circuit breaker configuration for the breaker used by the
+ *                                        GraphQL HTTP client; if `null`, uses the global default configuration
  */
 public record BraintreeConfiguration(@NotBlank String merchantId,
                                      @NotBlank String publicKey,
@@ -31,15 +33,6 @@ public record BraintreeConfiguration(@NotBlank String merchantId,
                                      @Valid @NotEmpty Map<PaymentMethod, Set<@NotBlank String>> supportedCurrenciesByPaymentMethod,
                                      @NotBlank String graphqlUrl,
                                      @NotEmpty Map<String, String> merchantAccounts,
-                                     @NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
+                                     @Nullable String circuitBreakerConfigurationName,
                                      @Valid @NotNull PubSubPublisherFactory pubSubPublisher) {
-
-  public BraintreeConfiguration {
-    if (circuitBreaker == null) {
-      // It’s a little counter-intuitive, but this compact constructor allows a default value
-      // to be used when one isn’t specified (e.g. in YAML), allowing the field to still be
-      // validated as @NotNull
-      circuitBreaker = new CircuitBreakerConfiguration();
-    }
-  }
 }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/Cdn3StorageManagerConfiguration.java

@@ -5,6 +5,7 @@ import jakarta.validation.constraints.NotNull;
 import java.util.Collections;
 import java.util.Map;
 import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
+import javax.annotation.Nullable;
 
 /**
  * Configuration for the cdn3 storage manager
@@ -16,8 +17,10 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
  *                       storage-manager when copying to determine how to read a source object. Current schemes are
  *                       'gcs' and 'r2'
  * @param numHttpClients The number http clients to use with the storage-manager to support request striping
- * @param circuitBreaker A circuit breaker configuration for the storage-manager http client
+ * @param circuitBreakerConfigurationName The name of a circuit breaker configuration for the storage-manager http
- * @param retry          A retry configuration for the storage-manager http client
+ *                                        client; if `null`, uses the global default configuration
+ * @param retryConfigurationName          The name of a retry configuration for the storage-manager http client; if
+ *                                        `null`, uses the global default configuration
  */
 public record Cdn3StorageManagerConfiguration(
     @NotNull String baseUri,
@@ -25,8 +28,8 @@ public record Cdn3StorageManagerConfiguration(
     @NotNull SecretString clientSecret,
     @NotNull Map<Integer, String> sourceSchemes,
     @NotNull Integer numHttpClients,
-    @NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
+    @Nullable String circuitBreakerConfigurationName,
-    @NotNull @Valid RetryConfiguration retry) {
+    @Nullable String retryConfigurationName) {
 
   public Cdn3StorageManagerConfiguration {
     if (numHttpClients == null) {
@@ -35,11 +38,5 @@ public record Cdn3StorageManagerConfiguration(
     if (sourceSchemes == null) {
       sourceSchemes = Collections.emptyMap();
     }
-    if (circuitBreaker == null) {
-      circuitBreaker = new CircuitBreakerConfiguration();
-    }
-    if (retry == null) {
-      retry = new RetryConfiguration();
-    }
   }
 }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/ClientCdnConfiguration.java

@@ -1,29 +0,0 @@
-package org.whispersystems.textsecuregcm.configuration;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import jakarta.validation.Valid;
-import jakarta.validation.constraints.NotNull;
-
-/**
- * Configuration used to interact with a cdn via HTTP
- */
-public class ClientCdnConfiguration {
-
-  @JsonProperty
-  @NotNull
-  @Valid
-  CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
-
-  @JsonProperty
-  @NotNull
-  @Valid
-  RetryConfiguration retry = new RetryConfiguration();
-
-  public CircuitBreakerConfiguration getCircuitBreaker() {
-    return circuitBreaker;
-  }
-
-  public RetryConfiguration getRetry() {
-    return retry;
-  }
-}

service/src/main/java/org/whispersystems/textsecuregcm/configuration/CloudflareTurnConfiguration.java

@@ -15,6 +15,7 @@ import java.time.Duration;
 import java.util.List;
 import jakarta.validation.constraints.Positive;
 import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
+import javax.annotation.Nullable;
 
 /**
  * Configuration properties for Cloudflare TURN integration.
@@ -27,8 +28,10 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
  * @param urlsWithIps a collection of {@link String#format(String, Object...)} patterns to be populated with resolved IP
  *                    addresses for {@link #hostname} in responses to clients; each pattern must include a single
  *                    {@code %s} placeholder for the IP address
- * @param circuitBreaker a circuit breaker for requests to Cloudflare
+ * @param circuitBreakerConfigurationName the name of a circuit breaker configuration for requests to Cloudflare; if
- * @param retry a retry policy for requests to Cloudflare
+ *                                        `null`, uses the global default configuration
+ * @param retryConfigurationName the name of a retry policy for requests to Cloudflare; if `null`, uses the global
+ *                               default configuration
  * @param hostname the hostname to resolve to IP addresses for use with {@link #urlsWithIps}; also transmitted to
  *                 clients for use as an SNI when connecting to pre-resolved hosts
  * @param numHttpClients the number of parallel HTTP clients to use to communicate with Cloudflare
@@ -39,24 +42,11 @@ public record CloudflareTurnConfiguration(@NotNull SecretString apiToken,
                                           @NotNull Duration clientCredentialTtl,
                                           @NotNull @NotEmpty @Valid List<@NotBlank String> urls,
                                           @NotNull @NotEmpty @Valid List<@NotBlank String> urlsWithIps,
-                                          @NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
+                                          @Nullable String circuitBreakerConfigurationName,
-                                          @NotNull @Valid RetryConfiguration retry,
+                                          @Nullable String retryConfigurationName,
                                           @NotBlank String hostname,
                                           @Positive int numHttpClients) {
 
-  public CloudflareTurnConfiguration {
-    if (circuitBreaker == null) {
-      // It’s a little counter-intuitive, but this compact constructor allows a default value
-      // to be used when one isn’t specified (e.g. in YAML), allowing the field to still be
-      // validated as @NotNull
-      circuitBreaker = new CircuitBreakerConfiguration();
-    }
-
-    if (retry == null) {
-      retry = new RetryConfiguration();
-    }
-  }
-
   @AssertTrue
   @Schema(hidden = true)
   public boolean isClientTtlShorterThanRequestedTtl() {

service/src/main/java/org/whispersystems/textsecuregcm/configuration/RedisClusterConfiguration.java

@@ -9,10 +9,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.annotations.VisibleForTesting;
 import io.lettuce.core.resource.ClientResources;
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
 import java.time.Duration;
+import javax.annotation.Nullable;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 
 @JsonTypeName("default")
@@ -27,9 +27,8 @@ public class RedisClusterConfiguration implements FaultTolerantRedisClusterFacto
   private Duration timeout = Duration.ofSeconds(1);
 
   @JsonProperty
-  @NotNull
+  @Nullable
-  @Valid
+  private String circuitBreakerConfigurationName;
-  private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
 
   @VisibleForTesting
   void setConfigurationUri(final String configurationUri) {
@@ -44,8 +43,8 @@ public class RedisClusterConfiguration implements FaultTolerantRedisClusterFacto
     return timeout;
   }
 
-  public CircuitBreakerConfiguration getCircuitBreakerConfiguration() {
+  @Nullable public String getCircuitBreakerConfigurationName() {
-    return circuitBreaker;
+    return circuitBreakerConfigurationName;
   }
 
   @Override

service/src/main/java/org/whispersystems/textsecuregcm/configuration/RedisConfiguration.java

@@ -9,10 +9,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.annotations.VisibleForTesting;
 import io.lettuce.core.resource.ClientResources;
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
 import java.time.Duration;
+import javax.annotation.Nullable;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 
 @JsonTypeName("default")
@@ -27,9 +27,8 @@ public class RedisConfiguration implements FaultTolerantRedisClientFactory {
   private Duration timeout = Duration.ofSeconds(1);
 
   @JsonProperty
-  @NotNull
+  @Nullable
-  @Valid
+  private String circuitBreakerConfigurationName;
-  private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
 
   public String getUri() {
     return uri;
@@ -44,8 +43,8 @@ public class RedisConfiguration implements FaultTolerantRedisClientFactory {
     return timeout;
   }
 
-  public @NotNull @Valid CircuitBreakerConfiguration getCircuitBreakerConfiguration() {
+  @Nullable public String getCircuitBreakerConfigurationName() {
-    return circuitBreaker;
+    return circuitBreakerConfigurationName;
   }
 
   @Override

service/src/main/java/org/whispersystems/textsecuregcm/configuration/SecureStorageServiceConfiguration.java

@@ -5,24 +5,16 @@
 
 package org.whispersystems.textsecuregcm.configuration;
 
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.NotEmpty;
 import jakarta.validation.constraints.NotNull;
 import java.util.List;
+import javax.annotation.Nullable;
 import org.whispersystems.textsecuregcm.configuration.secrets.SecretBytes;
 
 public record SecureStorageServiceConfiguration(@NotNull SecretBytes userAuthenticationTokenSharedSecret,
                                                 @NotBlank String uri,
                                                 @NotEmpty List<@NotBlank String> storageCaCertificates,
-                                                @Valid CircuitBreakerConfiguration circuitBreaker,
+                                                @Nullable String circuitBreakerConfigurationName,
-                                                @Valid RetryConfiguration retry) {
+                                                @Nullable String retryConfigurationName) {
-  public SecureStorageServiceConfiguration {
-    if (circuitBreaker == null) {
-      circuitBreaker = new CircuitBreakerConfiguration();
-    }
-    if (retry == null) {
-      retry = new RetryConfiguration();
-    }
-  }
 }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/SecureValueRecoveryConfiguration.java

@@ -11,22 +11,13 @@ import jakarta.validation.constraints.NotNull;
 import java.util.List;
 import org.whispersystems.textsecuregcm.configuration.secrets.SecretBytes;
 import org.whispersystems.textsecuregcm.util.ExactlySize;
+import javax.annotation.Nullable;
 
 public record SecureValueRecoveryConfiguration(
     @NotBlank String uri,
     @ExactlySize(32) SecretBytes userAuthenticationTokenSharedSecret,
     @ExactlySize(32) SecretBytes userIdTokenSharedSecret,
     @NotEmpty List<@NotBlank String> svrCaCertificates,
-    @NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
+    @Nullable String circuitBreakerConfigurationName,
-    @NotNull @Valid RetryConfiguration retry) {
+    @Nullable String retryConfigurationName) {
-
-  public SecureValueRecoveryConfiguration {
-    if (circuitBreaker == null) {
-      circuitBreaker = new CircuitBreakerConfiguration();
-    }
-
-    if (retry == null) {
-      retry = new RetryConfiguration();
-    }
-  }
 }

service/src/main/java/org/whispersystems/textsecuregcm/http/FaultTolerantHttpClient.java

@@ -9,6 +9,7 @@ import com.google.common.annotations.VisibleForTesting;
 import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 import io.github.resilience4j.retry.Retry;
 import io.github.resilience4j.retry.RetryConfig;
+import io.micrometer.core.instrument.Tags;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
@@ -16,6 +17,8 @@ import java.security.KeyStore;
 import java.security.cert.CertificateException;
 import java.time.Duration;
 import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
@@ -24,10 +27,8 @@ import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.IntStream;
-import io.micrometer.core.instrument.Tags;
+import javax.annotation.Nullable;
 import org.glassfish.jersey.SslConfigurator;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 import org.whispersystems.textsecuregcm.util.CertificateUtil;
 import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
 import org.whispersystems.textsecuregcm.util.ExceptionUtils;
@@ -36,43 +37,31 @@ public class FaultTolerantHttpClient {
 
   private final List<HttpClient> httpClients;
   private final Duration defaultRequestTimeout;
-  private final ScheduledExecutorService retryExecutor;
+  @Nullable private final ScheduledExecutorService retryExecutor;
-  private final Retry retry;
+  @Nullable private final Retry retry;
   private final CircuitBreaker breaker;
 
   public static final String SECURITY_PROTOCOL_TLS_1_2 = "TLSv1.2";
   public static final String SECURITY_PROTOCOL_TLS_1_3 = "TLSv1.3";
 
-  public static Builder newBuilder() {
+  public static Builder newBuilder(final String name, final Executor executor) {
-    return new Builder();
+    return new Builder(name, executor);
   }
 
   @VisibleForTesting
-  FaultTolerantHttpClient(String name, List<HttpClient> httpClients, ScheduledExecutorService retryExecutor,
+  FaultTolerantHttpClient(final List<HttpClient> httpClients,
-      Duration defaultRequestTimeout, RetryConfiguration retryConfiguration,
+      final Duration defaultRequestTimeout,
-      final Predicate<Throwable> retryOnException, CircuitBreakerConfiguration circuitBreakerConfiguration) {
+      @Nullable final ScheduledExecutorService retryExecutor,
+      @Nullable final Retry retry,
+      final CircuitBreaker circuitBreaker) {
 
     this.httpClients = httpClients;
-    this.retryExecutor = retryExecutor;
     this.defaultRequestTimeout = defaultRequestTimeout;
-    this.breaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
+    this.retryExecutor = retryExecutor;
+    this.retry = retry;
+    this.breaker = circuitBreaker;
 
     CircuitBreakerUtil.registerMetrics(breaker, FaultTolerantHttpClient.class, Tags.empty());
-
-    if (retryConfiguration != null) {
-      if (this.retryExecutor == null) {
-        throw new IllegalArgumentException("retryExecutor must be specified with retryConfiguration");
-      }
-      final RetryConfig.Builder<HttpResponse> retryConfig = retryConfiguration.<HttpResponse>toRetryConfigBuilder()
-          .retryOnResult(o -> o.statusCode() >= 500);
-      if (retryOnException != null) {
-        retryConfig.retryOnException(retryOnException);
-      }
-      this.retry = Retry.of(name + "-retry", retryConfig.build());
-      CircuitBreakerUtil.registerMetrics(retry, FaultTolerantHttpClient.class);
-    } else {
-      this.retry = null;
-    }
   }
 
   private HttpClient httpClient() {
@@ -80,28 +69,30 @@ public class FaultTolerantHttpClient {
   }
 
   public <T> CompletableFuture<HttpResponse<T>> sendAsync(HttpRequest request,
-      HttpResponse.BodyHandler<T> bodyHandler) {
+      final HttpResponse.BodyHandler<T> bodyHandler) {
+
     if (request.timeout().isEmpty()) {
-      request = HttpRequest.newBuilder(request, (n, v) -> true)
+      request = HttpRequest.newBuilder(request, (_, _) -> true)
           .timeout(defaultRequestTimeout)
           .build();
     }
 
-    Supplier<CompletionStage<HttpResponse<T>>> asyncRequest = sendAsync(httpClient(), request, bodyHandler);
+    final Supplier<CompletionStage<HttpResponse<T>>> asyncRequestSupplier =
+        sendAsync(httpClient(), request, bodyHandler);
 
     if (retry != null) {
-      return breaker.executeCompletionStage(retryableCompletionStage(asyncRequest)).toCompletableFuture();
+      assert retryExecutor != null;
-    } else {
-      return breaker.executeCompletionStage(asyncRequest).toCompletableFuture();
-    }
-  }
 
-  private <T> Supplier<CompletionStage<T>> retryableCompletionStage(Supplier<CompletionStage<T>> supplier) {
+      return breaker.executeCompletionStage(retry.decorateCompletionStage(retryExecutor, asyncRequestSupplier))
-    return () -> retry.executeCompletionStage(retryExecutor, supplier);
+          .toCompletableFuture();
+    } else {
+      return breaker.executeCompletionStage(asyncRequestSupplier).toCompletableFuture();
+    }
   }
 
   private <T> Supplier<CompletionStage<HttpResponse<T>>> sendAsync(HttpClient client, HttpRequest request,
       HttpResponse.BodyHandler<T> bodyHandler) {
+
     return () -> client.sendAsync(request, bodyHandler);
   }
 
@@ -113,21 +104,18 @@ public class FaultTolerantHttpClient {
     private Duration requestTimeout = Duration.ofSeconds(60);
     private int numClients = 1;
 
-    private String name;
+    private final String name;
     private Executor executor;
-    private ScheduledExecutorService retryExecutor;
     private KeyStore trustStore;
     private String securityProtocol = SECURITY_PROTOCOL_TLS_1_2;
-    private RetryConfiguration retryConfiguration;
+    private String retryConfigurationName;
+    private ScheduledExecutorService retryExecutor;
     private Predicate<Throwable> retryOnException;
-    private CircuitBreakerConfiguration circuitBreakerConfiguration;
+    @Nullable private String circuitBreakerConfigurationName;
 
-    private Builder() {
+    private Builder(final String name, final Executor executor) {
-    }
+      this.name = Objects.requireNonNull(name);
-
+      this.executor = Objects.requireNonNull(executor);
-    public Builder withName(String name) {
-      this.name = name;
-      return this;
     }
 
     public Builder withVersion(HttpClient.Version version) {
@@ -140,11 +128,6 @@ public class FaultTolerantHttpClient {
       return this;
     }
 
-    public Builder withExecutor(Executor executor) {
-      this.executor = executor;
-      return this;
-    }
-
     public Builder withConnectTimeout(Duration connectTimeout) {
       this.connectTimeout = connectTimeout;
       return this;
@@ -155,18 +138,15 @@ public class FaultTolerantHttpClient {
       return this;
     }
 
-    public Builder withRetry(RetryConfiguration retryConfiguration) {
+    public Builder withRetry(@Nullable final String retryConfigurationName, final ScheduledExecutorService retryExecutor) {
-      this.retryConfiguration = retryConfiguration;
+      this.retryConfigurationName = retryConfigurationName;
-      return this;
-    }
-
-    public Builder withRetryExecutor(ScheduledExecutorService retryExecutor) {
       this.retryExecutor = retryExecutor;
+
       return this;
     }
 
-    public Builder withCircuitBreaker(CircuitBreakerConfiguration circuitBreakerConfiguration) {
+    public Builder withCircuitBreaker(@Nullable final String circuitBreakerConfigurationName) {
-      this.circuitBreakerConfiguration = circuitBreakerConfiguration;
+      this.circuitBreakerConfigurationName = circuitBreakerConfigurationName;
       return this;
     }
 
@@ -206,10 +186,6 @@ public class FaultTolerantHttpClient {
     }
 
     public FaultTolerantHttpClient build() {
-      if (this.circuitBreakerConfiguration == null || this.name == null || this.executor == null) {
-        throw new IllegalArgumentException("Must specify circuit breaker config, name, and executor");
-      }
-
       if (numClients > 1 && version != HttpClient.Version.HTTP_2) {
         throw new IllegalArgumentException("Should not use additional HTTP clients unless using HTTP/2");
       }
@@ -232,8 +208,32 @@ public class FaultTolerantHttpClient {
             return builder.build();
           }).toList();
 
-      return new FaultTolerantHttpClient(name, httpClients, retryExecutor, requestTimeout, retryConfiguration,
+      @Nullable final Retry retry;
-          retryOnException, circuitBreakerConfiguration);
+
+      if (retryExecutor != null) {
+        final RetryConfig.Builder<HttpResponse<?>> retryConfigBuilder =
+            RetryConfig.from(Optional.ofNullable(retryConfigurationName)
+                .flatMap(name -> CircuitBreakerUtil.getRetryRegistry().getConfiguration(name))
+                .orElseGet(() -> CircuitBreakerUtil.getRetryRegistry().getDefaultConfig()));
+
+        retryConfigBuilder.retryOnResult(response -> response.statusCode() >= 500);
+
+        if (retryOnException != null) {
+          retryConfigBuilder.retryOnException(retryOnException);
+        }
+
+        retry = CircuitBreakerUtil.getRetryRegistry().retry(name + "-retry", retryConfigBuilder.build());
+      } else {
+        retry = null;
+      }
+
+      final String circuitBreakerName = name + "-breaker";
+
+      final CircuitBreaker circuitBreaker = circuitBreakerConfigurationName != null
+          ? CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName, circuitBreakerConfigurationName)
+          : CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName);
+
+      return new FaultTolerantHttpClient(httpClients, requestTimeout, retryExecutor, retry, circuitBreaker);
     }
   }
 

service/src/main/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClient.java

@@ -1,5 +1,6 @@
 package org.whispersystems.textsecuregcm.redis;
 
+import com.google.common.annotations.VisibleForTesting;
 import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 import io.lettuce.core.ClientOptions;
 import io.lettuce.core.RedisClient;
@@ -15,8 +16,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
 import org.whispersystems.textsecuregcm.configuration.RedisConfiguration;
+import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
 
 public class FaultTolerantRedisClient {
 
@@ -38,14 +39,17 @@ public class FaultTolerantRedisClient {
     this(name, clientResourcesBuilder,
         RedisUriUtil.createRedisUriWithTimeout(redisConfiguration.getUri(), redisConfiguration.getTimeout()),
         redisConfiguration.getTimeout(),
-        redisConfiguration.getCircuitBreakerConfiguration());
+        redisConfiguration.getCircuitBreakerConfigurationName() != null
+            ? CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(name + "-breaker", redisConfiguration.getCircuitBreakerConfigurationName())
+            : CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(name + "-breaker"));
   }
 
+  @VisibleForTesting
   FaultTolerantRedisClient(String name,
                            final ClientResources.Builder clientResourcesBuilder,
                            final RedisURI redisUri,
                            final Duration commandTimeout,
-                           final CircuitBreakerConfiguration circuitBreakerConfiguration) {
+                           final CircuitBreaker circuitBreaker) {
 
     this.name = name;
 
@@ -73,7 +77,7 @@ public class FaultTolerantRedisClient {
     this.stringConnection = redisClient.connect();
     this.binaryConnection = redisClient.connect(ByteArrayCodec.INSTANCE);
 
-    this.circuitBreaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
+    this.circuitBreaker = circuitBreaker;
   }
 
   public void shutdown() {

service/src/main/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClusterClient.java

@@ -26,7 +26,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import javax.annotation.Nullable;
 import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
 import reactor.core.scheduler.Schedulers;
 
@@ -57,7 +57,7 @@ public class FaultTolerantRedisClusterClient {
         Collections.singleton(RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
             clusterConfiguration.getTimeout())),
         clusterConfiguration.getTimeout(),
-        clusterConfiguration.getCircuitBreakerConfiguration());
+        clusterConfiguration.getCircuitBreakerConfigurationName());
 
   }
 
@@ -65,7 +65,7 @@ public class FaultTolerantRedisClusterClient {
       final ClientResources.Builder clientResourcesBuilder,
       final Iterable<RedisURI> redisUris,
       final Duration commandTimeout,
-      final CircuitBreakerConfiguration circuitBreakerConfig) {
+      @Nullable final String circuitBreakerConfigurationName) {
 
     this.name = name;
 
@@ -83,7 +83,7 @@ public class FaultTolerantRedisClusterClient {
     });
 
     final LettuceShardCircuitBreaker lettuceShardCircuitBreaker = new LettuceShardCircuitBreaker(name,
-        circuitBreakerConfig.toCircuitBreakerConfig(), Schedulers.newSingle("topology-changed-" + name, true));
+        circuitBreakerConfigurationName, Schedulers.newSingle("topology-changed-" + name, true));
     this.clusterClient = RedisClusterClient.create(
         clientResourcesBuilder.nettyCustomizer(lettuceShardCircuitBreaker).
             build(),

service/src/main/java/org/whispersystems/textsecuregcm/redis/LettuceShardCircuitBreaker.java

@@ -8,7 +8,6 @@ package org.whispersystems.textsecuregcm.redis;
 import com.google.common.annotations.VisibleForTesting;
 import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
 import io.github.resilience4j.circuitbreaker.CircuitBreaker;
-import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
 import io.lettuce.core.RedisNoScriptException;
 import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
 import io.lettuce.core.cluster.models.partitions.RedisClusterNode;
@@ -31,6 +30,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
+import javax.annotation.Nullable;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,7 +54,8 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
   private static final Logger logger = LoggerFactory.getLogger(LettuceShardCircuitBreaker.class);
 
   private final String clusterName;
-  private final CircuitBreakerConfig circuitBreakerConfig;
+  @Nullable
+  private final String circuitBreakerConfigurationName;
   private final Scheduler scheduler;
   // this set will be shared with all child channel breakers
   private final Set<String> upstreamAddresses = ConcurrentHashMap.newKeySet();
@@ -62,10 +63,12 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
   // resources, which cannot be built without this NettyCustomizer
   private EventBus eventBus;
 
-  public LettuceShardCircuitBreaker(final String clusterName, final CircuitBreakerConfig circuitBreakerConfig,
+  public LettuceShardCircuitBreaker(final String clusterName,
+      @Nullable final String circuitBreakerConfigurationName,
       final Scheduler scheduler) {
+
     this.clusterName = clusterName;
-    this.circuitBreakerConfig = circuitBreakerConfig;
+    this.circuitBreakerConfigurationName = circuitBreakerConfigurationName;
     this.scheduler = scheduler;
   }
 
@@ -110,7 +113,7 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
     }
 
     final ChannelCircuitBreakerHandler channelCircuitBreakerHandler = new ChannelCircuitBreakerHandler(clusterName,
-        circuitBreakerConfig, upstreamAddresses, eventBus, scheduler);
+        circuitBreakerConfigurationName, upstreamAddresses, eventBus, scheduler);
 
     final String commandHandlerName = StreamSupport.stream(channel.pipeline().spliterator(), false)
         .filter(entry -> entry.getValue() instanceof CommandHandler)
@@ -127,7 +130,7 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
     private static final String CLUSTER_TAG_NAME = "cluster";
 
     private final String clusterName;
-    private final CircuitBreakerConfig circuitBreakerConfig;
+    @Nullable private final String circuitBreakerConfigurationName;
     private final AtomicBoolean registeredMetrics = new AtomicBoolean(false);
     private final Set<String> upstreamAddresses;
 
@@ -136,11 +139,12 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
     @VisibleForTesting
     CircuitBreaker breaker;
 
-    public ChannelCircuitBreakerHandler(final String name, final CircuitBreakerConfig circuitBreakerConfig,
+    public ChannelCircuitBreakerHandler(final String name,
+        @Nullable final String circuitBreakerConfigurationName,
         final Set<String> upstreamAddresses,
         final EventBus eventBus, final Scheduler scheduler) {
       this.clusterName = name;
-      this.circuitBreakerConfig = circuitBreakerConfig;
+      this.circuitBreakerConfigurationName = circuitBreakerConfigurationName;
       this.upstreamAddresses = upstreamAddresses;
 
       eventBus.get()
@@ -183,7 +187,12 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
 
       // In some cases, like the default connection, the remote address includes the DNS hostname, which we want to exclude.
       shardAddress = StringUtils.substringAfter(remoteAddress.toString(), "/");
-      breaker = CircuitBreaker.of("%s/%s-breaker".formatted(clusterName, shardAddress), circuitBreakerConfig);
+
+      final String circuitBreakerName = "%s/%s-breaker".formatted(clusterName, shardAddress);
+
+      breaker = circuitBreakerConfigurationName != null
+          ? CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName, circuitBreakerConfigurationName)
+          : CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName);
 
       if (upstreamAddresses.contains(shardAddress)) {
         registerMetrics();

service/src/main/java/org/whispersystems/textsecuregcm/securestorage/SecureStorageClient.java

@@ -43,15 +43,12 @@ public class SecureStorageClient {
       throws CertificateException {
     this.storageServiceCredentialsGenerator = storageServiceCredentialsGenerator;
     this.deleteUri = URI.create(configuration.uri()).resolve(DELETE_PATH);
-    this.httpClient = FaultTolerantHttpClient.newBuilder()
+    this.httpClient = FaultTolerantHttpClient.newBuilder("secure-storage", executor)
-        .withCircuitBreaker(configuration.circuitBreaker())
+        .withCircuitBreaker(configuration.circuitBreakerConfigurationName())
-        .withRetry(configuration.retry())
+        .withRetry(configuration.retryConfigurationName(), retryExecutor)
-        .withRetryExecutor(retryExecutor)
         .withVersion(HttpClient.Version.HTTP_1_1)
         .withConnectTimeout(Duration.ofSeconds(10))
         .withRedirect(HttpClient.Redirect.NEVER)
-        .withExecutor(executor)
-        .withName("secure-storage")
         .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_3)
         .withTrustedServerCertificates(configuration.storageCaCertificates().toArray(new String[0]))
         .build();

service/src/main/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecoveryClient.java

@@ -53,15 +53,12 @@ public class SecureValueRecoveryClient {
     this.secureValueRecoveryCredentialsGenerator = secureValueRecoveryCredentialsGenerator;
     this.deleteUri = URI.create(configuration.uri()).resolve(DELETE_PATH);
     this.allowedDeletionErrorStatusCodes = allowedDeletionErrorStatusCodes;
-    this.httpClient = FaultTolerantHttpClient.newBuilder()
+    this.httpClient = FaultTolerantHttpClient.newBuilder("secure-value-recovery", executor)
-        .withCircuitBreaker(configuration.circuitBreaker())
+        .withCircuitBreaker(configuration.circuitBreakerConfigurationName())
-        .withRetry(configuration.retry())
+        .withRetry(configuration.retryConfigurationName(), retryExecutor)
-        .withRetryExecutor(retryExecutor)
         .withVersion(HttpClient.Version.HTTP_1_1)
         .withConnectTimeout(Duration.ofSeconds(10))
         .withRedirect(HttpClient.Redirect.NEVER)
-        .withExecutor(executor)
-        .withName("secure-value-recovery")
         .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_2)
         .withTrustedServerCertificates(configuration.svrCaCertificates().toArray(new String[0]))
         .build();

service/src/main/java/org/whispersystems/textsecuregcm/subscriptions/AppleAppStoreManager.java

@@ -19,17 +19,20 @@ import com.apple.itunes.storekit.verification.SignedDataVerifier;
 import com.apple.itunes.storekit.verification.VerificationException;
 import com.google.common.annotations.VisibleForTesting;
 import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
 import io.micrometer.core.instrument.Metrics;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigDecimal;
+import java.net.http.HttpResponse;
 import java.time.Instant;
 import java.util.Base64;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
@@ -37,13 +40,14 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
 import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
 import org.whispersystems.textsecuregcm.storage.PaymentTime;
 import org.whispersystems.textsecuregcm.storage.SubscriptionException;
+import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
 import org.whispersystems.textsecuregcm.util.ExceptionUtils;
 
 /**
@@ -82,12 +86,12 @@ public class AppleAppStoreManager implements SubscriptionPaymentProcessor {
       final String subscriptionGroupId,
       final Map<String, Long> productIdToLevel,
       final List<String> base64AppleRootCerts,
-      final RetryConfiguration retryConfiguration,
+      @Nullable final String retryConfigurationName,
       final ExecutorService executor,
       final ScheduledExecutorService retryExecutor) {
     this(new AppStoreServerAPIClient(encodedKey, keyId, issuerId, bundleId, env),
         new SignedDataVerifier(decodeRootCerts(base64AppleRootCerts), bundleId, appAppleId, env, true),
-        subscriptionGroupId, productIdToLevel, retryConfiguration, executor, retryExecutor);
+        subscriptionGroupId, productIdToLevel, retryConfigurationName, executor, retryExecutor);
   }
 
   @VisibleForTesting
@@ -96,18 +100,24 @@ public class AppleAppStoreManager implements SubscriptionPaymentProcessor {
       final SignedDataVerifier signedDataVerifier,
       final String subscriptionGroupId,
       final Map<String, Long> productIdToLevel,
-      final RetryConfiguration retryConfiguration,
+      @Nullable final String retryConfigurationName,
       final ExecutorService executor,
       final ScheduledExecutorService retryExecutor) {
     this.apiClient = apiClient;
     this.signedDataVerifier = signedDataVerifier;
     this.subscriptionGroupId = subscriptionGroupId;
     this.productIdToLevel = productIdToLevel;
-    this.retry = Retry.of("appstore-retry", retryConfiguration
-        .toRetryConfigBuilder()
-        .retryOnException(AppleAppStoreManager::shouldRetry).build());
     this.executor = Objects.requireNonNull(executor);
     this.retryExecutor = Objects.requireNonNull(retryExecutor);
+
+    final RetryConfig.Builder<HttpResponse<?>> retryConfigBuilder =
+        RetryConfig.from(Optional.ofNullable(retryConfigurationName)
+            .flatMap(name -> CircuitBreakerUtil.getRetryRegistry().getConfiguration(name))
+            .orElseGet(() -> CircuitBreakerUtil.getRetryRegistry().getDefaultConfig()));
+
+    retryConfigBuilder.retryOnException(AppleAppStoreManager::shouldRetry);
+
+    this.retry = CircuitBreakerUtil.getRetryRegistry().retry("appstore-retry", retryConfigBuilder.build());
   }
 
   @Override

service/src/main/java/org/whispersystems/textsecuregcm/subscriptions/BraintreeManager.java

@@ -42,7 +42,6 @@ import java.util.concurrent.ScheduledExecutorService;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
 import org.whispersystems.textsecuregcm.currency.CurrencyConversionManager;
 import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
 import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
@@ -81,7 +80,7 @@ public class BraintreeManager implements CustomerAwareSubscriptionPaymentProcess
       final String graphqlUri,
       final CurrencyConversionManager currencyConversionManager,
       final PublisherInterface pubsubPublisher,
-      final CircuitBreakerConfiguration circuitBreakerConfiguration,
+      @Nullable final String circuitBreakerConfigurationName,
       final Executor executor,
       final ScheduledExecutorService retryExecutor) {
 
@@ -89,11 +88,8 @@ public class BraintreeManager implements CustomerAwareSubscriptionPaymentProcess
             braintreePrivateKey),
         supportedCurrenciesByPaymentMethod,
         currenciesToMerchantAccounts,
-        new BraintreeGraphqlClient(FaultTolerantHttpClient.newBuilder()
+        new BraintreeGraphqlClient(FaultTolerantHttpClient.newBuilder("braintree-graphql", executor)
-            .withName("braintree-graphql")
+            .withCircuitBreaker(circuitBreakerConfigurationName)
-            .withCircuitBreaker(circuitBreakerConfiguration)
-            .withExecutor(executor)
-            .withRetryExecutor(retryExecutor)
             // Braintree documents its internal timeout at 60 seconds, and we want to make sure we don’t miss
             // a response
             // https://developer.paypal.com/braintree/docs/reference/general/best-practices/java#timeouts

service/src/main/java/org/whispersystems/textsecuregcm/util/CircuitBreakerUtil.java

@@ -8,11 +8,15 @@ package org.whispersystems.textsecuregcm.util;
 import static com.codahale.metrics.MetricRegistry.name;
 
 import io.github.resilience4j.circuitbreaker.CircuitBreaker;
+import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
 import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryRegistry;
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Metrics;
 import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Tags;
+import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 
 public class CircuitBreakerUtil {
 
@@ -23,6 +27,22 @@ public class CircuitBreakerUtil {
   private static final String BREAKER_NAME_TAG_NAME = "breakerName";
   private static final String OUTCOME_TAG_NAME = "outcome";
 
+  private static final CircuitBreakerRegistry CIRCUIT_BREAKER_REGISTRY =
+      CircuitBreakerRegistry.of(new CircuitBreakerConfiguration().toCircuitBreakerConfig());
+
+  private static final RetryRegistry RETRY_REGISTRY =
+      RetryRegistry.of(new RetryConfiguration().toRetryConfigBuilder().build());
+
+  public static CircuitBreakerRegistry getCircuitBreakerRegistry() {
+    return CIRCUIT_BREAKER_REGISTRY;
+  }
+
+  public static RetryRegistry getRetryRegistry() {
+    return RETRY_REGISTRY;
+  }
+
+  /// @deprecated [CircuitBreakerRegistry] maintains its own set of metrics, and manually managing them is unnecessary
+  @Deprecated(forRemoval = true)
   public static void registerMetrics(CircuitBreaker circuitBreaker, Class<?> clazz, Tags additionalTags) {
     final String breakerName = clazz.getSimpleName() + "/" + circuitBreaker.getName();
 
@@ -57,6 +77,8 @@ public class CircuitBreakerUtil {
         circuitBreaker, breaker -> breaker.getState().getOrder());
   }
 
+  /// @deprecated [RetryRegistry] maintains its own set of metrics, and manually managing them is unnecessary
+  @Deprecated(forRemoval = true)
   public static void registerMetrics(Retry retry, Class<?> clazz) {
     final String retryName = clazz.getSimpleName() + "/" + retry.getName();
 

service/src/test/java/org/whispersystems/textsecuregcm/auth/CloudflareTurnCredentialsManagerTest.java

@@ -35,8 +35,6 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 
 public class CloudflareTurnCredentialsManagerTest {
   @RegisterExtension
@@ -77,9 +75,9 @@ public class CloudflareTurnCredentialsManagerTest {
         IP_URL_PATTERNS,
         TURN_HOSTNAME,
         2,
-        new CircuitBreakerConfiguration(),
+        null,
         httpExecutor,
-        new RetryConfiguration(),
+        null,
         retryExecutor,
         dnsResolver
     );

service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java

@@ -27,8 +27,6 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
 import org.whispersystems.textsecuregcm.util.CompletableFutureTestUtil;
 import org.whispersystems.textsecuregcm.util.SystemMapper;
@@ -58,8 +56,8 @@ public class Cdn3RemoteStorageManagerTest {
             new SecretString("clientSecret"),
             Map.of(2, "gcs", 3, "r2"),
             2,
-            new CircuitBreakerConfiguration(),
+            null,
-            new RetryConfiguration()));
+            null));
   }
 
   @ParameterizedTest

service/src/test/java/org/whispersystems/textsecuregcm/http/FaultTolerantHttpClientTest.java

@@ -11,6 +11,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -19,6 +20,8 @@ import static org.mockito.Mockito.when;
 
 import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
 import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
+import io.github.resilience4j.circuitbreaker.CircuitBreaker;
+import io.github.resilience4j.retry.Retry;
 import java.io.IOException;
 import java.net.URI;
 import java.net.http.HttpClient;
@@ -39,6 +42,7 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
 import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
+import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
 
 class FaultTolerantHttpClientTest {
 
@@ -71,12 +75,8 @@ class FaultTolerantHttpClientTest {
             .withHeader("Content-Type", "text/plain")
             .withBody("Pong!")));
 
-    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder()
+    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder("testSimpleGet", httpExecutor)
-        .withCircuitBreaker(new CircuitBreakerConfiguration())
+        .withRetry(null, retryExecutor)
-        .withRetry(new RetryConfiguration())
-        .withExecutor(httpExecutor)
-        .withRetryExecutor(retryExecutor)
-        .withName("test")
         .withVersion(HttpClient.Version.HTTP_2)
         .build();
 
@@ -101,12 +101,8 @@ class FaultTolerantHttpClientTest {
                                              .withHeader("Content-Type", "text/plain")
                                              .withBody("Pong!")));
 
-    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder()
+    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder("testRetryGet", httpExecutor)
-        .withCircuitBreaker(new CircuitBreakerConfiguration())
+        .withRetry(null, retryExecutor)
-        .withRetry(new RetryConfiguration())
-        .withExecutor(httpExecutor)
-        .withRetryExecutor(retryExecutor)
-        .withName("test")
         .withVersion(HttpClient.Version.HTTP_2)
         .build();
 
@@ -126,14 +122,17 @@ class FaultTolerantHttpClientTest {
   @Test
   void testRetryGetOnException() {
     final HttpClient mockHttpClient = mock(HttpClient.class);
+
+    final Retry retry = Retry.of("test", new RetryConfiguration().toRetryConfigBuilder()
+        .retryOnException(throwable -> throwable instanceof IOException)
+        .build());
+
     final FaultTolerantHttpClient client = new FaultTolerantHttpClient(
-        "test",
         List.of(mockHttpClient),
-        retryExecutor,
         Duration.ofSeconds(1),
-        new RetryConfiguration(),
+        retryExecutor,
-        throwable -> throwable instanceof IOException,
+        retry,
-        new CircuitBreakerConfiguration());
+        CircuitBreaker.ofDefaults("test"));
 
     when(mockHttpClient.sendAsync(any(), any()))
         .thenReturn(CompletableFuture.failedFuture(new IOException("test exception")));
@@ -156,14 +155,17 @@ class FaultTolerantHttpClientTest {
   void testMultipleClients() throws IOException, InterruptedException {
     final HttpClient mockHttpClient1 = mock(HttpClient.class);
     final HttpClient mockHttpClient2 = mock(HttpClient.class);
+
+    final Retry retry = Retry.of("test", new RetryConfiguration().toRetryConfigBuilder()
+        .retryOnException(throwable -> throwable instanceof IOException)
+        .build());
+
     final FaultTolerantHttpClient client = new FaultTolerantHttpClient(
-        "test",
         List.of(mockHttpClient1, mockHttpClient2),
-        retryExecutor,
         Duration.ofSeconds(1),
-        new RetryConfiguration(),
+        retryExecutor,
-        throwable -> throwable instanceof IOException,
+        retry,
-        new CircuitBreakerConfiguration());
+        CircuitBreaker.ofDefaults("test"));
 
     // Just to get a dummy HttpResponse
     wireMock.stubFor(get(urlEqualTo("/ping"))
@@ -201,68 +203,50 @@ class FaultTolerantHttpClientTest {
 
   @Test
   void testNetworkFailureCircuitBreaker() throws InterruptedException {
-    CircuitBreakerConfiguration circuitBreakerConfiguration = new CircuitBreakerConfiguration();
+    final CircuitBreakerConfiguration circuitBreakerConfiguration = new CircuitBreakerConfiguration();
     circuitBreakerConfiguration.setSlidingWindowSize(2);
     circuitBreakerConfiguration.setSlidingWindowMinimumNumberOfCalls(2);
     circuitBreakerConfiguration.setPermittedNumberOfCallsInHalfOpenState(1);
     circuitBreakerConfiguration.setFailureRateThreshold(50);
     circuitBreakerConfiguration.setWaitDurationInOpenState(Duration.ofSeconds(1));
 
-    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder()
+    final String circuitBreakerConfigurationName = getClass().getSimpleName() + "#testNetworkFailureCircuitBreaker";
-        .withCircuitBreaker(circuitBreakerConfiguration)
+
-        .withRetry(new RetryConfiguration())
+    CircuitBreakerUtil.getCircuitBreakerRegistry()
-        .withRetryExecutor(retryExecutor)
+        .addConfiguration(circuitBreakerConfigurationName, circuitBreakerConfiguration.toCircuitBreakerConfig());
-        .withExecutor(httpExecutor)
+
-        .withName("test")
+    final FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder("testNetworkFailureCircuitBreaker", httpExecutor)
+        .withCircuitBreaker(circuitBreakerConfigurationName)
+        .withRetry(null, retryExecutor)
         .withVersion(HttpClient.Version.HTTP_2)
         .build();
 
-    HttpRequest request = HttpRequest.newBuilder()
+    final HttpRequest request = HttpRequest.newBuilder()
-                                     .uri(URI.create("http://localhost:" + 39873 + "/failure"))
+        .uri(URI.create("http://localhost:" + 39873 + "/failure"))
-                                     .GET()
+        .GET()
-                                     .build();
+        .build();
 
-    try {
+    assertThatThrownBy(() -> client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join())
-      client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join();
+        .isInstanceOf(CompletionException.class)
-      throw new AssertionError("Should have failed!");
+        .hasCauseInstanceOf(IOException.class);
-    } catch (CompletionException e) {
-      assertThat(e.getCause()).isInstanceOf(IOException.class);
-      // good
-    }
 
-    try {
+    assertThatThrownBy(() -> client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join())
-      client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join();
+        .isInstanceOf(CompletionException.class)
-      throw new AssertionError("Should have failed!");
+        .hasCauseInstanceOf(IOException.class);
-    } catch (CompletionException e) {
-      assertThat(e.getCause()).isInstanceOf(IOException.class);
-      // good
-    }
 
-    try {
+    assertThatThrownBy(() -> client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join())
-      client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join();
+        .isInstanceOf(CompletionException.class)
-      throw new AssertionError("Should have failed!");
+        .hasCauseInstanceOf(CallNotPermittedException.class);
-    } catch (CompletionException e) {
-      assertThat(e.getCause()).isInstanceOf(CallNotPermittedException.class);
-      // good
-    }
 
     Thread.sleep(1001);
 
-    try {
+    assertThatThrownBy(() -> client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join())
-      client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join();
+        .isInstanceOf(CompletionException.class)
-      throw new AssertionError("Should have failed!");
+        .hasCauseInstanceOf(IOException.class);
-    } catch (CompletionException e) {
-      assertThat(e.getCause()).isInstanceOf(IOException.class);
-      // good
-    }
 
-    try {
+    assertThatThrownBy(() -> client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join())
-      client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).join();
+        .isInstanceOf(CompletionException.class)
-      throw new AssertionError("Should have failed!");
+        .hasCauseInstanceOf(CallNotPermittedException.class);
-    } catch (CompletionException e) {
-      assertThat(e.getCause()).isInstanceOf(CallNotPermittedException.class);
-      // good
-    }
   }
 
 }

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClientTest.java

@@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
+import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 import io.lettuce.core.RedisCommandTimeoutException;
 import io.lettuce.core.RedisException;
 import io.lettuce.core.resource.ClientResources;
@@ -34,9 +35,14 @@ class FaultTolerantRedisClientTest {
       @Nullable final CircuitBreakerConfiguration circuitBreakerConfiguration,
       final ClientResources.Builder clientResourcesBuilder) {
 
-    return new FaultTolerantRedisClient("test", clientResourcesBuilder,
+    final CircuitBreaker circuitBreaker = CircuitBreaker.of("test", Optional.ofNullable(circuitBreakerConfiguration)
-        RedisServerExtension.getRedisURI(), TIMEOUT,
+            .orElseGet(CircuitBreakerConfiguration::new).toCircuitBreakerConfig());
-        Optional.ofNullable(circuitBreakerConfiguration).orElseGet(CircuitBreakerConfiguration::new));
+
+    return new FaultTolerantRedisClient("test",
+        clientResourcesBuilder,
+        RedisServerExtension.getRedisURI(),
+        TIMEOUT,
+        circuitBreaker);
   }
 
   @AfterEach

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClusterClientTest.java

@@ -46,7 +46,6 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
@@ -61,6 +60,7 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
 import org.whispersystems.textsecuregcm.util.Pair;
 import org.whispersystems.textsecuregcm.util.RedisClusterUtil;
 
@@ -71,6 +71,8 @@ class FaultTolerantRedisClusterClientTest {
 
   private static final Duration TIMEOUT = Duration.ofMillis(200);
 
+  private static int circuitBreakerConfigurationCount = 0;
+
   @RegisterExtension
   static final RedisClusterExtension REDIS_CLUSTER_EXTENSION = RedisClusterExtension.builder()
       .timeout(TIMEOUT)
@@ -79,13 +81,24 @@ class FaultTolerantRedisClusterClientTest {
   private FaultTolerantRedisClusterClient cluster;
 
   private static FaultTolerantRedisClusterClient buildCluster(
+      final String name,
       @Nullable final CircuitBreakerConfiguration circuitBreakerConfiguration,
       final ClientResources.Builder clientResourcesBuilder) {
 
-    return new FaultTolerantRedisClusterClient("test",
+    final String circuitBreakerConfigurationName;
+
+    if (circuitBreakerConfiguration != null) {
+      circuitBreakerConfigurationName = FaultTolerantRedisClusterClientTest.class.getSimpleName() + "-" + circuitBreakerConfigurationCount++;
+      CircuitBreakerUtil.getCircuitBreakerRegistry().addConfiguration(circuitBreakerConfigurationName, circuitBreakerConfiguration.toCircuitBreakerConfig());
+    } else {
+      circuitBreakerConfigurationName = null;
+    }
+
+    return new FaultTolerantRedisClusterClient(name,
         clientResourcesBuilder.socketAddressResolver(REDIS_CLUSTER_EXTENSION.getSocketAddressResolver()),
-        RedisClusterExtension.getRedisURIs(), TIMEOUT,
+        RedisClusterExtension.getRedisURIs(),
-        Optional.ofNullable(circuitBreakerConfiguration).orElseGet(CircuitBreakerConfiguration::new));
+        TIMEOUT,
+        circuitBreakerConfigurationName);
   }
 
   @AfterEach
@@ -95,7 +108,7 @@ class FaultTolerantRedisClusterClientTest {
 
   @Test
   void testTimeout() {
-    cluster = buildCluster(null, ClientResources.builder());
+    cluster = buildCluster("testTimeout", null, ClientResources.builder());
 
     final ExecutionException asyncException = assertThrows(ExecutionException.class,
         () -> cluster.withCluster(connection -> connection.async().blpop(10 * TIMEOUT.toMillis() / 1000d, "key"))
@@ -119,7 +132,7 @@ class FaultTolerantRedisClusterClientTest {
     circuitBreakerConfig.setSlidingWindowSize(1);
     circuitBreakerConfig.setWaitDurationInOpenState(breakerWaitDuration);
 
-    cluster = buildCluster(circuitBreakerConfig, ClientResources.builder());
+    cluster = buildCluster("testTimeoutCircuitBreaker", circuitBreakerConfig, ClientResources.builder());
 
     final String key = "key";
 
@@ -149,7 +162,7 @@ class FaultTolerantRedisClusterClientTest {
     final ClientResources.Builder builder = CompositeNettyCustomizerClientResourcesBuilder.builder()
         .nettyCustomizer(testBreakerManager);
 
-    cluster = buildCluster(circuitBreakerConfig, builder);
+    cluster = buildCluster("testShardUnavailable", circuitBreakerConfig, builder);
 
     // this test will open the breaker on one shard and check that other shards are still available,
     // so we get two nodes and a slot+key on each to test
@@ -202,7 +215,7 @@ class FaultTolerantRedisClusterClientTest {
     final ClientResources.Builder builder = CompositeNettyCustomizerClientResourcesBuilder.builder()
         .nettyCustomizer(testBreakerManager);
 
-    cluster = buildCluster(circuitBreakerConfig, builder);
+    cluster = buildCluster("testShardUnavailablePubSub", circuitBreakerConfig, builder);
 
     cluster.useCluster(
         connection -> connection.sync().upstream().commands().configSet("notify-keyspace-events", "K$glz"));

service/src/test/java/org/whispersystems/textsecuregcm/redis/LettuceShardCircuitBreakerTest.java

@@ -33,19 +33,15 @@ import io.netty.channel.ChannelPromise;
 import io.netty.channel.embedded.EmbeddedChannel;
 import java.io.IOException;
 import java.net.SocketAddress;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.StreamSupport;
-import javax.annotation.Nullable;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
 import reactor.core.publisher.Flux;
 import reactor.core.scheduler.Schedulers;
 
@@ -59,15 +55,15 @@ class LettuceShardCircuitBreakerTest {
     eventBus = mock(EventBus.class);
     when(eventBus.get()).thenReturn(Flux.never());
     channelCircuitBreakerHandler = new LettuceShardCircuitBreaker.ChannelCircuitBreakerHandler(
-        "test", new CircuitBreakerConfiguration().toCircuitBreakerConfig(), Collections.emptySet(), eventBus,
+        "test", null, Collections.emptySet(), eventBus, Schedulers.immediate());
-        Schedulers.immediate());
   }
 
   @Test
   void testAfterChannelInitialized() {
 
-    final LettuceShardCircuitBreaker lettuceShardCircuitBreaker = new LettuceShardCircuitBreaker("test",
+    final LettuceShardCircuitBreaker lettuceShardCircuitBreaker =
-        new CircuitBreakerConfiguration().toCircuitBreakerConfig(), Schedulers.immediate());
+        new LettuceShardCircuitBreaker("test", null, Schedulers.immediate());
+
     lettuceShardCircuitBreaker.setEventBus(eventBus);
 
     final Channel channel = new EmbeddedChannel(

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisClusterExtension.java

@@ -22,6 +22,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.jupiter.api.extension.AfterEachCallback;
 import org.junit.jupiter.api.extension.BeforeAllCallback;
 import org.junit.jupiter.api.extension.BeforeEachCallback;
@@ -30,6 +31,7 @@ import org.testcontainers.containers.ComposeContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.containers.wait.strategy.WaitStrategy;
 import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
 import org.whispersystems.textsecuregcm.util.TestcontainersImages;
 
 public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallback, AfterEachCallback, ExtensionContext.Store.CloseableResource {
@@ -43,6 +45,9 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
   private ClientResources redisClientResources;
   private FaultTolerantRedisClusterClient redisClusterClient;
 
+  private static final String CIRCUIT_BREAKER_CONFIGURATION_NAME =
+      RedisClusterExtension.class.getSimpleName() + "-" + RandomStringUtils.insecure().nextAlphanumeric(8);
+
   private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(2);
 
   private static final int REDIS_PORT = 6379;
@@ -97,6 +102,11 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
   @Override
   public void beforeAll(final ExtensionContext context) throws Exception {
     if (composeContainer == null) {
+      final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
+      circuitBreakerConfig.setWaitDurationInOpenState(Duration.ofMillis(500));
+
+      CircuitBreakerUtil.getCircuitBreakerRegistry().addConfiguration(CIRCUIT_BREAKER_CONFIGURATION_NAME, circuitBreakerConfig.toCircuitBreakerConfig());
+
       final File clusterComposeFile = File.createTempFile("redis-cluster", ".yml");
       clusterComposeFile.deleteOnExit();
 
@@ -181,14 +191,11 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
         .socketAddressResolver(getSocketAddressResolver())
         .build();
 
-    final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
-    circuitBreakerConfig.setWaitDurationInOpenState(Duration.ofMillis(500));
-
     redisClusterClient = new FaultTolerantRedisClusterClient("test-cluster",
         redisClientResources.mutate(),
         getRedisURIs(),
         timeout,
-        circuitBreakerConfig);
+        CIRCUIT_BREAKER_CONFIGURATION_NAME);
 
     redisClusterClient.useCluster(connection -> connection.sync().flushall(FlushMode.SYNC));
   }

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisServerExtension.java

@@ -6,6 +6,7 @@
 package org.whispersystems.textsecuregcm.redis;
 
 import com.redis.testcontainers.RedisContainer;
+import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 import io.lettuce.core.FlushMode;
 import io.lettuce.core.RedisURI;
 import io.lettuce.core.resource.ClientResources;
@@ -64,7 +65,7 @@ public class RedisServerExtension implements BeforeAllCallback, BeforeEachCallba
         redisClientResources.mutate(),
         getRedisURI(),
         Duration.ofSeconds(2),
-        circuitBreakerConfig);
+        CircuitBreaker.of("test", circuitBreakerConfig.toCircuitBreakerConfig()));
 
     faultTolerantRedisClient.useConnection(connection -> connection.sync().flushall(FlushMode.SYNC));
   }

service/src/test/java/org/whispersystems/textsecuregcm/securestorage/SecureStorageClientTest.java

@@ -32,7 +32,6 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
 import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 import org.whispersystems.textsecuregcm.configuration.SecureStorageServiceConfiguration;
 
 class SecureStorageClientTest {
@@ -104,8 +103,8 @@ class SecureStorageClientTest {
             lY6ZKNA81Lm3YADYtObmK1IUrOPo9BeIaPy0UM08SmN880Vunqa91Q==
             -----END CERTIFICATE-----
             """),
-        new CircuitBreakerConfiguration(),
+        null,
-        new RetryConfiguration());
+        null);
 
     secureStorageClient = new SecureStorageClient(credentialsGenerator, httpExecutor, retryExecutor, config);
   }

service/src/test/java/org/whispersystems/textsecuregcm/subscriptions/AppleAppStoreManagerTest.java

@@ -39,7 +39,6 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.EnumSource;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
 import org.whispersystems.textsecuregcm.storage.SubscriptionException;
 import org.whispersystems.textsecuregcm.util.CompletableFutureTestUtil;
 
@@ -63,7 +62,7 @@ class AppleAppStoreManagerTest {
     reset(apiClient, signedDataVerifier);
     executor = Executors.newSingleThreadScheduledExecutor();
     appleAppStoreManager = new AppleAppStoreManager(apiClient, signedDataVerifier,
-        SUBSCRIPTION_GROUP_ID, Map.of(PRODUCT_ID, LEVEL), new RetryConfiguration(), executor, executor);
+        SUBSCRIPTION_GROUP_ID, Map.of(PRODUCT_ID, LEVEL), null, executor, executor);
   }
 
   @AfterEach