remove datadog/statsd support

pom.xml

@@ -42,7 +42,6 @@
     <commons-csv.version>1.14.1</commons-csv.version>
     <commons-io.version>2.20.0</commons-io.version>
     <dropwizard.version>5.0.0</dropwizard.version>
-    <dropwizard-metrics-datadog.version>1.1.14</dropwizard-metrics-datadog.version>
     <!-- Note: when updating FoundationDB, also include a copy of `libfdb_c.so` from the FoundationDB release at
     src/main/jib/usr/lib/libfdb_c.so. We use x86_64 builds without AVX instructions enabled (i.e. FoundationDB versions
     with even-numbered patch versions). Also when updating FoundationDB, make sure to update the version of FoundationDB
@@ -231,26 +230,6 @@
         <artifactId>commons-csv</artifactId>
         <version>${commons-csv.version}</version>
       </dependency>
-      <dependency>
-        <groupId>org.coursera</groupId>
-        <artifactId>dropwizard-metrics-datadog</artifactId>
-        <version>${dropwizard-metrics-datadog.version}</version>
-        <exclusions>
-          <!-- resolve convergence error: jetty 12 ee10 uses 9.x -->
-          <exclusion>
-            <groupId>org.ow2.asm</groupId>
-            <artifactId>asm-commons</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.ow2.asm</groupId>
-            <artifactId>asm-tree</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.ow2.asm</groupId>
-            <artifactId>asm</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
       <dependency>
         <groupId>org.foundationdb</groupId>
         <artifactId>fdb-java</artifactId>

service/config/sample-secrets-bundle.yml

@@ -1,5 +1,3 @@
-datadog.apiKey: unset
-
 stripe.apiKey: unset
 stripe.idempotencyKeyGenerator: abcdefg12345678= # base64 for creating request idempotency hash
 

service/config/sample.yml

@@ -10,35 +10,7 @@ logging:
       threshold: ALL
       timeZone: UTC
       target: stdout
-    - type: logstashtcpsocket
-      destination: example.com:10516
-      apiKey: secret://datadog.apiKey
-      environment: staging
-
-metrics:
-  reporters:
-    - type: signal-datadog
-      frequency: 10 seconds
-      tags:
-        - "env:staging"
-        - "service:chat"
-      udpTransport:
-        statsdHost: localhost
-        port: 8125
-      excludesAttributes:
-        - m1_rate
-        - m5_rate
-        - m15_rate
-        - mean_rate
-        - stddev
-      useRegexFilters: true
-      excludes:
-        - ^.+\.total$
-        - ^.+\.request\.filtering$
-        - ^.+\.response\.filtering$
-        - ^executor\..+$
-        - ^lettuce\..+$
-  reportOnStop: true
+    - type: otlp
 
 tlsKeyStore:
   password: secret://tlsKeyStore.password
@@ -294,10 +266,6 @@ cdn3StorageManager:
     2: gcs
     3: r2
 
-dogstatsd:
-  environment: dev
-  host: 127.0.0.1
-
 openTelemetry:
   enabled: true
   environment: dev

service/pom.xml

@@ -346,18 +346,10 @@
       <groupId>io.micrometer</groupId>
       <artifactId>micrometer-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.micrometer</groupId>
-      <artifactId>micrometer-registry-statsd</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.micrometer</groupId>
       <artifactId>micrometer-registry-otlp</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.coursera</groupId>
-      <artifactId>dropwizard-metrics-datadog</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java

@@ -26,11 +26,9 @@ import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfigur
 import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
 import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
 import org.whispersystems.textsecuregcm.configuration.ClientReleaseConfiguration;
-import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DefaultAwsCredentialsFactory;
 import org.whispersystems.textsecuregcm.configuration.DeviceCheckConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DirectoryV2Configuration;
-import org.whispersystems.textsecuregcm.configuration.DogstatsdConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DynamoDbClientFactory;
 import org.whispersystems.textsecuregcm.configuration.DynamoDbTables;
 import org.whispersystems.textsecuregcm.configuration.ExternalRequestFilterConfiguration;
@@ -137,11 +135,6 @@ public class WhisperServerConfiguration extends Configuration {
   @JsonProperty
   private Cdn3StorageManagerConfiguration cdn3StorageManager;
 
-  @NotNull
-  @Valid
-  @JsonProperty
-  private DatadogConfiguration dogstatsd = new DogstatsdConfiguration();
-
   @NotNull
   @Valid
   @JsonProperty
@@ -463,10 +456,6 @@ public class WhisperServerConfiguration extends Configuration {
     return cdn3StorageManager;
   }
 
-  public DatadogConfiguration getDatadogConfiguration() {
-    return dogstatsd;
-  }
-
   public OpenTelemetryConfiguration getOpenTelemetryConfiguration() {
     return openTelemetry;
   }

service/src/main/java/org/whispersystems/textsecuregcm/configuration/DatadogConfiguration.java

@@ -1,21 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.configuration;
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import io.dropwizard.jackson.Discoverable;
-import io.micrometer.statsd.StatsdConfig;
-import java.time.Duration;
-
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = DogstatsdConfiguration.class)
-public interface DatadogConfiguration extends StatsdConfig, Discoverable {
-
-  boolean enabled();
-
-  String getEnvironment();
-
-  Duration getShutdownWaitDuration();
-}

service/src/main/java/org/whispersystems/textsecuregcm/configuration/DogstatsdConfiguration.java

@@ -1,68 +0,0 @@
-/*
- * Copyright 2013 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.configuration;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import io.micrometer.statsd.StatsdFlavor;
-import jakarta.validation.constraints.NotBlank;
-import jakarta.validation.constraints.NotNull;
-import java.time.Duration;
-
-@JsonTypeName("default")
-public class DogstatsdConfiguration implements DatadogConfiguration {
-
-  @JsonProperty
-  private boolean enabled = true;
-
-  @JsonProperty
-  @NotNull
-  private Duration step = Duration.ofSeconds(10);
-
-  @JsonProperty
-  @NotBlank
-  private String environment;
-
-  @JsonProperty
-  @NotBlank
-  private String host;
-
-  @Override
-  public boolean enabled() {
-    return enabled;
-  }
-
-  @Override
-  public Duration step() {
-    return step;
-  }
-
-  @Override
-  public String getEnvironment() {
-    return environment;
-  }
-
-  @Override
-  public StatsdFlavor flavor() {
-    return StatsdFlavor.DATADOG;
-  }
-
-  @Override
-  public String get(final String key) {
-    // We have no Micrometer key/value pairs to report, so always return `null`
-    return null;
-  }
-
-  @Override
-  public String host() {
-    return host;
-  }
-
-  @Override
-  public Duration getShutdownWaitDuration() {
-    return step().plus(step.dividedBy(2));
-  }
-}

service/src/main/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicConfiguration.java

@@ -50,7 +50,7 @@ public class DynamicConfiguration {
 
   @JsonProperty
   @Valid
-  DynamicMetricsConfiguration metricsConfiguration = new DynamicMetricsConfiguration(false, false, null);
+  DynamicMetricsConfiguration metricsConfiguration = new DynamicMetricsConfiguration(false, false);
 
   @JsonProperty
   @Valid

service/src/main/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicMetricsConfiguration.java

@@ -5,15 +5,11 @@
 
 package org.whispersystems.textsecuregcm.configuration.dynamic;
 
-import java.util.Set;
-import javax.annotation.Nullable;
-
 /**
  * @param enableLettuceRemoteTag whether the `remote` tag should be added. Note: although this is dynamic, meters are
  *                               cached after creation, so changes will only affect servers launched after the change.
  * @param enableAwsSdkMetrics whether to record AWS SDK metrics. Note: although this is dynamic, meters are cached after
  *                            creation, so changes will only affect servers launched after the change.
- * @param datadogAllowList if present, a list of metrics to send to datadog; others will be filtered out. If null, this filtering is not performed.
  */
-public record DynamicMetricsConfiguration(boolean enableLettuceRemoteTag, boolean enableAwsSdkMetrics, @Nullable Set<String> datadogAllowList) {
+public record DynamicMetricsConfiguration(boolean enableLettuceRemoteTag, boolean enableAwsSdkMetrics) {
 }

service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java

@@ -12,7 +12,6 @@ import io.dropwizard.lifecycle.Managed;
 import io.micrometer.core.instrument.Meter;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Metrics;
-import io.micrometer.core.instrument.Tags;
 import io.micrometer.core.instrument.binder.jetty.JettySslHandshakeMetrics;
 import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
 import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
@@ -21,7 +20,6 @@ import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
 import io.micrometer.core.instrument.config.MeterFilter;
 import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
 import io.micrometer.registry.otlp.OtlpMeterRegistry;
-import io.micrometer.statsd.StatsdMeterRegistry;
 import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter;
 import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender;
 import io.opentelemetry.sdk.OpenTelemetrySdk;
@@ -29,13 +27,10 @@ import io.opentelemetry.sdk.logs.SdkLoggerProvider;
 import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor;
 import io.opentelemetry.sdk.resources.Resource;
 import io.opentelemetry.sdk.resources.ResourceBuilder;
-import io.opentelemetry.semconv.ServiceAttributes;
-
 import java.time.Duration;
 import java.util.Map;
 import java.util.Optional;
 import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
-import org.whispersystems.textsecuregcm.WhisperServerVersion;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
 import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
 import org.whispersystems.textsecuregcm.util.Constants;
@@ -75,23 +70,6 @@ public class MetricsUtil {
 
     Duration shutdownWaitDuration = Duration.ZERO;
 
-    if (config.getDatadogConfiguration().enabled()) {
-      final StatsdMeterRegistry dogstatsdMeterRegistry = new StatsdMeterRegistry(
-          config.getDatadogConfiguration(), io.micrometer.core.instrument.Clock.SYSTEM);
-
-      dogstatsdMeterRegistry.config().commonTags(
-          Tags.of(
-              "service", "chat",
-              "version", WhisperServerVersion.getServerVersion(),
-              "env", config.getDatadogConfiguration().getEnvironment()));
-
-      configureMeterFilters(dogstatsdMeterRegistry.config(), dynamicConfigurationManager);
-      configureDatadogAllowList(dogstatsdMeterRegistry.config(), dynamicConfigurationManager);
-      Metrics.addRegistry(dogstatsdMeterRegistry);
-
-      shutdownWaitDuration = config.getDatadogConfiguration().getShutdownWaitDuration();
-    }
-
     if (config.getOpenTelemetryConfiguration().enabled()) {
       final OtlpMeterRegistry otlpMeterRegistry = new OtlpMeterRegistry(
         config.getOpenTelemetryConfiguration(), io.micrometer.core.instrument.Clock.SYSTEM);
@@ -99,9 +77,7 @@ public class MetricsUtil {
       configureMeterFilters(otlpMeterRegistry.config(), dynamicConfigurationManager);
       Metrics.addRegistry(otlpMeterRegistry);
 
-      if (config.getOpenTelemetryConfiguration().shutdownWaitDuration().compareTo(shutdownWaitDuration) > 0) {
-        shutdownWaitDuration = config.getOpenTelemetryConfiguration().shutdownWaitDuration();
-      }
+      shutdownWaitDuration = config.getOpenTelemetryConfiguration().shutdownWaitDuration();
     }
 
     environment.lifecycle().addServerLifecycleListener(
@@ -189,14 +165,6 @@ public class MetricsUtil {
             && id.getName().startsWith(awsSdkMetricNamePrefix)));
   }
 
-  @VisibleForTesting
-  static void configureDatadogAllowList(
-      final MeterRegistry.Config config, final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
-    config.meterFilter(MeterFilter.denyUnless(id ->
-            dynamicConfigurationManager.getConfiguration().getMetricsConfiguration().datadogAllowList() == null ||
-            dynamicConfigurationManager.getConfiguration().getMetricsConfiguration().datadogAllowList().contains(id.getName())));
-  }
-
   static void registerSystemResourceMetrics() {
     new ProcessorMetrics().bindTo(Metrics.globalRegistry);
     new FileDescriptorMetrics().bindTo(Metrics.globalRegistry);

service/src/main/java/org/whispersystems/textsecuregcm/metrics/SignalDatadogReporterFactory.java

@@ -1,109 +0,0 @@
-/*
- * Copyright 2023 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-/*
- * This is derived from Coursera's dropwizard datadog reporter.
- * https://github.com/coursera/metrics-datadog
- */
-
-package org.whispersystems.textsecuregcm.metrics;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.ScheduledReporter;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import io.dropwizard.metrics.common.BaseReporterFactory;
-import jakarta.validation.Valid;
-import jakarta.validation.constraints.Min;
-import jakarta.validation.constraints.NotNull;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import org.apache.commons.lang3.StringUtils;
-import org.coursera.metrics.datadog.DatadogReporter;
-import org.coursera.metrics.datadog.DatadogReporter.Expansion;
-import org.coursera.metrics.datadog.DefaultMetricNameFormatterFactory;
-import org.coursera.metrics.datadog.DynamicTagsCallbackFactory;
-import org.coursera.metrics.datadog.MetricNameFormatterFactory;
-import org.coursera.metrics.datadog.transport.UdpTransport;
-import org.whispersystems.textsecuregcm.WhisperServerVersion;
-
-@JsonTypeName("signal-datadog")
-public class SignalDatadogReporterFactory extends BaseReporterFactory {
-
-  @JsonProperty
-  private List<String> tags = null;
-
-  @Valid
-  @JsonProperty
-  private DynamicTagsCallbackFactory dynamicTagsCallback = null;
-
-  @JsonProperty
-  private String prefix = null;
-
-  @Valid
-  @NotNull
-  @JsonProperty
-  private MetricNameFormatterFactory metricNameFormatter = new DefaultMetricNameFormatterFactory();
-
-  @Valid
-  @NotNull
-  @JsonProperty("udpTransport")
-  private UdpTransportConfig udpTransportConfig;
-
-  private static final EnumSet<Expansion> EXPANSIONS = EnumSet.of(
-      Expansion.COUNT,
-      Expansion.MIN,
-      Expansion.MAX,
-      Expansion.MEAN,
-      Expansion.MEDIAN,
-      Expansion.P75,
-      Expansion.P95,
-      Expansion.P99,
-      Expansion.P999
-  );
-
-  public ScheduledReporter build(final MetricRegistry registry) {
-    final List<String> combinedTags;
-
-    {
-      final String versionTag = "version:" + WhisperServerVersion.getServerVersion();
-
-      if (tags != null) {
-        combinedTags = new ArrayList<>(tags);
-        combinedTags.add(versionTag);
-      } else {
-        combinedTags = new ArrayList<>((List.of(versionTag)));
-      }
-    }
-
-    final String entityId = StringUtils.stripToNull(System.getenv("DD_ENTITY_ID"));
-    if (entityId != null) {
-      combinedTags.add("dd.internal.entity_id:" + entityId);
-    }
-
-    return DatadogReporter.forRegistry(registry)
-        .withTransport(udpTransportConfig.udpTransport())
-        .withTags(combinedTags)
-        .withPrefix(prefix)
-        .withExpansions(EXPANSIONS)
-        .withMetricNameFormatter(metricNameFormatter.build())
-        .withDynamicTagCallback(dynamicTagsCallback != null ? dynamicTagsCallback.build() : null)
-        .filter(getFilter())
-        .convertDurationsTo(getDurationUnit())
-        .convertRatesTo(getRateUnit())
-        .build();
-  }
-
-  public record UdpTransportConfig(@NotNull String statsdHost, @Min(1) int port) {
-
-    public UdpTransport udpTransport() {
-      return new UdpTransport.Builder()
-          .withStatsdHost(statsdHost)
-          .withPort(port)
-          .build();
-    }
-  }
-}

service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable

@@ -1,5 +1,4 @@
 org.whispersystems.textsecuregcm.configuration.AwsCredentialsProviderFactory
-org.whispersystems.textsecuregcm.configuration.DatadogConfiguration
 org.whispersystems.textsecuregcm.configuration.DynamoDbClientFactory
 org.whispersystems.textsecuregcm.configuration.FaultTolerantRedisClusterFactory
 org.whispersystems.textsecuregcm.configuration.FaultTolerantRedisClientFactory

service/src/main/resources/META-INF/services/io.dropwizard.metrics.common.ReporterFactory

@@ -1 +0,0 @@
-org.whispersystems.textsecuregcm.metrics.SignalDatadogReporterFactory

service/src/test/java/org/whispersystems/textsecuregcm/configuration/NoWaitDogstatsdConfiguration.java

@@ -1,18 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.configuration;
-
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import java.time.Duration;
-
-@JsonTypeName("nowait")
-public class NoWaitDogstatsdConfiguration extends DogstatsdConfiguration {
-
-  @Override
-  public Duration getShutdownWaitDuration() {
-    return Duration.ZERO;
-  }
-}

service/src/test/java/org/whispersystems/textsecuregcm/metrics/MetricsUtilTest.java

@@ -39,7 +39,7 @@ class MetricsUtilTest {
   @ValueSource(booleans = {true, false})
   void lettuceTagRejection(final boolean enableLettuceRemoteTag) {
     final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
-    final DynamicMetricsConfiguration metricsConfiguration = new DynamicMetricsConfiguration(enableLettuceRemoteTag, false, null);
+    final DynamicMetricsConfiguration metricsConfiguration = new DynamicMetricsConfiguration(enableLettuceRemoteTag, false);
     when(dynamicConfiguration.getMetricsConfiguration()).thenReturn(metricsConfiguration);
     @SuppressWarnings("unchecked") final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager =
         mock(DynamicConfigurationManager.class);
@@ -72,7 +72,7 @@ class MetricsUtilTest {
         mock(DynamicConfigurationManager.class);
 
     final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
-    final DynamicMetricsConfiguration metricsConfiguration = new DynamicMetricsConfiguration(false, enableAwsSdkMetrics, null);
+    final DynamicMetricsConfiguration metricsConfiguration = new DynamicMetricsConfiguration(false, enableAwsSdkMetrics);
 
     when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
     when(dynamicConfiguration.getMetricsConfiguration()).thenReturn(metricsConfiguration);
@@ -84,24 +84,4 @@ class MetricsUtilTest {
     assertThat(registry.getMeters()).hasSize(enableAwsSdkMetrics ? 1 : 0);
   }
 
-  @ParameterizedTest
-  @ValueSource(booleans={true, false})
-  void datadogAllowList(boolean useAllowlist) {
-    @SuppressWarnings("unchecked") final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager =
-        mock(DynamicConfigurationManager.class);
-
-    final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
-    final DynamicMetricsConfiguration metricsConfiguration = new DynamicMetricsConfiguration(true, true, useAllowlist ? Set.of("chat.ImportantMetrics.messages") : null);
-
-    when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
-    when(dynamicConfiguration.getMetricsConfiguration()).thenReturn(metricsConfiguration);
-
-    final MeterRegistry registry = new SimpleMeterRegistry();
-    MetricsUtil.configureDatadogAllowList(registry.config(), dynamicConfigurationManager);
-    registry.counter("chat.ImportantMetrics.messages").increment();
-    registry.counter("chat.ChattyMetrics.days_since_last_incident").increment();
-
-    assertThat(registry.getMeters()).hasSize(useAllowlist ? 1 : 2);
-  }
-
 }

service/src/test/resources/META-INF/services/org.whispersystems.textsecuregcm.configuration.DatadogConfiguration

@@ -1 +0,0 @@
-org.whispersystems.textsecuregcm.configuration.NoWaitDogstatsdConfiguration

service/src/test/resources/config/test.yml

@@ -14,31 +14,6 @@ awsCredentialsProvider:
   accessKeyId: secret://aws.accessKeyId
   secretAccessKey: secret://aws.secretAccessKey
 
-metrics:
-  reporters:
-    - type: signal-datadog
-      frequency: 10 seconds
-      tags:
-        - "env:test"
-        - "service:chat"
-      udpTransport:
-        statsdHost: localhost
-        port: 8125
-      excludesAttributes:
-        - m1_rate
-        - m5_rate
-        - m15_rate
-        - mean_rate
-        - stddev
-      useRegexFilters: true
-      excludes:
-        - ^.+\.total$
-        - ^.+\.request\.filtering$
-        - ^.+\.response\.filtering$
-        - ^executor\..+$
-        - ^lettuce\..+$
-  reportOnStop: true
-
 tlsKeyStore:
   password: secret://tlsKeyStore.password
 
@@ -293,11 +268,6 @@ cdn3StorageManager:
     2: gcs
     3: r2
 
-dogstatsd:
-  type: nowait
-  environment: dev
-  host: 127.0.0.1
-
 openTelemetry:
   enabled: false
   environment: dev