Compare commits

...

42 Commits

Author SHA1 Message Date
Chris Eager
b41ed9d810 Update sample.yml config 2021-06-07 17:21:36 -04:00
Jon Chambers
58d3a12eff Set hostname to lowercase to avoid strange case mismatch issues; log hostname failures. 2021-06-07 17:17:46 -04:00
Jon Chambers
88c4b2be97 Correct a misunderstanding about the metrics host tag. 2021-06-07 16:29:44 -04:00
Jon Chambers
6cbd57f19f Include environment/service/version as common metric tags. 2021-06-04 18:17:09 -04:00
Jon Chambers
5522376584 Include a host tag with metrics. 2021-06-04 18:17:09 -04:00
Jon Chambers
5089c37d28 Drop a pair of unused commands. 2021-06-04 12:35:06 -04:00
Jon Chambers
1ccf24e68c Add a command to check dynamic config files. 2021-06-04 12:34:48 -04:00
Jon Chambers
411f7298f2 Enforce validation constraints for dynamic configuration objects. 2021-06-04 12:34:48 -04:00
Jon Chambers
5b0214c6f2 Make pre-key take operations more null-safe 2021-06-04 11:18:59 -04:00
Jon Chambers
735573e61b Make reporting intervals configurable. 2021-06-03 17:50:41 -04:00
Graeme Connell
c545cff1b3 Switch DynamoDB to AWSv2.
Switch from using com.amazonaws.services.dynamodbv2 to using
software.amazon.awssdk.services.dynamodb for all current DynamoDB uses.
2021-06-03 13:37:10 -06:00
Jon Chambers
cbd9681e3e Configure histograms and exclude high-cardinality metrics. 2021-06-03 14:12:02 -04:00
Jon Chambers
ca876e40ca Add a second metric aggregator. 2021-06-03 14:12:02 -04:00
Jon Chambers
76f5a71727 Include server version in logging tags 2021-06-03 11:24:25 -04:00
Jon Chambers
117de2382d Verify that API consumers can skip/clear VOIP tokens. 2021-06-02 16:50:49 -05:00
Jon Chambers
25e7036451 Send a payload with mutable content for non-VOIP topics. 2021-06-02 16:50:49 -05:00
Jon Chambers
3131bd3dd9 Allow iOS callers to specify whether they're providing a VOIP token for preauth. 2021-06-02 16:50:49 -05:00
Chris Eager
1cf9397bbd Bump dropwizard to 2.0.22 2021-06-02 12:30:30 -05:00
brock-signal
c97be15e79 Fix NPE when a null message comes in from a client 2021-06-01 15:00:41 -06:00
Ehren Kret
164fc40990 Rename receipt type and add new client-to-client plaintext type for decryption error receipts 2021-05-28 11:33:44 -05:00
Ehren Kret
6456af6284 Upgrade to latest protobuf
This upgrades to protobuf 3.17 and uses maven to automatically rebuild
the generated code instead of using prefabricated checked in Java
files.
2021-05-28 11:33:44 -05:00
Chris Eager
81212cc13a Add jgitver configuration to ignore branch names 2021-05-27 14:35:28 -05:00
Ehren Kret
6f0750790c Add metric to count number of legacy messages sent 2021-05-27 11:13:42 -05:00
Chris Eager
3e61b5c49d Add call chain and mismatch check for push token timestamp 2021-05-27 11:10:58 -05:00
Ehren Kret
50c4df4f45 Add deploy phase bindings 2021-05-26 19:42:45 -05:00
Ehren Kret
1eb946f5fe Add jgitver extension 2021-05-26 19:42:45 -05:00
Ehren Kret
7bd402b48d Build refactor in preparations for bringing in jgitver 2021-05-26 19:42:42 -05:00
Chris Eager
90444d5b91 Bump version to 5.95 2021-05-26 11:11:00 -05:00
Chris Eager
5ee093f87c Add mismatch for signed pre-key; remove mismatch for migration version 2021-05-26 10:58:23 -05:00
Chris Eager
623743286c Bump version to 5.94 2021-05-25 11:00:44 -05:00
Chris Eager
67067f1d2d Remove last-seen and registration lock comparisons 2021-05-25 10:47:57 -05:00
Ehren Kret
07f9bb112e Use separate object for multi recipient response
`needsSync` was being sent back from the server in the JSON response
which is an unnecessary and constantly false field in multi-recipient
message sending endpoint as it's always sealed sender.
2021-05-25 10:30:39 -05:00
Ehren Kret
417d48c452 Block downgrading sender key support
Disallow linking an additional device to an account that has already
upgraded to having sender key support where the linked device does not
have sender key support. This should prompt the person attempting to
link the older application to upgrade in order to complete the linking
process.
2021-05-25 10:30:26 -05:00
Chris Eager
358412c78a Bump version to 5.93 2021-05-24 12:15:46 -05:00
Chris Eager
215621a9b0 Remove temporary adaptation for nested IncomingMessage.online 2021-05-24 11:36:15 -05:00
Graeme Connell
c3f53c4dd9 Fix infinite loop in TorExitNodeManager. 2021-05-21 14:50:15 -06:00
Graeme Connell
01514f83a0 Fix up AWS2 config issues introduced in rebase. 2021-05-21 14:50:15 -06:00
Graeme Connell
c10b64c367 Simplify S3ObjectMonitor API, try-with-resource. 2021-05-21 14:50:15 -06:00
Graeme Connell
722055c8b5 Switch S3ObjectMonitor to AWSv2 SDK. 2021-05-21 14:50:15 -06:00
Graeme Connell
680e501f83 Add dependency on AWS 2.x s3. 2021-05-21 14:50:15 -06:00
Ehren Kret
f13f7a5ff4 Bump version to 5.92 2021-05-20 15:13:14 -05:00
Ehren Kret
5290656c3b Fix typo 2021-05-20 15:11:44 -05:00
83 changed files with 2259 additions and 6841 deletions

11
.gitignore vendored
View File

@@ -9,10 +9,13 @@ config/production.yml
config/federated.yml
config/staging.yml
config/testing.yml
service/config/production.yml
service/config/federated.yml
service/config/staging.yml
service/config/testing.yml
config/deploy.properties
/service/config/production.yml
/service/config/federated.yml
/service/config/staging.yml
/service/config/testing.yml
/service/config/deploy.properties
/service/dependency-reduced-pom.xml
.opsmanage
put.sh
deployer-staging.properties

9
.mvn/extensions.xml Normal file
View File

@@ -0,0 +1,9 @@
<extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
<extension>
<groupId>fr.brouillard.oss</groupId>
<artifactId>jgitver-maven-plugin</artifactId>
<version>1.7.1</version>
</extension>
</extensions>

14
.mvn/jgitver.config.xml Normal file
View File

@@ -0,0 +1,14 @@
<configuration xmlns="http://jgitver.github.io/maven/configuration/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://jgitver.github.io/maven/configuration/1.1.0 https://jgitver.github.io/maven/configuration/jgitver-configuration-v1_1_0.xsd">
<useDirty>true</useDirty>
<useDefaultBranchingPolicy>false</useDefaultBranchingPolicy>
<branchPolicies>
<branchPolicy>
<pattern>(.*)</pattern>
<transformations>
<transformation>IGNORE</transformation>
</transformations>
</branchPolicy>
</branchPolicies>
</configuration>

View File

@@ -5,18 +5,15 @@
<parent>
<artifactId>TextSecureServer</artifactId>
<groupId>org.whispersystems.textsecure</groupId>
<version>1.0</version>
<version>JGITVER</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>gcm-sender-async</artifactId>
<version>${TextSecureServer.version}</version>
<dependencies>
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-retry</artifactId>
<version>${resilience4j.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -44,6 +41,11 @@
<artifactId>jcl-over-slf4j</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-nop</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

220
pom.xml
View File

@@ -4,9 +4,6 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<prerequisites>
<maven>3.0.0</maven>
</prerequisites>
<repositories>
<repository>
@@ -32,20 +29,36 @@
</modules>
<properties>
<dropwizard.version>2.0.21</dropwizard.version>
<aws.sdk.version>1.11.939</aws.sdk.version>
<mockito.version>2.25.1</mockito.version>
<aws.sdk2.version>2.16.66</aws.sdk2.version>
<commons-codec.version>1.15</commons-codec.version>
<commons-csv.version>1.8</commons-csv.version>
<commons-io.version>2.9.0</commons-io.version>
<dropwizard.version>2.0.22</dropwizard.version>
<guava.version>30.1.1-jre</guava.version>
<jaxb.version>2.3.1</jaxb.version>
<jedis.version>2.9.0</jedis.version>
<lettuce.version>6.0.4.RELEASE</lettuce.version>
<libphonenumber.version>8.12.23</libphonenumber.version>
<logstash.logback.version>6.6</logstash.logback.version>
<micrometer.version>1.5.3</micrometer.version>
<mockito.version>2.25.1</mockito.version>
<netty.version>4.1.65.Final</netty.version>
<netty.tcnative-boringssl-static.version>2.0.39.Final</netty.tcnative-boringssl-static.version>
<opentest4j.version>1.2.0</opentest4j.version>
<postgresql.version>9.4-1201-jdbc41</postgresql.version>
<protobuf.version>3.17.1</protobuf.version>
<pushy.version>0.14.2</pushy.version>
<resilience4j.version>1.5.0</resilience4j.version>
<semver4j.version>3.1.0</semver4j.version>
<slf4j.version>1.7.30</slf4j.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<TextSecureServer.version>5.91</TextSecureServer.version>
</properties>
<groupId>org.whispersystems.textsecure</groupId>
<artifactId>TextSecureServer</artifactId>
<version>1.0</version>
<version>JGITVER</version>
<dependencyManagement>
<dependencies>
@@ -56,6 +69,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>${netty.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
@@ -63,6 +83,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${aws.sdk2.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-bom</artifactId>
@@ -77,6 +104,119 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.eatthepath</groupId>
<artifactId>pushy</artifactId>
<version>${pushy.version}</version>
</dependency>
<dependency>
<groupId>com.eatthepath</groupId>
<artifactId>pushy-dropwizard-metrics-listener</artifactId>
<version>${pushy.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.googlecode.libphonenumber</groupId>
<artifactId>libphonenumber</artifactId>
<version>${libphonenumber.version}</version>
</dependency>
<dependency>
<groupId>com.vdurmont</groupId>
<artifactId>semver4j</artifactId>
<version>${semver4j.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons-codec.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>io.lettuce</groupId>
<artifactId>lettuce-core</artifactId>
<version>${lettuce.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
<version>${netty.tcnative-boringssl-static.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>${jaxb.version}</version>
</dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>${logstash.logback.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>${commons-csv.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>${jaxb.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.opentest4j</groupId>
<artifactId>opentest4j</artifactId>
<version>${opentest4j.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-nop</artifactId>
<version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>${jedis.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
@@ -123,20 +263,46 @@
</dependencies>
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.7.0</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>0.6.1</version>
<configuration>
<protocArtifact>com.google.protobuf:protoc:3.17.2:exe:${os.detected.classifier}</protocArtifact>
</configuration>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<version>3.8.1</version>
<configuration>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.1.1</version>
<version>3.2.0</version>
<configuration>
<archive>
<manifest>
@@ -183,11 +349,39 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.4.1</version>
<version>3.0.0-M3</version>
<executions>
<execution>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<dependencyConvergence/>
<requireMavenVersion>
<version>3.0.0</version>
</requireMavenVersion>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>3.0.0-M1</version>
<configuration>
<rules>
<dependencyConvergence/>
</rules>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>3.0.0-M1</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>

View File

@@ -5,12 +5,10 @@
<parent>
<artifactId>TextSecureServer</artifactId>
<groupId>org.whispersystems.textsecure</groupId>
<version>1.0</version>
<version>JGITVER</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>redis-dispatch</artifactId>
<version>${TextSecureServer.version}</version>
<dependencies>
<dependency>

View File

@@ -1,6 +1,6 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
<id>bin</id>
<includeBaseDirectory>false</includeBaseDirectory>
<formats>
@@ -18,8 +18,8 @@
<directory>${project.build.directory}</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>${parent.artifactId}-${TextSecureServer.version}.jar</include>
<include>${parent.artifactId}-${project.version}.jar</include>
</includes>
</fileSet>
</fileSets>
</assembly>
</assembly>

View File

@@ -21,9 +21,6 @@ twilio: # Twilio gateway configuration
push:
queueSize: # Size of push pending queue
redphone:
authKey: # Deprecated
turn: # TURN server configuration
secret: # TURN server secret
uris:
@@ -36,6 +33,23 @@ cacheCluster: # Redis server configuration for cache cluster
urls:
- redis://redis.example.com:6379/
clientPresenceCluster: # Redis server configuration for client presence cluster
urls:
- redis://redis.example.com:6379/
pubsub: # Redis server configuration for pubsub cluster
url: redis://redis.example.com:6379/
replicaUrls:
- redis://redis.example.com:6379/
pushSchedulerCluster: # Redis server configuration for push scheduler cluster
urls:
- redis://redis.example.com:6379/
rateLimitersCluster: # Redis server configuration for rate limiters cluster
urls:
- redis://redis.example.com:6379/
directory:
client: # Configuration for interfacing with Contact Discovery Service cluster
userAuthenticationTokenSharedSecret: # hex-encoded secret shared with CDS used to generate auth tokens for Signal users
@@ -43,13 +57,13 @@ directory:
sqs:
accessKey: # AWS SQS accessKey
accessSecret: # AWS SQS accessSecret
queueUrl: # AWS SQS queue url
server:
replicationUrl: # CDS replication endpoint base url
replicationPassword: # CDS replication endpoint password
replicationCaCertificate: # CDS replication endpoint TLS certificate trust root
reconciliationChunkSize: # CDS reconciliation chunk size
reconciliationChunkIntervalMs: # CDS reconciliation chunk interval, in milliseconds
queueUrls: # AWS SQS queue urls
- https://sqs.example.com/directory.fifo
server: # One or more CDS servers
- replicationName: # CDS replication name
replicationUrl: # CDS replication endpoint base url
replicationPassword: # CDS replication endpoint password
replicationCaCertificate: # CDS replication endpoint TLS certificate trust root
messageCache: # Redis server configuration for message store cache
persistDelayMinutes:
@@ -58,16 +72,39 @@ messageCache: # Redis server configuration for message store cache
urls:
- redis://redis.example.com:6379/
messageStore: # Postgresql database configuration for message store
driverClass: org.postgresql.Driver
user:
password:
url:
metricsCluster:
urls:
- redis://redis.example.com:6379/
messageDynamoDb: # DynamoDB table configuration
region:
tableName:
keysDynamoDb: # DynamoDB table configuration
region:
tableName:
accountsDynamoDb: # DynamoDB table configuration
region:
tableName:
phoneNumberTableName:
migrationDeletedAccountsDynamoDb: # DynamoDB table configuration
region:
tableName:
migrationRetryAccountsDynamoDb: # DynamoDB table configuration
region:
tableName:
pushChallengeDynamoDb: # DynamoDB table configuration
region:
tableName:
reportMessageDynamoDb: # DynamoDB table configuration
region:
tableName:
awsAttachments: # AWS S3 configuration
accessKey:
accessSecret:
@@ -81,18 +118,22 @@ gcpAttachments: # GCP Storage configuration
pathPrefix:
rsaSigningKey:
profiles: # AWS S3 configuration
accessKey:
accessSecret:
bucket:
region:
database: # Postgresql database configuration
abuseDatabase: # Postgresql database configuration
driverClass: org.postgresql.Driver
user:
password:
url:
accountsDatabase: # Postgresql database configuration
driverClass: org.postgresql.Driver
user:
password:
url:
accountDatabaseCrawler:
chunkSize: # accounts per run
chunkIntervalMs: # time per run
apn: # Apple Push Notifications configuration
sandbox: true
bundleId:
@@ -104,11 +145,52 @@ gcm: # GCM Configuration
senderId:
apiKey:
micrometer: # Micrometer metrics config
- name: "example"
- uri: "https://metrics.example.com/"
- apiKey:
- accountId:
cdn:
accessKey: # AWS Access Key ID
accessSecret: # AWS Access Secret
bucket: # S3 Bucket name
region: # AWS region
wavefront: # Wavefront micrometer metrics config
uri: # Wavefront proxy endpoint
batchSize: # Number of measurements to send per request
datadog:
apiKey:
environment:
unidentifiedDelivery:
certificate:
privateKey:
expiresDays:
voiceVerification:
url: https://cdn-ca.signal.org/verification/
locales:
- en
recaptcha:
secret:
storageService:
uri:
userAuthenticationTokenSharedSecret:
storageCaCertificate:
backupService:
uri:
userAuthenticationTokenSharedSecret:
backupCaCertificate:
zkConfig:
serverPublic:
serverSecret:
enabled:
appConfig:
application:
environment:
configuration:
remoteConfig:
authorizedTokens:
@@ -118,9 +200,22 @@ remoteConfig:
- # Nth authorized token
globalConfig: # keys and values that are given to clients on GET /v1/config
paymentService:
paymentsService:
userAuthenticationTokenSharedSecret: # hex-encoded 32-byte secret shared with MobileCoin services used to generate auth tokens for Signal users
torExitNodeList:
s3Region:
s3Bucket:
objectKey:
maxSize:
asnTable:
s3Region:
s3Bucket:
objectKey:
maxSize:
donation:
uri: # value
apiKey: # value

View File

@@ -5,16 +5,10 @@
<parent>
<artifactId>TextSecureServer</artifactId>
<groupId>org.whispersystems.textsecure</groupId>
<version>1.0</version>
<version>JGITVER</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>service</artifactId>
<version>${TextSecureServer.version}</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
@@ -33,29 +27,28 @@
<dependency>
<groupId>org.whispersystems.textsecure</groupId>
<artifactId>redis-dispatch</artifactId>
<version>${TextSecureServer.version}</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.whispersystems.textsecure</groupId>
<artifactId>websocket-resources</artifactId>
<version>${TextSecureServer.version}</version>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.whispersystems.textsecure</groupId>
<artifactId>gcm-sender-async</artifactId>
<version>${TextSecureServer.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.signal</groupId>
<artifactId>zkgroup-java</artifactId>
<version>0.7.0</version>
</dependency>
<dependency>
<groupId>org.whispersystems</groupId>
<artifactId>curve25519-java</artifactId>
<version>0.5.0</version>
</dependency>
<dependency>
<groupId>io.dropwizard</groupId>
@@ -134,7 +127,6 @@
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>6.6</version>
</dependency>
<dependency>
@@ -178,8 +170,6 @@
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>2.3.1</version>
<scope>runtime</scope>
</dependency>
<dependency>
@@ -204,17 +194,10 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.13</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.8</version>
</dependency>
<dependency>
<groupId>io.vavr</groupId>
<artifactId>vavr</artifactId>
<version>0.10.2</version>
</dependency>
<dependency>
@@ -225,12 +208,6 @@
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-circuitbreaker</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.github.resilience4j</groupId>
@@ -245,6 +222,10 @@
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-wavefront</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-datadog</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -271,6 +252,14 @@
<artifactId>jackson-jaxrs-json-provider</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>dynamodb</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
@@ -287,84 +276,56 @@
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-appconfig</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.0</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.lettuce</groupId>
<artifactId>lettuce-core</artifactId>
<version>6.0.4.RELEASE</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>9.4-1201-jdbc41</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.whispersystems</groupId>
<artifactId>curve25519-java</artifactId>
<version>0.5.0</version>
</dependency>
<dependency>
<groupId>com.eatthepath</groupId>
<artifactId>pushy</artifactId>
<version>${pushy.version}</version>
</dependency>
<dependency>
<groupId>com.eatthepath</groupId>
<artifactId>pushy-dropwizard-metrics-listener</artifactId>
<version>${pushy.version}</version>
<exclusions>
<exclusion>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
<version>2.0.34.Final</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.vdurmont</groupId>
<artifactId>semver4j</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>30.1.1-jre</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.6.1</version>
</dependency>
<dependency>
<groupId>com.googlecode.libphonenumber</groupId>
<artifactId>libphonenumber</artifactId>
<version>8.12.21</version>
</dependency>
<dependency>
@@ -466,12 +427,12 @@
<build>
<finalName>${parent.artifactId}-${TextSecureServer.version}</finalName>
<finalName>${project.parent.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>1.6</version>
<version>3.2.4</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<filters>
@@ -504,8 +465,9 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<version>3.3.0</version>
<configuration>
<descriptors>
<descriptor>assembly.xml</descriptor>
@@ -521,8 +483,58 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>properties-maven-plugin</artifactId>
<version>1.0.0</version>
<executions>
<execution>
<id>read-deploy-configuration</id>
<phase>deploy</phase>
<goals>
<goal>read-project-properties</goal>
</goals>
<configuration>
<files>${project.basedir}/config/deploy.properties</files>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.bazaarvoice.maven.plugins</groupId>
<artifactId>s3-upload-maven-plugin</artifactId>
<version>1.5</version>
<configuration>
<source>${project.build.directory}/${project.build.finalName}-bin.tar.gz</source>
<bucketName>${deploy.bucketName}</bucketName>
<destination>${project.build.finalName}-bin.tar.gz</destination>
</configuration>
<executions>
<execution>
<id>deploy-to-s3</id>
<phase>deploy</phase>
<goals>
<goal>s3-upload</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>templating-maven-plugin</artifactId>
<version>1.0.0</version>
<executions>
<execution>
<id>filter-src</id>
<goals>
<goal>filter-sources</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@@ -1,3 +0,0 @@
all:
protoc --java_out=../src/main/java/ TextSecure.proto PubSubMessage.proto

View File

@@ -0,0 +1,15 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm;
public class WhisperServerVersion {
private static final String VERSION = "${project.version}";
public static String getServerVersion() {
return VERSION;
}
}

View File

@@ -21,6 +21,7 @@ import org.whispersystems.textsecuregcm.configuration.AppConfigConfiguration;
import org.whispersystems.textsecuregcm.configuration.AwsAttachmentsConfiguration;
import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
import org.whispersystems.textsecuregcm.configuration.DirectoryConfiguration;
import org.whispersystems.textsecuregcm.configuration.DonationConfiguration;
import org.whispersystems.textsecuregcm.configuration.DynamoDbConfiguration;
@@ -29,7 +30,7 @@ import org.whispersystems.textsecuregcm.configuration.GcpAttachmentsConfiguratio
import org.whispersystems.textsecuregcm.configuration.MaxDeviceConfiguration;
import org.whispersystems.textsecuregcm.configuration.MessageCacheConfiguration;
import org.whispersystems.textsecuregcm.configuration.MessageDynamoDbConfiguration;
import org.whispersystems.textsecuregcm.configuration.MicrometerConfiguration;
import org.whispersystems.textsecuregcm.configuration.WavefrontConfiguration;
import org.whispersystems.textsecuregcm.configuration.PaymentsServiceConfiguration;
import org.whispersystems.textsecuregcm.configuration.PushConfiguration;
import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration;
@@ -79,7 +80,12 @@ public class WhisperServerConfiguration extends Configuration {
@NotNull
@Valid
@JsonProperty
private MicrometerConfiguration micrometer;
private WavefrontConfiguration wavefront;
@NotNull
@Valid
@JsonProperty
private DatadogConfiguration datadog;
@NotNull
@Valid
@@ -393,8 +399,12 @@ public class WhisperServerConfiguration extends Configuration {
return cdn;
}
public MicrometerConfiguration getMicrometerConfiguration() {
return micrometer;
public WavefrontConfiguration getWavefrontConfiguration() {
return wavefront;
}
public DatadogConfiguration getDatadogConfiguration() {
return datadog;
}
public UnidentifiedDeliveryConfiguration getDeliveryCertificate() {

View File

@@ -6,17 +6,10 @@ package org.whispersystems.textsecuregcm;
import static com.codahale.metrics.MetricRegistry.name;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.codahale.metrics.SharedMetricRegistries;
@@ -39,16 +32,24 @@ import io.dropwizard.setup.Bootstrap;
import io.dropwizard.setup.Environment;
import io.lettuce.core.resource.ClientResources;
import io.micrometer.core.instrument.Clock;
import io.micrometer.core.instrument.Meter.Id;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.config.MeterFilter;
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
import io.micrometer.datadog.DatadogConfig;
import io.micrometer.datadog.DatadogMeterRegistry;
import io.micrometer.wavefront.WavefrontConfig;
import io.micrometer.wavefront.WavefrontMeterRegistry;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.net.http.HttpClient;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
@@ -65,6 +66,8 @@ import org.jdbi.v3.core.Jdbi;
import org.signal.zkgroup.ServerSecretParams;
import org.signal.zkgroup.auth.ServerZkAuthOperations;
import org.signal.zkgroup.profiles.ServerZkProfileOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.dispatch.DispatchManager;
import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
@@ -119,6 +122,7 @@ import org.whispersystems.textsecuregcm.metrics.FreeMemoryGauge;
import org.whispersystems.textsecuregcm.metrics.GarbageCollectionGauges;
import org.whispersystems.textsecuregcm.metrics.MaxFileDescriptorGauge;
import org.whispersystems.textsecuregcm.metrics.MetricsApplicationEventListener;
import org.whispersystems.textsecuregcm.metrics.MetricsRequestEventListener;
import org.whispersystems.textsecuregcm.metrics.NetworkReceivedGauge;
import org.whispersystems.textsecuregcm.metrics.NetworkSentGauge;
import org.whispersystems.textsecuregcm.metrics.NstatCounters;
@@ -188,32 +192,37 @@ import org.whispersystems.textsecuregcm.storage.Usernames;
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.util.AsnManager;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
import org.whispersystems.textsecuregcm.util.TorExitNodeManager;
import org.whispersystems.textsecuregcm.websocket.AuthenticatedConnectListener;
import org.whispersystems.textsecuregcm.websocket.DeadLetterHandler;
import org.whispersystems.textsecuregcm.websocket.ProvisioningConnectListener;
import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator;
import org.whispersystems.textsecuregcm.workers.CertificateCommand;
import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand;
import org.whispersystems.textsecuregcm.workers.DeleteUserCommand;
import org.whispersystems.textsecuregcm.workers.GetRedisCommandStatsCommand;
import org.whispersystems.textsecuregcm.workers.GetRedisSlowlogCommand;
import org.whispersystems.textsecuregcm.workers.ServerVersionCommand;
import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask;
import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask;
import org.whispersystems.textsecuregcm.workers.VacuumCommand;
import org.whispersystems.textsecuregcm.workers.ZkParamsCommand;
import org.whispersystems.websocket.WebSocketResourceProviderFactory;
import org.whispersystems.websocket.setup.WebSocketEnvironment;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
public class WhisperServerService extends Application<WhisperServerConfiguration> {
private static final Logger log = LoggerFactory.getLogger(WhisperServerService.class);
@Override
public void initialize(Bootstrap<WhisperServerConfiguration> bootstrap) {
bootstrap.addCommand(new VacuumCommand());
bootstrap.addCommand(new DeleteUserCommand());
bootstrap.addCommand(new CertificateCommand());
bootstrap.addCommand(new ZkParamsCommand());
bootstrap.addCommand(new GetRedisSlowlogCommand());
bootstrap.addCommand(new GetRedisCommandStatsCommand());
bootstrap.addCommand(new ServerVersionCommand());
bootstrap.addCommand(new CheckDynamicConfigurationCommand());
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("accountdb", "accountsdb.xml") {
@Override
@@ -239,8 +248,13 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
@Override
public void run(WhisperServerConfiguration config, Environment environment)
throws Exception {
SharedMetricRegistries.add(Constants.METRICS_NAME, environment.metrics());
final DistributionStatisticConfig defaultDistributionStatisticConfig = DistributionStatisticConfig.builder()
.percentiles(.75, .95, .99, .999)
.build();
final WavefrontConfig wavefrontConfig = new WavefrontConfig() {
@Override
public String get(final String key) {
@@ -249,25 +263,80 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
@Override
public String uri() {
return config.getMicrometerConfiguration().getUri();
return config.getWavefrontConfiguration().getUri();
}
@Override
public int batchSize() {
return config.getMicrometerConfiguration().getBatchSize();
return config.getWavefrontConfiguration().getBatchSize();
}
};
Metrics.addRegistry(new WavefrontMeterRegistry(wavefrontConfig, Clock.SYSTEM) {
@Override
protected DistributionStatisticConfig defaultHistogramConfig() {
return DistributionStatisticConfig.builder()
.percentiles(.75, .95, .99, .999)
.build()
.merge(super.defaultHistogramConfig());
return defaultDistributionStatisticConfig.merge(super.defaultHistogramConfig());
}
});
{
final String hostname;
{
String localHostName = "unknown";
try {
localHostName = InetAddress.getLocalHost().getHostName().toLowerCase(Locale.US);
log.info("Setting host tag to {}", localHostName);
} catch (final UnknownHostException e) {
log.warn("Failed to get hostname", e);
}
hostname = localHostName;
}
final DatadogMeterRegistry datadogMeterRegistry = new DatadogMeterRegistry(new DatadogConfig() {
@Override
public String get(final String key) {
return null;
}
@Override
public String apiKey() {
return config.getDatadogConfiguration().getApiKey();
}
@Override
public Duration step() {
return config.getDatadogConfiguration().getStep();
}
@Override
public String hostTag() {
return "host";
}
}, Clock.SYSTEM);
datadogMeterRegistry.config().commonTags(
Tags.of(
"service", "chat",
"host", hostname,
"version", WhisperServerVersion.getServerVersion(),
"env", config.getDatadogConfiguration().getEnvironment()))
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.REQUEST_COUNTER_NAME))
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.ANDROID_REQUEST_COUNTER_NAME))
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.DESKTOP_REQUEST_COUNTER_NAME))
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.IOS_REQUEST_COUNTER_NAME))
.meterFilter(new MeterFilter() {
@Override
public DistributionStatisticConfig configure(final Id id, final DistributionStatisticConfig config) {
return defaultDistributionStatisticConfig.merge(config);
}
});
Metrics.addRegistry(datadogMeterRegistry);
}
environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
environment.getObjectMapper().setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
environment.getObjectMapper().setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
@@ -279,80 +348,40 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("accounts_database", accountJdbi, config.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
FaultTolerantDatabase abuseDatabase = new FaultTolerantDatabase("abuse_database", abuseJdbi, config.getAbuseDatabaseConfiguration().getCircuitBreakerConfiguration());
AmazonDynamoDBClientBuilder messageDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(config.getMessageDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) config.getMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDbClient messageDynamoDb = DynamoDbFromConfig.client(config.getMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
AmazonDynamoDBClientBuilder keysDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(config.getKeysDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getKeysDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) config.getKeysDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDbClient preKeyDynamoDb = DynamoDbFromConfig.client(config.getKeysDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
AmazonDynamoDBClientBuilder accountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(config.getAccountsDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) config.getAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(config.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
// The thread pool core & max sizes are set via dynamic configuration within AccountsDynamoDb
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
new LinkedBlockingDeque<>());
AmazonDynamoDBAsyncClientBuilder accountsDynamoDbAsyncClientBuilder = AmazonDynamoDBAsyncClientBuilder
.standard()
.withRegion(accountsDynamoDbClientBuilder.getRegion())
.withClientConfiguration(accountsDynamoDbClientBuilder.getClientConfiguration())
.withCredentials(accountsDynamoDbClientBuilder.getCredentials())
.withExecutorFactory(() -> accountsDynamoDbMigrationThreadPool);
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(config.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
accountsDynamoDbMigrationThreadPool);
AmazonDynamoDBClientBuilder migrationDeletedAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(config.getMigrationDeletedAccountsDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getMigrationDeletedAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) config.getMigrationDeletedAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDbClient recentlyDeletedAccountsDynamoDb = DynamoDbFromConfig.client(config.getMigrationDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
AmazonDynamoDBClientBuilder migrationRetryAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(config.getMigrationRetryAccountsDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getMigrationRetryAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) config.getMigrationRetryAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDbClient pushChallengeDynamoDbClient = DynamoDbFromConfig.client(config.getPushChallengeDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
AmazonDynamoDBClientBuilder pushChallengeDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(config.getPushChallengeDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getPushChallengeDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) config.getPushChallengeDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDbClient reportMessageDynamoDbClient = DynamoDbFromConfig.client(config.getReportMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
AmazonDynamoDBClientBuilder reportMessageDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(config.getReportMessageDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getReportMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) config.getReportMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDB messageDynamoDb = new DynamoDB(messageDynamoDbClientBuilder.build());
DynamoDB preKeyDynamoDb = new DynamoDB(keysDynamoDbClientBuilder.build());
AmazonDynamoDB accountsDynamoDbClient = accountsDynamoDbClientBuilder.build();
AmazonDynamoDBAsync accountsDynamodbAsyncClient = accountsDynamoDbAsyncClientBuilder.build();
DynamoDB recentlyDeletedAccountsDynamoDb = new DynamoDB(migrationDeletedAccountsDynamoDbClientBuilder.build());
DynamoDB migrationRetryAccountsDynamoDb = new DynamoDB(migrationRetryAccountsDynamoDbClientBuilder.build());
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(config.getMigrationRetryAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(recentlyDeletedAccountsDynamoDb, config.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, config.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
Accounts accounts = new Accounts(accountDatabase);
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamodbAsyncClient, accountsDynamoDbMigrationThreadPool, new DynamoDB(accountsDynamoDbClient), config.getAccountsDynamoDbConfiguration().getTableName(), config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, config.getAccountsDynamoDbConfiguration().getTableName(), config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
PendingAccounts pendingAccounts = new PendingAccounts(accountDatabase);
PendingDevices pendingDevices = new PendingDevices (accountDatabase);
Usernames usernames = new Usernames(accountDatabase);
@@ -362,8 +391,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb, config.getMessageDynamoDbConfiguration().getTableName(), config.getMessageDynamoDbConfiguration().getTimeToLive());
AbusiveHostRules abusiveHostRules = new AbusiveHostRules(abuseDatabase);
RemoteConfigs remoteConfigs = new RemoteConfigs(accountDatabase);
PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(new DynamoDB(pushChallengeDynamoDbClientBuilder.build()), config.getPushChallengeDynamoDbConfiguration().getTableName());
ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(new DynamoDB(reportMessageDynamoDbClientBuilder.build()), config.getReportMessageDynamoDbConfiguration().getTableName());
PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(pushChallengeDynamoDbClient, config.getPushChallengeDynamoDbConfiguration().getTableName());
ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(reportMessageDynamoDbClient, config.getReportMessageDynamoDbConfiguration().getTableName());
RedisClientFactory pubSubClientFactory = new RedisClientFactory("pubsub_cache", config.getPubsubCacheConfiguration().getUrl(), config.getPubsubCacheConfiguration().getReplicaUrls(), config.getPubsubCacheConfiguration().getCircuitBreakerConfiguration());
ReplicatedJedisPool pubsubClient = pubSubClientFactory.getRedisClientPool();

View File

@@ -0,0 +1,38 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.time.Duration;
public class DatadogConfiguration {
@JsonProperty
@NotBlank
private String apiKey;
@JsonProperty
@NotNull
private Duration step = Duration.ofSeconds(10);
@JsonProperty
@NotBlank
private String environment;
public String getApiKey() {
return apiKey;
}
public Duration getStep() {
return step;
}
public String getEnvironment() {
return environment;
}
}

View File

@@ -9,7 +9,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.constraints.Positive;
public class MicrometerConfiguration {
public class WavefrontConfiguration {
@JsonProperty
private String uri;

View File

@@ -170,7 +170,8 @@ public class AccountController {
@Path("/{type}/preauth/{token}/{number}")
public Response getPreAuth(@PathParam("type") String pushType,
@PathParam("token") String pushToken,
@PathParam("number") String number)
@PathParam("number") String number,
@QueryParam("voip") Optional<Boolean> useVoip)
{
if (!"apn".equals(pushType) && !"fcm".equals(pushType)) {
return Response.status(400).build();
@@ -190,7 +191,7 @@ public class AccountController {
if ("fcm".equals(pushType)) {
gcmSender.sendMessage(new GcmMessage(pushToken, number, 0, GcmMessage.Type.CHALLENGE, Optional.of(storedVerificationCode.getPushCode())));
} else if ("apn".equals(pushType)) {
apnSender.sendMessage(new ApnMessage(pushToken, number, 0, true, ApnMessage.Type.CHALLENGE, Optional.of(storedVerificationCode.getPushCode())));
apnSender.sendMessage(new ApnMessage(pushToken, number, 0, useVoip.orElse(true), ApnMessage.Type.CHALLENGE, Optional.of(storedVerificationCode.getPushCode())));
} else {
throw new AssertionError();
}

View File

@@ -234,6 +234,10 @@ public class DeviceController {
private boolean isCapabilityDowngrade(Account account, DeviceCapabilities capabilities, String userAgent) {
boolean isDowngrade = false;
if (account.isSenderKeySupported() && !capabilities.isSenderKey()) {
isDowngrade = true;
}
if (account.isGv1MigrationSupported() && !capabilities.isGv1Migration()) {
isDowngrade = true;
}

View File

@@ -76,6 +76,7 @@ import org.whispersystems.textsecuregcm.entities.MultiRecipientMessage.Recipient
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntity;
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntityList;
import org.whispersystems.textsecuregcm.entities.SendMessageResponse;
import org.whispersystems.textsecuregcm.entities.SendMultiRecipientMessageResponse;
import org.whispersystems.textsecuregcm.entities.StaleDevices;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeException;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
@@ -133,6 +134,7 @@ public class MessageController {
private final ClusterLuaScript recordInternationalUnsealedSenderMetricsScript;
private static final String LEGACY_MESSAGE_SENT_COUNTER = name(MessageController.class, "legacyMessageSent");
private static final String SENT_MESSAGE_COUNTER_NAME = name(MessageController.class, "sentMessages");
private static final String REJECT_UNSEALED_SENDER_COUNTER_NAME = name(MessageController.class, "rejectUnsealedSenderLimit");
private static final String INTERNATIONAL_UNSEALED_SENDER_COUNTER_NAME = name(MessageController.class, "internationalUnsealedSender");
@@ -298,17 +300,8 @@ public class MessageController {
validateCompleteDeviceList(destination.get(), messages.getMessages(), isSyncMessage);
validateRegistrationIds(destination.get(), messages.getMessages());
// iOS versions prior to 5.5.0.7 send `online` on IncomingMessageList.message, rather on the top-level entity.
// This causes some odd client behaviors, such as persisted typing indicators, so we have a temporary
// server-side adaptation.
final boolean online = messages.getMessages()
.stream()
.findFirst()
.map(IncomingMessage::isOnline)
.orElse(messages.isOnline());
final List<Tag> tags = List.of(UserAgentTagUtil.getPlatformTag(userAgent),
Tag.of(EPHEMERAL_TAG_NAME, String.valueOf(online)),
Tag.of(EPHEMERAL_TAG_NAME, String.valueOf(messages.isOnline())),
Tag.of(SENDER_TYPE_TAG_NAME, senderType));
for (IncomingMessage incomingMessage : messages.getMessages()) {
@@ -316,7 +309,7 @@ public class MessageController {
if (destinationDevice.isPresent()) {
Metrics.counter(SENT_MESSAGE_COUNTER_NAME, tags).increment();
sendMessage(source, destination.get(), destinationDevice.get(), messages.getTimestamp(), online, incomingMessage);
sendMessage(source, destination.get(), destinationDevice.get(), messages.getTimestamp(), messages.isOnline(), incomingMessage);
}
}
@@ -426,7 +419,7 @@ public class MessageController {
uuids404.add(destinationAccount.getUuid());
}
}
return Response.ok(new SendMessageResponse(uuids404)).build();
return Response.ok(new SendMultiRecipientMessageResponse(uuids404)).build();
}
private void checkAccessKeys(CombinedUnidentifiedSenderAccessKeys accessKeys, Map<UUID, Account> uuidToAccountMap) {
@@ -556,7 +549,7 @@ public class MessageController {
account.getAuthenticatedDevice().get().getId(),
source, timestamp);
if (message.isPresent() && message.get().getType() != Envelope.Type.RECEIPT_VALUE) {
if (message.isPresent() && message.get().getType() != Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE) {
receiptSender.sendReceipt(account,
message.get().getSource(),
message.get().getTimestamp());
@@ -578,7 +571,7 @@ public class MessageController {
if (message.isPresent()) {
WebSocketConnection.recordMessageDeliveryDuration(message.get().getTimestamp(), account.getAuthenticatedDevice().get());
if (!Util.isEmpty(message.get().getSource()) && message.get().getType() != Envelope.Type.RECEIPT_VALUE) {
if (!Util.isEmpty(message.get().getSource()) && message.get().getType() != Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE) {
receiptSender.sendReceipt(account, message.get().getSource(), message.get().getTimestamp());
}
}
@@ -612,7 +605,7 @@ public class MessageController {
Optional<byte[]> messageContent = getMessageContent(incomingMessage);
Envelope.Builder messageBuilder = Envelope.newBuilder();
messageBuilder.setType(Envelope.Type.valueOf(incomingMessage.getType()))
messageBuilder.setType(Envelope.Type.forNumber(incomingMessage.getType()))
.setTimestamp(timestamp == 0 ? System.currentTimeMillis() : timestamp)
.setServerTimestamp(System.currentTimeMillis());
@@ -623,6 +616,7 @@ public class MessageController {
}
if (messageBody.isPresent()) {
Metrics.counter(LEGACY_MESSAGE_SENT_COUNTER).increment();
messageBuilder.setLegacyMessage(ByteString.copyFrom(messageBody.get()));
}
@@ -647,7 +641,7 @@ public class MessageController {
byte[] payload = new byte[1 + recipientKeyMaterial.length + commonPayload.length];
payload[0] = MultiRecipientMessageProvider.VERSION;
System.arraycopy(recipientKeyMaterial, 0, payload, 1, recipientKeyMaterial.length);
System.arraycopy(commonPayload, 0, payload, 1 + recipientKeyMaterial.length, payload.length);
System.arraycopy(commonPayload, 0, payload, 1 + recipientKeyMaterial.length, commonPayload.length);
messageBuilder
.setType(Type.UNIDENTIFIED_SENDER)

View File

@@ -32,9 +32,6 @@ public class IncomingMessage {
@JsonProperty
private long timestamp; // deprecated
@JsonProperty
private Boolean online; // use IncomingMessageList.online - this is a temporary adaptation for older clients
public String getDestination() {
return destination;
}
@@ -62,8 +59,4 @@ public class IncomingMessage {
public String getContent() {
return content;
}
public Boolean isOnline() {
return online;
}
}

View File

@@ -4,6 +4,8 @@
*/
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.Valid;
@@ -15,6 +17,7 @@ public class IncomingMessageList {
@JsonProperty
@NotNull
@Valid
@JsonInclude(Include.NON_NULL)
private List<IncomingMessage> messages;
@JsonProperty

View File

@@ -6,24 +6,15 @@
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
import java.util.UUID;
public class SendMessageResponse {
@JsonProperty
private boolean needsSync;
@JsonProperty
private List<UUID> uuids404;
public SendMessageResponse() {}
public SendMessageResponse(boolean needsSync) {
this.needsSync = needsSync;
}
public SendMessageResponse(List<UUID> uuids404) {
this.uuids404 = uuids404;
}
}

View File

@@ -0,0 +1,22 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
import java.util.UUID;
public class SendMultiRecipientMessageResponse {
@JsonProperty
private List<UUID> uuids404;
public SendMultiRecipientMessageResponse() {
}
public SendMultiRecipientMessageResponse(final List<UUID> uuids404) {
this.uuids404 = uuids404;
}
}

View File

@@ -26,6 +26,7 @@ import java.time.Duration;
import javax.validation.constraints.NotEmpty;
import net.logstash.logback.appender.LogstashTcpSocketAppender;
import net.logstash.logback.encoder.LogstashEncoder;
import org.whispersystems.textsecuregcm.WhisperServerVersion;
@JsonTypeName("logstashtcpsocket")
public class LogstashTcpSocketAppenderFactory extends AbstractAppenderFactory<ILoggingEvent> {
@@ -83,7 +84,8 @@ public class LogstashTcpSocketAppenderFactory extends AbstractAppenderFactory<IL
}
customFieldsNode.set("service", TextNode.valueOf("chat"));
customFieldsNode.set("ddsource", TextNode.valueOf("logstash"));
customFieldsNode.set("ddtags", TextNode.valueOf("env:" + environment));
customFieldsNode.set("ddtags", TextNode.valueOf("env:" + environment + ",version:" + WhisperServerVersion.getServerVersion()));
encoder.setCustomFields(customFieldsNode.toString());
final LayoutWrappingEncoder<ILoggingEvent> prefix = new LayoutWrappingEncoder<>();
final PatternLayout layout = new PatternLayout();

View File

@@ -29,18 +29,18 @@ import java.util.regex.Pattern;
/**
* Gathers and reports request-level metrics.
*/
class MetricsRequestEventListener implements RequestEventListener {
public class MetricsRequestEventListener implements RequestEventListener {
static final String REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "request");
static final String PATH_TAG = "path";
static final String STATUS_CODE_TAG = "status";
static final String TRAFFIC_SOURCE_TAG = "trafficSource";
public static final String REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "request");
public static final String ANDROID_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "androidRequest");
public static final String DESKTOP_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "desktopRequest");
public static final String IOS_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "iosRequest");
static final String ANDROID_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "androidRequest");
static final String DESKTOP_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "desktopRequest");
static final String IOS_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "iosRequest");
static final String OS_TAG = "os";
static final String SDK_TAG = "sdkVersion";
static final String PATH_TAG = "path";
static final String STATUS_CODE_TAG = "status";
static final String TRAFFIC_SOURCE_TAG = "trafficSource";
static final String OS_TAG = "os";
static final String SDK_TAG = "sdkVersion";
private static final Set<String> ACCEPTABLE_DESKTOP_OS_STRINGS = Set.of("linux", "macos", "windows");

View File

@@ -16,7 +16,8 @@ public class ApnMessage {
NOTIFICATION, CHALLENGE, RATE_LIMIT_CHALLENGE
}
public static final String APN_NOTIFICATION_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}}";
public static final String APN_VOIP_NOTIFICATION_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}}";
public static final String APN_NSE_NOTIFICATION_PAYLOAD = "{\"aps\":{\"mutable-content\":1,\"alert\":{\"loc-key\":\"APN_Message\"}}}";
public static final String APN_CHALLENGE_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}, \"challenge\" : \"%s\"}";
public static final String APN_RATE_LIMIT_CHALLENGE_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}, \"rateLimitChallenge\" : \"%s\"}";
public static final long MAX_EXPIRATION = Integer.MAX_VALUE * 1000L;
@@ -48,7 +49,7 @@ public class ApnMessage {
public String getMessage() {
switch (type) {
case NOTIFICATION:
return APN_NOTIFICATION_PAYLOAD;
return this.isVoip() ? APN_VOIP_NOTIFICATION_PAYLOAD : APN_NSE_NOTIFICATION_PAYLOAD;
case CHALLENGE:
return String.format(APN_CHALLENGE_PAYLOAD, challengeData.orElseThrow(AssertionError::new));

View File

@@ -43,7 +43,7 @@ public class ReceiptSender {
.setSourceUuid(source.getUuid().toString())
.setSourceDevice((int) source.getAuthenticatedDevice().get().getId())
.setTimestamp(messageId)
.setType(Envelope.Type.RECEIPT);
.setType(Envelope.Type.SERVER_DELIVERY_RECEIPT);
if (source.getRelay().isPresent()) {
message.setRelay(source.getRelay().get());

View File

@@ -5,21 +5,18 @@
package org.whispersystems.textsecuregcm.storage;
import com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Page;
import com.amazonaws.services.dynamodbv2.document.QueryOutcome;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
@@ -29,7 +26,7 @@ import static io.micrometer.core.instrument.Metrics.timer;
public class AbstractDynamoDbStore {
private final DynamoDB dynamoDb;
private final DynamoDbClient dynamoDbClient;
private final Timer batchWriteItemsFirstPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "true");
private final Timer batchWriteItemsRetryPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "false");
@@ -41,44 +38,31 @@ public class AbstractDynamoDbStore {
public static final int DYNAMO_DB_MAX_BATCH_SIZE = 25; // This limit comes from Amazon Dynamo DB itself. It will reject batch writes larger than this.
public static final int RESULT_SET_CHUNK_SIZE = 100;
public AbstractDynamoDbStore(final DynamoDB dynamoDb) {
this.dynamoDb = dynamoDb;
public AbstractDynamoDbStore(final DynamoDbClient dynamoDbClient) {
this.dynamoDbClient = dynamoDbClient;
}
protected DynamoDB getDynamoDb() {
return dynamoDb;
protected DynamoDbClient db() {
return dynamoDbClient;
}
protected void executeTableWriteItemsUntilComplete(final TableWriteItems items) {
AtomicReference<BatchWriteItemOutcome> outcome = new AtomicReference<>();
batchWriteItemsFirstPass.record(() -> outcome.set(dynamoDb.batchWriteItem(items)));
protected void executeTableWriteItemsUntilComplete(final Map<String,List<WriteRequest>> items) {
AtomicReference<BatchWriteItemResponse> outcome = new AtomicReference<>();
batchWriteItemsFirstPass.record(() -> outcome.set(dynamoDbClient.batchWriteItem(BatchWriteItemRequest.builder().requestItems(items).build())));
int attemptCount = 0;
while (!outcome.get().getUnprocessedItems().isEmpty() && attemptCount < MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE) {
batchWriteItemsRetryPass.record(() -> outcome.set(dynamoDb.batchWriteItemUnprocessed(outcome.get().getUnprocessedItems())));
while (!outcome.get().unprocessedItems().isEmpty() && attemptCount < MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE) {
batchWriteItemsRetryPass.record(() -> outcome.set(dynamoDbClient.batchWriteItem(BatchWriteItemRequest.builder()
.requestItems(outcome.get().unprocessedItems())
.build())));
++attemptCount;
}
if (!outcome.get().getUnprocessedItems().isEmpty()) {
logger.error("Attempt count ({}) reached max ({}}) before applying all batch writes to dynamo. {} unprocessed items remain.", attemptCount, MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE, outcome.get().getUnprocessedItems().size());
batchWriteItemsUnprocessed.increment(outcome.get().getUnprocessedItems().size());
if (!outcome.get().unprocessedItems().isEmpty()) {
int totalItems = outcome.get().unprocessedItems().values().stream().mapToInt(List::size).sum();
logger.error("Attempt count ({}) reached max ({}}) before applying all batch writes to dynamo. {} unprocessed items remain.", attemptCount, MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE, totalItems);
batchWriteItemsUnprocessed.increment(totalItems);
}
}
protected long countItemsMatchingQuery(final Table table, final QuerySpec querySpec) {
// This is very confusing, but does appear to be the intended behavior. See:
//
// - https://github.com/aws/aws-sdk-java/issues/693
// - https://github.com/aws/aws-sdk-java/issues/915
// - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.Count
long matchingItems = 0;
for (final Page<Item, QueryOutcome> page : table.query(querySpec).pages()) {
matchingItems += page.getLowLevelResult().getQueryResult().getCount();
}
return matchingItems;
}
static <T> void writeInBatches(final Iterable<T> items, final Consumer<List<T>> action) {
final List<T> batch = new ArrayList<>(DYNAMO_DB_MAX_BATCH_SIZE);

View File

@@ -2,25 +2,6 @@ package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.amazonaws.handlers.AsyncHandler;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.CancellationReason;
import com.amazonaws.services.dynamodbv2.model.Delete;
import com.amazonaws.services.dynamodbv2.model.GetItemResult;
import com.amazonaws.services.dynamodbv2.model.Put;
import com.amazonaws.services.dynamodbv2.model.ReturnValuesOnConditionCheckFailure;
import com.amazonaws.services.dynamodbv2.model.TransactWriteItem;
import com.amazonaws.services.dynamodbv2.model.TransactWriteItemsRequest;
import com.amazonaws.services.dynamodbv2.model.TransactWriteItemsResult;
import com.amazonaws.services.dynamodbv2.model.TransactionCanceledException;
import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.Counter;
@@ -33,12 +14,27 @@ import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.CancellationReason;
import software.amazon.awssdk.services.dynamodb.model.Delete;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
import software.amazon.awssdk.services.dynamodb.model.Put;
import software.amazon.awssdk.services.dynamodb.model.ReturnValuesOnConditionCheckFailure;
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItem;
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
import software.amazon.awssdk.services.dynamodb.model.TransactionCanceledException;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountStore {
@@ -51,9 +47,8 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
static final String ATTR_MIGRATION_VERSION = "V";
private final AmazonDynamoDB client;
private final Table accountsTable;
private final AmazonDynamoDBAsync asyncClient;
private final DynamoDbClient client;
private final DynamoDbAsyncClient asyncClient;
private final ThreadPoolExecutor migrationThreadPool;
@@ -61,6 +56,7 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
private final MigrationRetryAccounts migrationRetryAccounts;
private final String phoneNumbersTableName;
private final String accountsTableName;
private static final Timer CREATE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "create"));
private static final Timer UPDATE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "update"));
@@ -70,18 +66,17 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
private final Logger logger = LoggerFactory.getLogger(AccountsDynamoDb.class);
public AccountsDynamoDb(AmazonDynamoDB client, AmazonDynamoDBAsync asyncClient,
ThreadPoolExecutor migrationThreadPool, DynamoDB dynamoDb, String accountsTableName, String phoneNumbersTableName,
public AccountsDynamoDb(DynamoDbClient client, DynamoDbAsyncClient asyncClient,
ThreadPoolExecutor migrationThreadPool, String accountsTableName, String phoneNumbersTableName,
MigrationDeletedAccounts migrationDeletedAccounts,
MigrationRetryAccounts accountsMigrationErrors) {
super(dynamoDb);
super(client);
this.client = client;
this.accountsTable = dynamoDb.getTable(accountsTableName);
this.phoneNumbersTableName = phoneNumbersTableName;
this.asyncClient = asyncClient;
this.phoneNumbersTableName = phoneNumbersTableName;
this.accountsTableName = accountsTableName;
this.migrationThreadPool = migrationThreadPool;
this.migrationDeletedAccounts = migrationDeletedAccounts;
@@ -90,32 +85,34 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
@Override
public boolean create(Account account) {
return CREATE_TIMER.record(() -> {
try {
TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid());
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder()
.conditionExpression("attribute_not_exists(#number) OR #number = :number")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(Map.of(":number", AttributeValues.fromString(account.getNumber()))));
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid());
final TransactWriteItemsRequest request = new TransactWriteItemsRequest()
.withTransactItems(phoneNumberConstraintPut, accountPut);
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(phoneNumberConstraintPut, accountPut)
.build();
try {
client.transactWriteItems(request);
} catch (TransactionCanceledException e) {
final CancellationReason accountCancellationReason = e.getCancellationReasons().get(1);
final CancellationReason accountCancellationReason = e.cancellationReasons().get(1);
if ("ConditionalCheckFailed".equals(accountCancellationReason.getCode())) {
if ("ConditionalCheckFailed".equals(accountCancellationReason.code())) {
throw new IllegalArgumentException("uuid present with different phone number");
}
final CancellationReason phoneNumberConstraintCancellationReason = e.getCancellationReasons().get(0);
final CancellationReason phoneNumberConstraintCancellationReason = e.cancellationReasons().get(0);
if ("ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.getCode())) {
if ("ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.code())) {
ByteBuffer actualAccountUuid = phoneNumberConstraintCancellationReason.getItem().get(KEY_ACCOUNT_UUID).getB();
ByteBuffer actualAccountUuid = phoneNumberConstraintCancellationReason.item().get(KEY_ACCOUNT_UUID).b().asByteBuffer();
account.setUuid(UUIDUtil.fromByteBuffer(actualAccountUuid));
update(account);
@@ -134,39 +131,37 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
});
}
private TransactWriteItem buildPutWriteItemForAccount(Account account, UUID uuid) throws JsonProcessingException {
return new TransactWriteItem()
.withPut(
new Put()
.withTableName(accountsTable.getTableName())
.withItem(Map.of(
KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid)),
ATTR_ACCOUNT_E164, new AttributeValue(account.getNumber()),
ATTR_ACCOUNT_DATA, new AttributeValue()
.withB(ByteBuffer.wrap(SystemMapper.getMapper().writeValueAsBytes(account))),
ATTR_MIGRATION_VERSION, new AttributeValue().withN(
String.valueOf(account.getDynamoDbMigrationVersion()))))
.withConditionExpression("attribute_not_exists(#number) OR #number = :number")
.withExpressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
.withExpressionAttributeValues(Map.of(":number", new AttributeValue(account.getNumber()))));
private TransactWriteItem buildPutWriteItemForAccount(Account account, UUID uuid, Put.Builder putBuilder) throws JsonProcessingException {
return TransactWriteItem.builder()
.put(putBuilder
.tableName(accountsTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid),
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
ATTR_ACCOUNT_DATA, AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
ATTR_MIGRATION_VERSION, AttributeValues.fromInt(account.getDynamoDbMigrationVersion())))
.build())
.build();
}
private TransactWriteItem buildPutWriteItemForPhoneNumberConstraint(Account account, UUID uuid) {
return new TransactWriteItem()
.withPut(
new Put()
.withTableName(phoneNumbersTableName)
.withItem(Map.of(
ATTR_ACCOUNT_E164, new AttributeValue(account.getNumber()),
KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid))))
.withConditionExpression(
return TransactWriteItem.builder()
.put(
Put.builder()
.tableName(phoneNumbersTableName)
.item(Map.of(
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
.conditionExpression(
"attribute_not_exists(#number) OR (attribute_exists(#number) AND #uuid = :uuid)")
.withExpressionAttributeNames(
.expressionAttributeNames(
Map.of("#uuid", KEY_ACCOUNT_UUID,
"#number", ATTR_ACCOUNT_E164))
.withExpressionAttributeValues(
Map.of(":uuid", new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid))))
.withReturnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD));
.expressionAttributeValues(
Map.of(":uuid", AttributeValues.fromUUID(uuid)))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
}
@Override
@@ -174,16 +169,18 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
UPDATE_TIMER.record(() -> {
UpdateItemRequest updateItemRequest;
try {
updateItemRequest = new UpdateItemRequest()
.withTableName(accountsTable.getTableName())
.withKey(Map.of(KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(account.getUuid()))))
.withUpdateExpression("SET #data = :data, #version = :version")
.withConditionExpression("attribute_exists(#number)")
.withExpressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
updateItemRequest = UpdateItemRequest.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.updateExpression("SET #data = :data, #version = :version")
.conditionExpression("attribute_exists(#number)")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
"#data", ATTR_ACCOUNT_DATA,
"#version", ATTR_MIGRATION_VERSION))
.withExpressionAttributeValues(Map.of(":data", new AttributeValue().withB(ByteBuffer.wrap(SystemMapper.getMapper().writeValueAsBytes(account))),
":version", new AttributeValue().withN(String.valueOf(account.getDynamoDbMigrationVersion()))));
.expressionAttributeValues(Map.of(
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":version", AttributeValues.fromInt(account.getDynamoDbMigrationVersion())))
.build();
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(e);
@@ -193,37 +190,42 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
});
}
@Override
public Optional<Account> get(String number) {
return GET_BY_NUMBER_TIMER.record(() -> {
final GetItemResult phoneNumberAndUuid = client.getItem(phoneNumbersTableName,
Map.of(ATTR_ACCOUNT_E164, new AttributeValue(number)), true);
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(phoneNumbersTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(number)))
.build());
return Optional.ofNullable(phoneNumberAndUuid.getItem())
.map(item -> item.get(KEY_ACCOUNT_UUID).getB())
.map(uuid -> accountsTable.getItem(new GetItemSpec()
.withPrimaryKey(KEY_ACCOUNT_UUID, uuid.array())
.withConsistentRead(true)))
return Optional.ofNullable(response.item())
.map(item -> item.get(KEY_ACCOUNT_UUID))
.map(uuid -> accountByUuid(uuid))
.map(AccountsDynamoDb::fromItem);
});
}
private Map<String, AttributeValue> accountByUuid(AttributeValue uuid) {
GetItemResponse r = client.getItem(GetItemRequest.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, uuid))
.consistentRead(true)
.build());
return r.item().isEmpty() ? null : r.item();
}
@Override
public Optional<Account> get(UUID uuid) {
Optional<Item> maybeItem = GET_BY_UUID_TIMER.record(() ->
Optional.ofNullable(accountsTable.getItem(new GetItemSpec().
withPrimaryKey(new PrimaryKey(KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(uuid)))
.withConsistentRead(true))));
return maybeItem.map(AccountsDynamoDb::fromItem);
return GET_BY_UUID_TIMER.record(() ->
Optional.ofNullable(accountByUuid(AttributeValues.fromUUID(uuid)))
.map(AccountsDynamoDb::fromItem));
}
@Override
public void delete(UUID uuid) {
DELETE_TIMER.record(() -> {
delete(uuid, true);
});
}
@@ -238,18 +240,22 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
maybeAccount.ifPresent(account -> {
TransactWriteItem phoneNumberDelete = new TransactWriteItem()
.withDelete(new Delete()
.withTableName(phoneNumbersTableName)
.withKey(Map.of(ATTR_ACCOUNT_E164, new AttributeValue(account.getNumber()))));
TransactWriteItem phoneNumberDelete = TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumbersTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber())))
.build())
.build();
TransactWriteItem accountDelete = new TransactWriteItem().withDelete(
new Delete()
.withTableName(accountsTable.getTableName())
.withKey(Map.of(KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid)))));
TransactWriteItem accountDelete = TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
.build())
.build();
TransactWriteItemsRequest request = new TransactWriteItemsRequest()
.withTransactItems(phoneNumberDelete, accountDelete);
TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(phoneNumberDelete, accountDelete).build();
client.transactWriteItems(request);
});
@@ -299,64 +305,62 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
try {
TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid());
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid());
accountPut.getPut()
.setConditionExpression("attribute_not_exists(#uuid) OR (attribute_exists(#uuid) AND #version < :version)");
accountPut.getPut()
.setExpressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID,
"#version", ATTR_MIGRATION_VERSION));
accountPut.getPut()
.setExpressionAttributeValues(
Map.of(":version", new AttributeValue().withN(String.valueOf(account.getDynamoDbMigrationVersion()))));
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder()
.conditionExpression("attribute_not_exists(#uuid) OR (attribute_exists(#uuid) AND #version < :version)")
.expressionAttributeNames(Map.of(
"#uuid", KEY_ACCOUNT_UUID,
"#version", ATTR_MIGRATION_VERSION))
.expressionAttributeValues(Map.of(
":version", AttributeValues.fromInt(account.getDynamoDbMigrationVersion()))));
final TransactWriteItemsRequest request = new TransactWriteItemsRequest()
.withTransactItems(phoneNumberConstraintPut, accountPut);
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(phoneNumberConstraintPut, accountPut).build();
final CompletableFuture<Boolean> resultFuture = new CompletableFuture<>();
asyncClient.transactWriteItemsAsync(request,
new AsyncHandler<>() {
@Override
public void onError(Exception exception) {
if (exception instanceof TransactionCanceledException) {
// account is already migrated
resultFuture.complete(false);
} else {
try {
migrationRetryAccounts.put(account.getUuid());
} catch (final Exception e) {
logger.error("Could not store account {}", account.getUuid());
}
resultFuture.completeExceptionally(exception);
}
}
@Override
public void onSuccess(TransactWriteItemsRequest request, TransactWriteItemsResult transactWriteItemsResult) {
resultFuture.complete(true);
}
});
asyncClient.transactWriteItems(request).whenCompleteAsync((result, exception) -> {
if (result != null) {
resultFuture.complete(true);
return;
}
if (exception instanceof CompletionException) {
// whenCompleteAsync can wrap exceptions in a CompletionException; unwrap it to get to the root cause.
exception = exception.getCause();
}
if (exception instanceof TransactionCanceledException) {
// account is already migrated
resultFuture.complete(false);
return;
}
try {
migrationRetryAccounts.put(account.getUuid());
} catch (final Exception e) {
logger.error("Could not store account {}", account.getUuid());
}
resultFuture.completeExceptionally(exception);
});
return resultFuture;
} catch (Exception e) {
return CompletableFuture.failedFuture(e);
}
}
private static String extractCancellationReasonCodes(final TransactionCanceledException exception) {
return exception.getCancellationReasons().stream()
.map(CancellationReason::getCode)
return exception.cancellationReasons().stream()
.map(CancellationReason::code)
.collect(Collectors.joining(", "));
}
@VisibleForTesting
static Account fromItem(Item item) {
static Account fromItem(Map<String, AttributeValue> item) {
if (!item.containsKey(ATTR_ACCOUNT_DATA) ||
!item.containsKey(ATTR_ACCOUNT_E164) ||
!item.containsKey(KEY_ACCOUNT_UUID)) {
throw new RuntimeException("item missing values");
}
try {
Account account = SystemMapper.getMapper().readValue(item.getBinary(ATTR_ACCOUNT_DATA), Account.class);
account.setNumber(item.getString(ATTR_ACCOUNT_E164));
account.setUuid(UUIDUtil.fromByteBuffer(item.getByteBuffer(KEY_ACCOUNT_UUID)));
Account account = SystemMapper.getMapper().readValue(item.get(ATTR_ACCOUNT_DATA).b().asByteArray(), Account.class);
account.setNumber(item.get(ATTR_ACCOUNT_E164).s());
account.setUuid(UUIDUtil.fromByteBuffer(item.get(KEY_ACCOUNT_UUID).b().asByteBuffer()));
return account;

View File

@@ -7,7 +7,6 @@ package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
@@ -21,12 +20,16 @@ import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
import net.logstash.logback.argument.StructuredArguments;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.auth.AmbiguousIdentifier;
@@ -38,6 +41,7 @@ import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.Util;
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
public class AccountsManager {
@@ -112,6 +116,7 @@ public class AccountsManager {
this.migrationComparisonMapper = mapper.copy();
migrationComparisonMapper.addMixIn(Account.class, AccountComparisonMixin.class);
migrationComparisonMapper.addMixIn(Device.class, DeviceComparisonMixin.class);
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.experimentEnrollmentManager = experimentEnrollmentManager;
@@ -459,8 +464,14 @@ public class AccountsManager {
}
try {
if (!serializedEquals(databaseAccount.getRegistrationLock(), dynamoAccount.getRegistrationLock())) {
return Optional.of("registrationLock");
if (databaseAccount.getMasterDevice().isPresent() && dynamoAccount.getMasterDevice().isPresent()) {
if (!Objects.equals(databaseAccount.getMasterDevice().get().getSignedPreKey(), dynamoAccount.getMasterDevice().get().getSignedPreKey())) {
return Optional.of("masterDeviceSignedPreKey");
}
if (!Objects.equals(databaseAccount.getMasterDevice().get().getPushTimestamp(), dynamoAccount.getMasterDevice().get().getPushTimestamp())) {
return Optional.of("masterDevicePushTimestamp");
}
}
if (!serializedEquals(databaseAccount.getDevices(), dynamoAccount.getDevices())) {
@@ -475,10 +486,6 @@ public class AccountsManager {
throw new RuntimeException(e);
}
if (databaseAccount.getDynamoDbMigrationVersion() != dynamoAccount.getDynamoDbMigrationVersion()) {
return Optional.of("migrationVersion");
}
return Optional.empty();
}
@@ -521,11 +528,26 @@ public class AccountsManager {
if (maybeUUid.isPresent()
&& dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isLogMismatches()) {
logger.info("Mismatched {} for {}", mismatchDescription, maybeUUid.get());
final String abbreviatedCallChain = getAbbreviatedCallChain(new RuntimeException().getStackTrace());
logger.info("Mismatched account data: {}", StructuredArguments.entries(Map.of(
"type", mismatchDescription,
"uuid", maybeUUid.get(),
"callChain", abbreviatedCallChain
)));
}
});
}
private String getAbbreviatedCallChain(final StackTraceElement[] stackTrace) {
return Arrays.stream(stackTrace)
.filter(stackTraceElement -> stackTraceElement.getClassName().contains("org.whispersystems"))
.filter(stackTraceElement -> !(stackTraceElement.getClassName().endsWith("AccountsManager") && stackTraceElement.getMethodName().contains("compare")))
.map(stackTraceElement -> StringUtils.substringAfterLast(stackTraceElement.getClassName(), ".") + ":" + stackTraceElement.getMethodName())
.collect(Collectors.joining(" -> "));
}
private static abstract class AccountComparisonMixin extends Account {
@JsonIgnore
@@ -533,6 +555,13 @@ public class AccountsManager {
}
private static abstract class DeviceComparisonMixin extends Device {
@JsonIgnore
private long lastSeen;
}
private boolean serializedEquals(final Object database, final Object dynamo) throws JsonProcessingException {
final byte[] databaseSerialized = migrationComparisonMapper.writeValueAsBytes(database);
final byte[] dynamoSerialized = migrationComparisonMapper.writeValueAsBytes(dynamo);

View File

@@ -18,8 +18,12 @@ import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.util.Util;
import javax.validation.ConstraintViolation;
import javax.validation.Validation;
import javax.validation.Validator;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
@@ -32,16 +36,19 @@ public class DynamicConfigurationManager {
private final AmazonAppConfig appConfigClient;
private final AtomicReference<DynamicConfiguration> configuration = new AtomicReference<>();
private final Logger logger = LoggerFactory.getLogger(DynamicConfigurationManager.class);
private GetConfigurationResult lastConfigResult;
private boolean initialized = false;
public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory())
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory())
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.registerModule(new JavaTimeModule());
private static final Validator VALIDATOR = Validation.buildDefaultValidatorFactory().getValidator();
private static final Logger logger = LoggerFactory.getLogger(DynamicConfigurationManager.class);
public DynamicConfigurationManager(String application, String environment, String configurationName) {
this(AmazonAppConfigClient.builder()
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(10000).withRequestTimeout(10000))
@@ -104,7 +111,9 @@ public class DynamicConfigurationManager {
if (!StringUtils.equals(lastConfigResult.getConfigurationVersion(), previousVersion)) {
logger.info("Received new config version: {}", lastConfigResult.getConfigurationVersion());
maybeDynamicConfiguration = Optional.of(OBJECT_MAPPER.readValue(StandardCharsets.UTF_8.decode(lastConfigResult.getContent().asReadOnlyBuffer()).toString(), DynamicConfiguration.class));
maybeDynamicConfiguration =
parseConfiguration(StandardCharsets.UTF_8.decode(lastConfigResult.getContent().asReadOnlyBuffer()).toString());
} else {
// No change since last version
maybeDynamicConfiguration = Optional.empty();
@@ -113,6 +122,23 @@ public class DynamicConfigurationManager {
return maybeDynamicConfiguration;
}
@VisibleForTesting
public static Optional<DynamicConfiguration> parseConfiguration(final String configurationYaml) throws JsonProcessingException {
final DynamicConfiguration configuration = OBJECT_MAPPER.readValue(configurationYaml, DynamicConfiguration.class);
final Set<ConstraintViolation<DynamicConfiguration>> violations = VALIDATOR.validate(configuration);
final Optional<DynamicConfiguration> maybeDynamicConfiguration;
if (violations.isEmpty()) {
maybeDynamicConfiguration = Optional.of(configuration);
} else {
logger.warn("Failed to validate configuration: {}", violations);
maybeDynamicConfiguration = Optional.empty();
}
return maybeDynamicConfiguration;
}
private DynamicConfiguration retrieveInitialDynamicConfiguration() {
for (;;) {
try {

View File

@@ -7,195 +7,230 @@ package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.amazonaws.services.dynamodbv2.document.DeleteItemOutcome;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
import com.amazonaws.services.dynamodbv2.model.Select;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.whispersystems.textsecuregcm.entities.PreKey;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse;
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
import software.amazon.awssdk.services.dynamodb.model.PutRequest;
import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
import software.amazon.awssdk.services.dynamodb.model.QueryResponse;
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.Select;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
public class KeysDynamoDb extends AbstractDynamoDbStore {
private final Table table;
private final String tableName;
static final String KEY_ACCOUNT_UUID = "U";
static final String KEY_DEVICE_ID_KEY_ID = "DK";
static final String KEY_PUBLIC_KEY = "P";
static final String KEY_ACCOUNT_UUID = "U";
static final String KEY_DEVICE_ID_KEY_ID = "DK";
static final String KEY_PUBLIC_KEY = "P";
private static final Timer STORE_KEYS_TIMER = Metrics.timer(name(KeysDynamoDb.class, "storeKeys"));
private static final Timer TAKE_KEY_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForDevice"));
private static final Timer TAKE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForAccount"));
private static final Timer GET_KEY_COUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "getKeyCount"));
private static final Timer DELETE_KEYS_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForDevice"));
private static final Timer DELETE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForAccount"));
private static final DistributionSummary CONTESTED_KEY_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "contestedKeys"));
private static final DistributionSummary KEY_COUNT_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "keyCount"));
private static final Timer STORE_KEYS_TIMER = Metrics.timer(name(KeysDynamoDb.class, "storeKeys"));
private static final Timer TAKE_KEY_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForDevice"));
private static final Timer TAKE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForAccount"));
private static final Timer GET_KEY_COUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "getKeyCount"));
private static final Timer DELETE_KEYS_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForDevice"));
private static final Timer DELETE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForAccount"));
private static final DistributionSummary CONTESTED_KEY_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "contestedKeys"));
private static final DistributionSummary KEY_COUNT_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "keyCount"));
public KeysDynamoDb(final DynamoDB dynamoDB, final String tableName) {
super(dynamoDB);
public KeysDynamoDb(final DynamoDbClient dynamoDB, final String tableName) {
super(dynamoDB);
this.tableName = tableName;
}
this.table = dynamoDB.getTable(tableName);
}
public void store(final Account account, final long deviceId, final List<PreKey> keys) {
STORE_KEYS_TIMER.record(() -> {
delete(account, deviceId);
public void store(final Account account, final long deviceId, final List<PreKey> keys) {
STORE_KEYS_TIMER.record(() -> {
delete(account, deviceId);
writeInBatches(keys, batch -> {
List<WriteRequest> items = new ArrayList<>();
for (final PreKey preKey : batch) {
items.add(WriteRequest.builder()
.putRequest(PutRequest.builder()
.item(getItemFromPreKey(account.getUuid(), deviceId, preKey))
.build())
.build());
}
executeTableWriteItemsUntilComplete(Map.of(tableName, items));
});
});
}
writeInBatches(keys, batch -> {
final TableWriteItems items = new TableWriteItems(table.getTableName());
public Optional<PreKey> take(final Account account, final long deviceId) {
return TAKE_KEY_FOR_DEVICE_TIMER.record(() -> {
final AttributeValue partitionKey = getPartitionKey(account.getUuid());
QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
.expressionAttributeValues(Map.of(
":uuid", partitionKey,
":sortprefix", getSortKeyPrefix(deviceId)))
.projectionExpression(KEY_DEVICE_ID_KEY_ID)
.consistentRead(false)
.build();
for (final PreKey preKey : batch) {
items.addItemToPut(getItemFromPreKey(account.getUuid(), deviceId, preKey));
}
int contestedKeys = 0;
executeTableWriteItemsUntilComplete(items);
});
});
}
try {
QueryResponse response = db().query(queryRequest);
for (Map<String, AttributeValue> candidate : response.items()) {
DeleteItemRequest deleteItemRequest = DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(
KEY_ACCOUNT_UUID, partitionKey,
KEY_DEVICE_ID_KEY_ID, candidate.get(KEY_DEVICE_ID_KEY_ID)))
.returnValues(ReturnValue.ALL_OLD)
.build();
DeleteItemResponse deleteItemResponse = db().deleteItem(deleteItemRequest);
if (deleteItemResponse.hasAttributes()) {
return Optional.of(getPreKeyFromItem(deleteItemResponse.attributes()));
}
public Optional<PreKey> take(final Account account, final long deviceId) {
return TAKE_KEY_FOR_DEVICE_TIMER.record(() -> {
final byte[] partitionKey = getPartitionKey(account.getUuid());
contestedKeys++;
}
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
.withValueMap(Map.of(":uuid", partitionKey,
":sortprefix", getSortKeyPrefix(deviceId)))
.withProjectionExpression(KEY_DEVICE_ID_KEY_ID)
.withConsistentRead(false);
return Optional.empty();
} finally {
CONTESTED_KEY_DISTRIBUTION.record(contestedKeys);
}
});
}
int contestedKeys = 0;
public Map<Long, PreKey> take(final Account account) {
return TAKE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
final Map<Long, PreKey> preKeysByDeviceId = new HashMap<>();
try {
for (final Item candidate : table.query(querySpec)) {
final DeleteItemSpec deleteItemSpec = new DeleteItemSpec().withPrimaryKey(KEY_ACCOUNT_UUID, partitionKey, KEY_DEVICE_ID_KEY_ID, candidate.getBinary(KEY_DEVICE_ID_KEY_ID))
.withReturnValues(ReturnValue.ALL_OLD);
for (final Device device : account.getDevices()) {
take(account, device.getId()).ifPresent(preKey -> preKeysByDeviceId.put(device.getId(), preKey));
}
final DeleteItemOutcome outcome = table.deleteItem(deleteItemSpec);
return preKeysByDeviceId;
});
}
if (outcome.getItem() != null) {
return Optional.of(getPreKeyFromItem(outcome.getItem()));
}
public int getCount(final Account account, final long deviceId) {
return GET_KEY_COUNT_TIMER.record(() -> {
QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
.expressionAttributeValues(Map.of(
":uuid", getPartitionKey(account.getUuid()),
":sortprefix", getSortKeyPrefix(deviceId)))
.select(Select.COUNT)
.consistentRead(false)
.build();
contestedKeys++;
}
int keyCount = 0;
// This is very confusing, but does appear to be the intended behavior. See:
//
// - https://github.com/aws/aws-sdk-java/issues/693
// - https://github.com/aws/aws-sdk-java/issues/915
// - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.Count
for (final QueryResponse page : db().queryPaginator(queryRequest)) {
keyCount += page.count();
}
KEY_COUNT_DISTRIBUTION.record(keyCount);
return keyCount;
});
}
return Optional.empty();
} finally {
CONTESTED_KEY_DISTRIBUTION.record(contestedKeys);
}
});
}
public void delete(final Account account) {
DELETE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#uuid = :uuid")
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID))
.expressionAttributeValues(Map.of(
":uuid", getPartitionKey(account.getUuid())))
.projectionExpression(KEY_DEVICE_ID_KEY_ID)
.consistentRead(true)
.build();
public Map<Long, PreKey> take(final Account account) {
return TAKE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
final Map<Long, PreKey> preKeysByDeviceId = new HashMap<>();
deleteItemsForAccountMatchingQuery(account, queryRequest);
});
}
for (final Device device : account.getDevices()) {
take(account, device.getId()).ifPresent(preKey -> preKeysByDeviceId.put(device.getId(), preKey));
}
@VisibleForTesting
void delete(final Account account, final long deviceId) {
DELETE_KEYS_FOR_DEVICE_TIMER.record(() -> {
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
.expressionAttributeValues(Map.of(
":uuid", getPartitionKey(account.getUuid()),
":sortprefix", getSortKeyPrefix(deviceId)))
.projectionExpression(KEY_DEVICE_ID_KEY_ID)
.consistentRead(true)
.build();
return preKeysByDeviceId;
});
}
deleteItemsForAccountMatchingQuery(account, queryRequest);
});
}
public int getCount(final Account account, final long deviceId) {
return GET_KEY_COUNT_TIMER.record(() -> {
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
.withValueMap(Map.of(":uuid", getPartitionKey(account.getUuid()),
":sortprefix", getSortKeyPrefix(deviceId)))
.withSelect(Select.COUNT)
.withConsistentRead(false);
private void deleteItemsForAccountMatchingQuery(final Account account, final QueryRequest querySpec) {
final AttributeValue partitionKey = getPartitionKey(account.getUuid());
final int keyCount = (int)countItemsMatchingQuery(table, querySpec);
writeInBatches(db().query(querySpec).items(), batch -> {
List<WriteRequest> deletes = new ArrayList<>();
for (final Map<String, AttributeValue> item : batch) {
deletes.add(WriteRequest.builder()
.deleteRequest(DeleteRequest.builder()
.key(Map.of(
KEY_ACCOUNT_UUID, partitionKey,
KEY_DEVICE_ID_KEY_ID, item.get(KEY_DEVICE_ID_KEY_ID)))
.build())
.build());
}
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
});
}
KEY_COUNT_DISTRIBUTION.record(keyCount);
return keyCount;
});
}
private static AttributeValue getPartitionKey(final UUID accountUuid) {
return AttributeValues.fromUUID(accountUuid);
}
public void delete(final Account account) {
DELETE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid")
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID))
.withValueMap(Map.of(":uuid", getPartitionKey(account.getUuid())))
.withProjectionExpression(KEY_DEVICE_ID_KEY_ID)
.withConsistentRead(true);
private static AttributeValue getSortKey(final long deviceId, final long keyId) {
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]);
byteBuffer.putLong(deviceId);
byteBuffer.putLong(keyId);
return AttributeValues.fromByteBuffer(byteBuffer.flip());
}
deleteItemsForAccountMatchingQuery(account, querySpec);
});
}
@VisibleForTesting
static AttributeValue getSortKeyPrefix(final long deviceId) {
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
byteBuffer.putLong(deviceId);
return AttributeValues.fromByteBuffer(byteBuffer.flip());
}
@VisibleForTesting
void delete(final Account account, final long deviceId) {
DELETE_KEYS_FOR_DEVICE_TIMER.record(() -> {
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
.withValueMap(Map.of(":uuid", getPartitionKey(account.getUuid()),
":sortprefix", getSortKeyPrefix(deviceId)))
.withProjectionExpression(KEY_DEVICE_ID_KEY_ID)
.withConsistentRead(true);
private Map<String, AttributeValue> getItemFromPreKey(final UUID accountUuid, final long deviceId, final PreKey preKey) {
return Map.of(
KEY_ACCOUNT_UUID, getPartitionKey(accountUuid),
KEY_DEVICE_ID_KEY_ID, getSortKey(deviceId, preKey.getKeyId()),
KEY_PUBLIC_KEY, AttributeValues.fromString(preKey.getPublicKey()));
}
deleteItemsForAccountMatchingQuery(account, querySpec);
});
}
private void deleteItemsForAccountMatchingQuery(final Account account, final QuerySpec querySpec) {
final byte[] partitionKey = getPartitionKey(account.getUuid());
writeInBatches(table.query(querySpec), batch -> {
final TableWriteItems writeItems = new TableWriteItems(table.getTableName());
for (final Item item : batch) {
writeItems.addPrimaryKeyToDelete(new PrimaryKey(KEY_ACCOUNT_UUID, partitionKey, KEY_DEVICE_ID_KEY_ID, item.getBinary(KEY_DEVICE_ID_KEY_ID)));
}
executeTableWriteItemsUntilComplete(writeItems);
});
}
private static byte[] getPartitionKey(final UUID accountUuid) {
return UUIDUtil.toBytes(accountUuid);
}
private static byte[] getSortKey(final long deviceId, final long keyId) {
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]);
byteBuffer.putLong(deviceId);
byteBuffer.putLong(keyId);
return byteBuffer.array();
}
private static byte[] getSortKeyPrefix(final long deviceId) {
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
byteBuffer.putLong(deviceId);
return byteBuffer.array();
}
private Item getItemFromPreKey(final UUID accountUuid, final long deviceId, final PreKey preKey) {
return new Item().withBinary(KEY_ACCOUNT_UUID, getPartitionKey(accountUuid))
.withBinary(KEY_DEVICE_ID_KEY_ID, getSortKey(deviceId, preKey.getKeyId()))
.withString(KEY_PUBLIC_KEY, preKey.getPublicKey());
}
private PreKey getPreKeyFromItem(final Item item) {
final long keyId = ByteBuffer.wrap(item.getBinary(KEY_DEVICE_ID_KEY_ID)).getLong(8);
return new PreKey(keyId, item.getString(KEY_PUBLIC_KEY));
}
private PreKey getPreKeyFromItem(Map<String, AttributeValue> item) {
final long keyId = item.get(KEY_DEVICE_ID_KEY_ID).b().asByteBuffer().getLong(8);
return new PreKey(keyId, item.get(KEY_PUBLIC_KEY).s());
}
}

View File

@@ -8,17 +8,7 @@ package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import static io.micrometer.core.instrument.Metrics.timer;
import com.amazonaws.services.dynamodbv2.document.DeleteItemOutcome;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Index;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
import com.amazonaws.services.dynamodbv2.document.api.QueryApi;
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
import com.google.common.collect.ImmutableMap;
import io.micrometer.core.instrument.Timer;
import java.nio.ByteBuffer;
import java.time.Duration;
@@ -27,11 +17,22 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import org.apache.commons.lang3.StringUtils;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntity;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse;
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
import software.amazon.awssdk.services.dynamodb.model.PutRequest;
import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
public class MessagesDynamoDb extends AbstractDynamoDbStore {
@@ -60,7 +61,7 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
private final String tableName;
private final Duration timeToLive;
public MessagesDynamoDb(DynamoDB dynamoDb, String tableName, Duration timeToLive) {
public MessagesDynamoDb(DynamoDbClient dynamoDb, String tableName, Duration timeToLive) {
super(dynamoDb);
this.tableName = tableName;
@@ -76,54 +77,61 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
throw new IllegalArgumentException("Maximum batch size of " + DYNAMO_DB_MAX_BATCH_SIZE + " execeeded with " + messages.size() + " messages");
}
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
TableWriteItems items = new TableWriteItems(tableName);
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
List<WriteRequest> writeItems = new ArrayList<>();
for (MessageProtos.Envelope message : messages) {
final UUID messageUuid = UUID.fromString(message.getServerGuid());
final Item item = new Item().withBinary(KEY_PARTITION, partitionKey)
.withBinary(KEY_SORT, convertSortKey(destinationDeviceId, message.getServerTimestamp(), messageUuid))
.withBinary(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT, convertLocalIndexMessageUuidSortKey(messageUuid))
.withInt(KEY_TYPE, message.getType().getNumber())
.withLong(KEY_TIMESTAMP, message.getTimestamp())
.withLong(KEY_TTL, getTtlForMessage(message));
final ImmutableMap.Builder<String, AttributeValue> item = ImmutableMap.<String, AttributeValue>builder()
.put(KEY_PARTITION, partitionKey)
.put(KEY_SORT, convertSortKey(destinationDeviceId, message.getServerTimestamp(), messageUuid))
.put(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT, convertLocalIndexMessageUuidSortKey(messageUuid))
.put(KEY_TYPE, AttributeValues.fromInt(message.getType().getNumber()))
.put(KEY_TIMESTAMP, AttributeValues.fromLong(message.getTimestamp()))
.put(KEY_TTL, AttributeValues.fromLong(getTtlForMessage(message)));
if (message.hasRelay() && message.getRelay().length() > 0) {
item.withString(KEY_RELAY, message.getRelay());
item.put(KEY_RELAY, AttributeValues.fromString(message.getRelay()));
}
if (message.hasSource()) {
item.withString(KEY_SOURCE, message.getSource());
item.put(KEY_SOURCE, AttributeValues.fromString(message.getSource()));
}
if (message.hasSourceUuid()) {
item.withBinary(KEY_SOURCE_UUID, UUIDUtil.toBytes(UUID.fromString(message.getSourceUuid())));
item.put(KEY_SOURCE_UUID, AttributeValues.fromUUID(UUID.fromString(message.getSourceUuid())));
}
if (message.hasSourceDevice()) {
item.withInt(KEY_SOURCE_DEVICE, message.getSourceDevice());
item.put(KEY_SOURCE_DEVICE, AttributeValues.fromInt(message.getSourceDevice()));
}
if (message.hasLegacyMessage()) {
item.withBinary(KEY_MESSAGE, message.getLegacyMessage().toByteArray());
item.put(KEY_MESSAGE, AttributeValues.fromByteArray(message.getLegacyMessage().toByteArray()));
}
if (message.hasContent()) {
item.withBinary(KEY_CONTENT, message.getContent().toByteArray());
item.put(KEY_CONTENT, AttributeValues.fromByteArray(message.getContent().toByteArray()));
}
items.addItemToPut(item);
writeItems.add(WriteRequest.builder().putRequest(PutRequest.builder()
.item(item.build())
.build()).build());
}
executeTableWriteItemsUntilComplete(items);
executeTableWriteItemsUntilComplete(Map.of(tableName, writeItems));
}
public List<OutgoingMessageEntity> load(final UUID destinationAccountUuid, final long destinationDeviceId, final int requestedNumberOfMessagesToFetch) {
return loadTimer.record(() -> {
final int numberOfMessagesToFetch = Math.min(requestedNumberOfMessagesToFetch, RESULT_SET_CHUNK_SIZE);
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
final QuerySpec querySpec = new QuerySpec().withConsistentRead(true)
.withKeyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.withNameMap(Map.of("#part", KEY_PARTITION,
"#sort", KEY_SORT))
.withValueMap(Map.of(":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
.withMaxResultSize(numberOfMessagesToFetch);
final Table table = getDynamoDb().getTable(tableName);
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.consistentRead(true)
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#sort", KEY_SORT))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
.limit(numberOfMessagesToFetch)
.build();
List<OutgoingMessageEntity> messageEntities = new ArrayList<>(numberOfMessagesToFetch);
for (Item message : table.query(querySpec)) {
for (Map<String, AttributeValue> message : db().query(queryRequest).items()) {
messageEntities.add(convertItemToOutgoingMessageEntity(message));
}
return messageEntities;
@@ -136,53 +144,63 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
throw new IllegalArgumentException("must specify a source");
}
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
final QuerySpec querySpec = new QuerySpec().withProjectionExpression(KEY_SORT)
.withConsistentRead(true)
.withKeyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.withFilterExpression("#source = :source AND #timestamp = :timestamp")
.withNameMap(Map.of("#part", KEY_PARTITION,
"#sort", KEY_SORT,
"#source", KEY_SOURCE,
"#timestamp", KEY_TIMESTAMP))
.withValueMap(Map.of(":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId),
":source", source,
":timestamp", timestamp));
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.projectionExpression(KEY_SORT)
.consistentRead(true)
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.filterExpression("#source = :source AND #timestamp = :timestamp")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#sort", KEY_SORT,
"#source", KEY_SOURCE,
"#timestamp", KEY_TIMESTAMP))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId),
":source", AttributeValues.fromString(source),
":timestamp", AttributeValues.fromLong(timestamp)))
.build();
final Table table = getDynamoDb().getTable(tableName);
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(table, partitionKey, querySpec, table);
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(partitionKey, queryRequest);
});
}
public Optional<OutgoingMessageEntity> deleteMessageByDestinationAndGuid(final UUID destinationAccountUuid, final long destinationDeviceId, final UUID messageUuid) {
return deleteByGuid.record(() -> {
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
final QuerySpec querySpec = new QuerySpec().withProjectionExpression(KEY_SORT)
.withConsistentRead(true)
.withKeyConditionExpression("#part = :part AND #uuid = :uuid")
.withNameMap(Map.of("#part", KEY_PARTITION,
"#uuid", LOCAL_INDEX_MESSAGE_UUID_KEY_SORT))
.withValueMap(Map.of(":part", partitionKey,
":uuid", convertLocalIndexMessageUuidSortKey(messageUuid)));
final Table table = getDynamoDb().getTable(tableName);
final Index index = table.getIndex(LOCAL_INDEX_MESSAGE_UUID_NAME);
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(table, partitionKey, querySpec, index);
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.indexName(LOCAL_INDEX_MESSAGE_UUID_NAME)
.projectionExpression(KEY_SORT)
.consistentRead(true)
.keyConditionExpression("#part = :part AND #uuid = :uuid")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#uuid", LOCAL_INDEX_MESSAGE_UUID_KEY_SORT))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":uuid", convertLocalIndexMessageUuidSortKey(messageUuid)))
.build();
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(partitionKey, queryRequest);
});
}
@Nonnull
private Optional<OutgoingMessageEntity> deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(Table table, byte[] partitionKey, QuerySpec querySpec, QueryApi queryApi) {
private Optional<OutgoingMessageEntity> deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(AttributeValue partitionKey, QueryRequest queryRequest) {
Optional<OutgoingMessageEntity> result = Optional.empty();
for (Item item : queryApi.query(querySpec)) {
final byte[] rangeKeyValue = item.getBinary(KEY_SORT);
DeleteItemSpec deleteItemSpec = new DeleteItemSpec().withPrimaryKey(KEY_PARTITION, partitionKey, KEY_SORT, rangeKeyValue);
for (Map<String, AttributeValue> item : db().query(queryRequest).items()) {
final byte[] rangeKeyValue = item.get(KEY_SORT).b().asByteArray();
DeleteItemRequest.Builder deleteItemRequest = DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_PARTITION, partitionKey, KEY_SORT, AttributeValues.fromByteArray(rangeKeyValue)));
if (result.isEmpty()) {
deleteItemSpec.withReturnValues(ReturnValue.ALL_OLD);
deleteItemRequest.returnValues(ReturnValue.ALL_OLD);
}
final DeleteItemOutcome deleteItemOutcome = table.deleteItem(deleteItemSpec);
if (deleteItemOutcome.getItem() != null && deleteItemOutcome.getItem().hasAttribute(KEY_PARTITION)) {
result = Optional.of(convertItemToOutgoingMessageEntity(deleteItemOutcome.getItem()));
final DeleteItemResponse deleteItemResponse = db().deleteItem(deleteItemRequest.build());
if (deleteItemResponse.attributes() != null && deleteItemResponse.attributes().containsKey(KEY_PARTITION)) {
result = Optional.of(convertItemToOutgoingMessageEntity(deleteItemResponse.attributes()));
}
}
return result;
@@ -190,74 +208,88 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
public void deleteAllMessagesForAccount(final UUID destinationAccountUuid) {
deleteByAccount.record(() -> {
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
final QuerySpec querySpec = new QuerySpec().withHashKey(KEY_PARTITION, partitionKey)
.withProjectionExpression(KEY_SORT)
.withConsistentRead(true);
deleteRowsMatchingQuery(partitionKey, querySpec);
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.projectionExpression(KEY_SORT)
.consistentRead(true)
.keyConditionExpression("#part = :part")
.expressionAttributeNames(Map.of("#part", KEY_PARTITION))
.expressionAttributeValues(Map.of(":part", partitionKey))
.build();
deleteRowsMatchingQuery(partitionKey, queryRequest);
});
}
public void deleteAllMessagesForDevice(final UUID destinationAccountUuid, final long destinationDeviceId) {
deleteByDevice.record(() -> {
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.withNameMap(Map.of("#part", KEY_PARTITION,
"#sort", KEY_SORT))
.withValueMap(Map.of(":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
.withProjectionExpression(KEY_SORT)
.withConsistentRead(true);
deleteRowsMatchingQuery(partitionKey, querySpec);
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#sort", KEY_SORT))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
.projectionExpression(KEY_SORT)
.consistentRead(true)
.build();
deleteRowsMatchingQuery(partitionKey, queryRequest);
});
}
private OutgoingMessageEntity convertItemToOutgoingMessageEntity(Item message) {
final SortKey sortKey = convertSortKey(message.getBinary(KEY_SORT));
final UUID messageUuid = convertLocalIndexMessageUuidSortKey(message.getBinary(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT));
final int type = message.getInt(KEY_TYPE);
final String relay = message.getString(KEY_RELAY);
final long timestamp = message.getLong(KEY_TIMESTAMP);
final String source = message.getString(KEY_SOURCE);
final UUID sourceUuid = message.hasAttribute(KEY_SOURCE_UUID) ? convertUuidFromBytes(message.getBinary(KEY_SOURCE_UUID), "message source uuid") : null;
final int sourceDevice = message.hasAttribute(KEY_SOURCE_DEVICE) ? message.getInt(KEY_SOURCE_DEVICE) : 0;
final byte[] messageBytes = message.getBinary(KEY_MESSAGE);
final byte[] content = message.getBinary(KEY_CONTENT);
private OutgoingMessageEntity convertItemToOutgoingMessageEntity(Map<String, AttributeValue> message) {
final SortKey sortKey = convertSortKey(message.get(KEY_SORT).b().asByteArray());
final UUID messageUuid = convertLocalIndexMessageUuidSortKey(message.get(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT).b().asByteArray());
final int type = AttributeValues.getInt(message, KEY_TYPE, 0);
final String relay = AttributeValues.getString(message, KEY_RELAY, null);
final long timestamp = AttributeValues.getLong(message, KEY_TIMESTAMP, 0L);
final String source = AttributeValues.getString(message, KEY_SOURCE, null);
final UUID sourceUuid = AttributeValues.getUUID(message, KEY_SOURCE_UUID, null);
final int sourceDevice = AttributeValues.getInt(message, KEY_SOURCE_DEVICE, 0);
final byte[] messageBytes = AttributeValues.getByteArray(message, KEY_MESSAGE, null);
final byte[] content = AttributeValues.getByteArray(message, KEY_CONTENT, null);
return new OutgoingMessageEntity(-1L, false, messageUuid, type, relay, timestamp, source, sourceUuid, sourceDevice, messageBytes, content, sortKey.getServerTimestamp());
}
private void deleteRowsMatchingQuery(byte[] partitionKey, QuerySpec querySpec) {
final Table table = getDynamoDb().getTable(tableName);
writeInBatches(table.query(querySpec), (itemBatch) -> deleteItems(partitionKey, itemBatch));
private void deleteRowsMatchingQuery(AttributeValue partitionKey, QueryRequest querySpec) {
writeInBatches(db().query(querySpec).items(), (itemBatch) -> deleteItems(partitionKey, itemBatch));
}
private void deleteItems(byte[] partitionKey, List<Item> items) {
final TableWriteItems tableWriteItems = new TableWriteItems(tableName);
items.stream().map(item -> new PrimaryKey(KEY_PARTITION, partitionKey, KEY_SORT, item.getBinary(KEY_SORT))).forEach(tableWriteItems::addPrimaryKeyToDelete);
executeTableWriteItemsUntilComplete(tableWriteItems);
private void deleteItems(AttributeValue partitionKey, List<Map<String, AttributeValue>> items) {
List<WriteRequest> deletes = items.stream()
.map(item -> WriteRequest.builder()
.deleteRequest(DeleteRequest.builder().key(Map.of(
KEY_PARTITION, partitionKey,
KEY_SORT, item.get(KEY_SORT))).build())
.build())
.collect(Collectors.toList());
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
}
private long getTtlForMessage(MessageProtos.Envelope message) {
return message.getServerTimestamp() / 1000 + timeToLive.getSeconds();
}
private static byte[] convertPartitionKey(final UUID destinationAccountUuid) {
return UUIDUtil.toBytes(destinationAccountUuid);
private static AttributeValue convertPartitionKey(final UUID destinationAccountUuid) {
return AttributeValues.fromUUID(destinationAccountUuid);
}
private static byte[] convertSortKey(final long destinationDeviceId, final long serverTimestamp, final UUID messageUuid) {
private static AttributeValue convertSortKey(final long destinationDeviceId, final long serverTimestamp, final UUID messageUuid) {
ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[32]);
byteBuffer.putLong(destinationDeviceId);
byteBuffer.putLong(serverTimestamp);
byteBuffer.putLong(messageUuid.getMostSignificantBits());
byteBuffer.putLong(messageUuid.getLeastSignificantBits());
return byteBuffer.array();
return AttributeValues.fromByteBuffer(byteBuffer.flip());
}
private static byte[] convertDestinationDeviceIdToSortKeyPrefix(final long destinationDeviceId) {
private static AttributeValue convertDestinationDeviceIdToSortKeyPrefix(final long destinationDeviceId) {
ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
byteBuffer.putLong(destinationDeviceId);
return byteBuffer.array();
return AttributeValues.fromByteBuffer(byteBuffer.flip());
}
private static SortKey convertSortKey(final byte[] bytes) {
@@ -273,8 +305,8 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
return new SortKey(destinationDeviceId, serverTimestamp, new UUID(mostSigBits, leastSigBits));
}
private static byte[] convertLocalIndexMessageUuidSortKey(final UUID messageUuid) {
return UUIDUtil.toBytes(messageUuid);
private static AttributeValue convertLocalIndexMessageUuidSortKey(final UUID messageUuid) {
return AttributeValues.fromUUID(messageUuid);
}
private static UUID convertLocalIndexMessageUuidSortKey(final byte[] bytes) {

View File

@@ -1,42 +1,52 @@
package org.whispersystems.textsecuregcm.storage;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import java.util.stream.Collectors;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
public class MigrationDeletedAccounts extends AbstractDynamoDbStore {
private final Table table;
private final String tableName;
static final String KEY_UUID = "U";
public MigrationDeletedAccounts(DynamoDB dynamoDb, String tableName) {
public MigrationDeletedAccounts(DynamoDbClient dynamoDb, String tableName) {
super(dynamoDb);
table = dynamoDb.getTable(tableName);
this.tableName = tableName;
}
public void put(UUID uuid) {
table.putItem(new Item()
.withPrimaryKey(primaryKey(uuid)));
db().putItem(PutItemRequest.builder()
.tableName(tableName)
.item(primaryKey(uuid))
.build());
}
public List<UUID> getRecentlyDeletedUuids() {
final List<UUID> uuids = new ArrayList<>();
Optional<ScanResponse> firstPage = db().scanPaginator(ScanRequest.builder()
.tableName(tableName)
.build()).stream().findAny(); // get the first available response
for (Item item : table.scan(new ScanSpec()).firstPage()) {
// only process one page each time. If we have a significant backlog at the end of the migration
// we can handle it separately
uuids.add(UUIDUtil.fromByteBuffer(item.getByteBuffer(KEY_UUID)));
if (firstPage.isPresent()) {
for (Map<String, AttributeValue> item : firstPage.get().items()) {
// only process one page each time. If we have a significant backlog at the end of the migration
// we can handle it separately
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
}
}
return uuids;
@@ -45,20 +55,17 @@ public class MigrationDeletedAccounts extends AbstractDynamoDbStore {
public void delete(List<UUID> uuids) {
writeInBatches(uuids, (batch) -> {
List<WriteRequest> deletes = batch.stream().map((uuid) -> WriteRequest.builder().deleteRequest(DeleteRequest.builder()
.key(primaryKey(uuid))
.build()).build()).collect(Collectors.toList());
final TableWriteItems deleteItems = new TableWriteItems(table.getTableName());
for (UUID uuid : batch) {
deleteItems.addPrimaryKeyToDelete(primaryKey(uuid));
}
executeTableWriteItemsUntilComplete(deleteItems);
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
});
}
@VisibleForTesting
public static PrimaryKey primaryKey(UUID uuid) {
return new PrimaryKey(KEY_UUID, UUIDUtil.toBytes(uuid));
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
}
}

View File

@@ -1,43 +1,44 @@
package org.whispersystems.textsecuregcm.storage;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Page;
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
public class MigrationRetryAccounts extends AbstractDynamoDbStore {
private final Table table;
private final String tableName;
static final String KEY_UUID = "U";
public MigrationRetryAccounts(DynamoDB dynamoDb, String tableName) {
public MigrationRetryAccounts(DynamoDbClient dynamoDb, String tableName) {
super(dynamoDb);
table = dynamoDb.getTable(tableName);
this.tableName = tableName;
}
public void put(UUID uuid) {
table.putItem(new Item()
.withPrimaryKey(primaryKey(uuid)));
db().putItem(PutItemRequest.builder()
.tableName(tableName)
.item(primaryKey(uuid))
.build());
}
public List<UUID> getUuids(int max) {
final List<UUID> uuids = new ArrayList<>();
for (Page<Item, ScanOutcome> page : table.scan(new ScanSpec()).pages()) {
for (ScanResponse response : db().scanPaginator(ScanRequest.builder().tableName(tableName).build())) {
for (Item item : page) {
uuids.add(UUIDUtil.fromByteBuffer(item.getByteBuffer(KEY_UUID)));
for (Map<String, AttributeValue> item : response.items()) {
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
if (uuids.size() >= max) {
break;
@@ -53,8 +54,8 @@ public class MigrationRetryAccounts extends AbstractDynamoDbStore {
}
@VisibleForTesting
public static PrimaryKey primaryKey(UUID uuid) {
return new PrimaryKey(KEY_UUID, UUIDUtil.toBytes(uuid));
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
}
}

View File

@@ -1,666 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: PubSubMessage.proto
package org.whispersystems.textsecuregcm.storage;
public final class PubSubProtos {
private PubSubProtos() {}
public static void registerAllExtensions(
com.google.protobuf.ExtensionRegistry registry) {
}
public interface PubSubMessageOrBuilder
extends com.google.protobuf.MessageOrBuilder {
// optional .textsecure.PubSubMessage.Type type = 1;
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
boolean hasType();
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type getType();
// optional bytes content = 2;
/**
* <code>optional bytes content = 2;</code>
*/
boolean hasContent();
/**
* <code>optional bytes content = 2;</code>
*/
com.google.protobuf.ByteString getContent();
}
/**
* Protobuf type {@code textsecure.PubSubMessage}
*/
public static final class PubSubMessage extends
com.google.protobuf.GeneratedMessage
implements PubSubMessageOrBuilder {
// Use PubSubMessage.newBuilder() to construct.
private PubSubMessage(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
super(builder);
this.unknownFields = builder.getUnknownFields();
}
private PubSubMessage(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
private static final PubSubMessage defaultInstance;
public static PubSubMessage getDefaultInstance() {
return defaultInstance;
}
public PubSubMessage getDefaultInstanceForType() {
return defaultInstance;
}
private final com.google.protobuf.UnknownFieldSet unknownFields;
@java.lang.Override
public final com.google.protobuf.UnknownFieldSet
getUnknownFields() {
return this.unknownFields;
}
private PubSubMessage(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws com.google.protobuf.InvalidProtocolBufferException {
initFields();
int mutable_bitField0_ = 0;
com.google.protobuf.UnknownFieldSet.Builder unknownFields =
com.google.protobuf.UnknownFieldSet.newBuilder();
try {
boolean done = false;
while (!done) {
int tag = input.readTag();
switch (tag) {
case 0:
done = true;
break;
default: {
if (!parseUnknownField(input, unknownFields,
extensionRegistry, tag)) {
done = true;
}
break;
}
case 8: {
int rawValue = input.readEnum();
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type value = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.valueOf(rawValue);
if (value == null) {
unknownFields.mergeVarintField(1, rawValue);
} else {
bitField0_ |= 0x00000001;
type_ = value;
}
break;
}
case 18: {
bitField0_ |= 0x00000002;
content_ = input.readBytes();
break;
}
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
throw e.setUnfinishedMessage(this);
} catch (java.io.IOException e) {
throw new com.google.protobuf.InvalidProtocolBufferException(
e.getMessage()).setUnfinishedMessage(this);
} finally {
this.unknownFields = unknownFields.build();
makeExtensionsImmutable();
}
}
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_descriptor;
}
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_fieldAccessorTable
.ensureFieldAccessorsInitialized(
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.class, org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Builder.class);
}
public static com.google.protobuf.Parser<PubSubMessage> PARSER =
new com.google.protobuf.AbstractParser<PubSubMessage>() {
public PubSubMessage parsePartialFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws com.google.protobuf.InvalidProtocolBufferException {
return new PubSubMessage(input, extensionRegistry);
}
};
@java.lang.Override
public com.google.protobuf.Parser<PubSubMessage> getParserForType() {
return PARSER;
}
/**
* Protobuf enum {@code textsecure.PubSubMessage.Type}
*/
public enum Type
implements com.google.protobuf.ProtocolMessageEnum {
/**
* <code>UNKNOWN = 0;</code>
*/
UNKNOWN(0, 0),
/**
* <code>QUERY_DB = 1;</code>
*/
QUERY_DB(1, 1),
/**
* <code>DELIVER = 2;</code>
*/
DELIVER(2, 2),
/**
* <code>KEEPALIVE = 3;</code>
*/
KEEPALIVE(3, 3),
/**
* <code>CLOSE = 4;</code>
*/
CLOSE(4, 4),
/**
* <code>CONNECTED = 5;</code>
*/
CONNECTED(5, 5),
;
/**
* <code>UNKNOWN = 0;</code>
*/
public static final int UNKNOWN_VALUE = 0;
/**
* <code>QUERY_DB = 1;</code>
*/
public static final int QUERY_DB_VALUE = 1;
/**
* <code>DELIVER = 2;</code>
*/
public static final int DELIVER_VALUE = 2;
/**
* <code>KEEPALIVE = 3;</code>
*/
public static final int KEEPALIVE_VALUE = 3;
/**
* <code>CLOSE = 4;</code>
*/
public static final int CLOSE_VALUE = 4;
/**
* <code>CONNECTED = 5;</code>
*/
public static final int CONNECTED_VALUE = 5;
public final int getNumber() { return value; }
public static Type valueOf(int value) {
switch (value) {
case 0: return UNKNOWN;
case 1: return QUERY_DB;
case 2: return DELIVER;
case 3: return KEEPALIVE;
case 4: return CLOSE;
case 5: return CONNECTED;
default: return null;
}
}
public static com.google.protobuf.Internal.EnumLiteMap<Type>
internalGetValueMap() {
return internalValueMap;
}
private static com.google.protobuf.Internal.EnumLiteMap<Type>
internalValueMap =
new com.google.protobuf.Internal.EnumLiteMap<Type>() {
public Type findValueByNumber(int number) {
return Type.valueOf(number);
}
};
public final com.google.protobuf.Descriptors.EnumValueDescriptor
getValueDescriptor() {
return getDescriptor().getValues().get(index);
}
public final com.google.protobuf.Descriptors.EnumDescriptor
getDescriptorForType() {
return getDescriptor();
}
public static final com.google.protobuf.Descriptors.EnumDescriptor
getDescriptor() {
return org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.getDescriptor().getEnumTypes().get(0);
}
private static final Type[] VALUES = values();
public static Type valueOf(
com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
if (desc.getType() != getDescriptor()) {
throw new java.lang.IllegalArgumentException(
"EnumValueDescriptor is not for this type.");
}
return VALUES[desc.getIndex()];
}
private final int index;
private final int value;
private Type(int index, int value) {
this.index = index;
this.value = value;
}
// @@protoc_insertion_point(enum_scope:textsecure.PubSubMessage.Type)
}
private int bitField0_;
// optional .textsecure.PubSubMessage.Type type = 1;
public static final int TYPE_FIELD_NUMBER = 1;
private org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type type_;
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
public boolean hasType() {
return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type getType() {
return type_;
}
// optional bytes content = 2;
public static final int CONTENT_FIELD_NUMBER = 2;
private com.google.protobuf.ByteString content_;
/**
* <code>optional bytes content = 2;</code>
*/
public boolean hasContent() {
return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
* <code>optional bytes content = 2;</code>
*/
public com.google.protobuf.ByteString getContent() {
return content_;
}
private void initFields() {
type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
content_ = com.google.protobuf.ByteString.EMPTY;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
memoizedIsInitialized = 1;
return true;
}
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
if (((bitField0_ & 0x00000001) == 0x00000001)) {
output.writeEnum(1, type_.getNumber());
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
output.writeBytes(2, content_);
}
getUnknownFields().writeTo(output);
}
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
.computeEnumSize(1, type_.getNumber());
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
size += com.google.protobuf.CodedOutputStream
.computeBytesSize(2, content_);
}
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
}
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
return PARSER.parseFrom(data);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
com.google.protobuf.ByteString data,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws com.google.protobuf.InvalidProtocolBufferException {
return PARSER.parseFrom(data, extensionRegistry);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(byte[] data)
throws com.google.protobuf.InvalidProtocolBufferException {
return PARSER.parseFrom(data);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
byte[] data,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws com.google.protobuf.InvalidProtocolBufferException {
return PARSER.parseFrom(data, extensionRegistry);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(java.io.InputStream input)
throws java.io.IOException {
return PARSER.parseFrom(input);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws java.io.IOException {
return PARSER.parseFrom(input, extensionRegistry);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return PARSER.parseDelimitedFrom(input);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws java.io.IOException {
return PARSER.parseDelimitedFrom(input, extensionRegistry);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
com.google.protobuf.CodedInputStream input)
throws java.io.IOException {
return PARSER.parseFrom(input);
}
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws java.io.IOException {
return PARSER.parseFrom(input, extensionRegistry);
}
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
Builder builder = new Builder(parent);
return builder;
}
/**
* Protobuf type {@code textsecure.PubSubMessage}
*/
public static final class Builder extends
com.google.protobuf.GeneratedMessage.Builder<Builder>
implements org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessageOrBuilder {
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_descriptor;
}
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_fieldAccessorTable
.ensureFieldAccessorsInitialized(
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.class, org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Builder.class);
}
// Construct using org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
private Builder(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
}
private void maybeForceBuilderInitialization() {
if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
}
}
private static Builder create() {
return new Builder();
}
public Builder clear() {
super.clear();
type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
bitField0_ = (bitField0_ & ~0x00000001);
content_ = com.google.protobuf.ByteString.EMPTY;
bitField0_ = (bitField0_ & ~0x00000002);
return this;
}
public Builder clone() {
return create().mergeFrom(buildPartial());
}
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_descriptor;
}
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage getDefaultInstanceForType() {
return org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.getDefaultInstance();
}
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage build() {
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage result = buildPartial();
if (!result.isInitialized()) {
throw newUninitializedMessageException(result);
}
return result;
}
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage buildPartial() {
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage result = new org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage(this);
int from_bitField0_ = bitField0_;
int to_bitField0_ = 0;
if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
to_bitField0_ |= 0x00000001;
}
result.type_ = type_;
if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
to_bitField0_ |= 0x00000002;
}
result.content_ = content_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
}
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage) {
return mergeFrom((org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage)other);
} else {
super.mergeFrom(other);
return this;
}
}
public Builder mergeFrom(org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage other) {
if (other == org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.getDefaultInstance()) return this;
if (other.hasType()) {
setType(other.getType());
}
if (other.hasContent()) {
setContent(other.getContent());
}
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
public final boolean isInitialized() {
return true;
}
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws java.io.IOException {
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parsedMessage = null;
try {
parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
parsedMessage = (org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage) e.getUnfinishedMessage();
throw e;
} finally {
if (parsedMessage != null) {
mergeFrom(parsedMessage);
}
}
return this;
}
private int bitField0_;
// optional .textsecure.PubSubMessage.Type type = 1;
private org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
public boolean hasType() {
return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type getType() {
return type_;
}
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
public Builder setType(org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type value) {
if (value == null) {
throw new NullPointerException();
}
bitField0_ |= 0x00000001;
type_ = value;
onChanged();
return this;
}
/**
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
*/
public Builder clearType() {
bitField0_ = (bitField0_ & ~0x00000001);
type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
onChanged();
return this;
}
// optional bytes content = 2;
private com.google.protobuf.ByteString content_ = com.google.protobuf.ByteString.EMPTY;
/**
* <code>optional bytes content = 2;</code>
*/
public boolean hasContent() {
return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
* <code>optional bytes content = 2;</code>
*/
public com.google.protobuf.ByteString getContent() {
return content_;
}
/**
* <code>optional bytes content = 2;</code>
*/
public Builder setContent(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
bitField0_ |= 0x00000002;
content_ = value;
onChanged();
return this;
}
/**
* <code>optional bytes content = 2;</code>
*/
public Builder clearContent() {
bitField0_ = (bitField0_ & ~0x00000002);
content_ = getDefaultInstance().getContent();
onChanged();
return this;
}
// @@protoc_insertion_point(builder_scope:textsecure.PubSubMessage)
}
static {
defaultInstance = new PubSubMessage(true);
defaultInstance.initFields();
}
// @@protoc_insertion_point(class_scope:textsecure.PubSubMessage)
}
private static com.google.protobuf.Descriptors.Descriptor
internal_static_textsecure_PubSubMessage_descriptor;
private static
com.google.protobuf.GeneratedMessage.FieldAccessorTable
internal_static_textsecure_PubSubMessage_fieldAccessorTable;
public static com.google.protobuf.Descriptors.FileDescriptor
getDescriptor() {
return descriptor;
}
private static com.google.protobuf.Descriptors.FileDescriptor
descriptor;
static {
java.lang.String[] descriptorData = {
"\n\023PubSubMessage.proto\022\ntextsecure\"\247\001\n\rPu" +
"bSubMessage\022,\n\004type\030\001 \001(\0162\036.textsecure.P" +
"ubSubMessage.Type\022\017\n\007content\030\002 \001(\014\"W\n\004Ty" +
"pe\022\013\n\007UNKNOWN\020\000\022\014\n\010QUERY_DB\020\001\022\013\n\007DELIVER" +
"\020\002\022\r\n\tKEEPALIVE\020\003\022\t\n\005CLOSE\020\004\022\r\n\tCONNECTE" +
"D\020\005B8\n(org.whispersystems.textsecuregcm." +
"storageB\014PubSubProtos"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
public com.google.protobuf.ExtensionRegistry assignDescriptors(
com.google.protobuf.Descriptors.FileDescriptor root) {
descriptor = root;
internal_static_textsecure_PubSubMessage_descriptor =
getDescriptor().getMessageTypes().get(0);
internal_static_textsecure_PubSubMessage_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_textsecure_PubSubMessage_descriptor,
new java.lang.String[] { "Type", "Content", });
return null;
}
};
com.google.protobuf.Descriptors.FileDescriptor
.internalBuildGeneratedFileFrom(descriptorData,
new com.google.protobuf.Descriptors.FileDescriptor[] {
}, assigner);
}
// @@protoc_insertion_point(outer_class_scope)
}

View File

@@ -5,25 +5,23 @@
package org.whispersystems.textsecuregcm.storage;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.PutItemSpec;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import java.time.Clock;
import java.time.Duration;
import java.util.Map;
import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
/**
* Stores push challenge tokens. Users may have at most one outstanding push challenge token at a time.
*/
public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
private final Table table;
private final String tableName;
private final Clock clock;
static final String KEY_ACCOUNT_UUID = "U";
@@ -33,15 +31,15 @@ public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
private static final Map<String, String> UUID_NAME_MAP = Map.of("#uuid", KEY_ACCOUNT_UUID);
private static final Map<String, String> CHALLENGE_TOKEN_NAME_MAP = Map.of("#challenge", ATTR_CHALLENGE_TOKEN);
public PushChallengeDynamoDb(final DynamoDB dynamoDB, final String tableName) {
public PushChallengeDynamoDb(final DynamoDbClient dynamoDB, final String tableName) {
this(dynamoDB, tableName, Clock.systemUTC());
}
@VisibleForTesting
PushChallengeDynamoDb(final DynamoDB dynamoDB, final String tableName, final Clock clock) {
PushChallengeDynamoDb(final DynamoDbClient dynamoDB, final String tableName, final Clock clock) {
super(dynamoDB);
this.table = dynamoDB.getTable(tableName);
this.tableName = tableName;
this.clock = clock;
}
@@ -57,13 +55,15 @@ public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
*/
public boolean add(final UUID accountUuid, final byte[] challengeToken, final Duration ttl) {
try {
table.putItem( new PutItemSpec()
.withItem(new Item()
.withBinary(KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(accountUuid))
.withBinary(ATTR_CHALLENGE_TOKEN, challengeToken)
.withNumber(ATTR_TTL, getExpirationTimestamp(ttl)))
.withConditionExpression("attribute_not_exists(#uuid)")
.withNameMap(UUID_NAME_MAP));
db().putItem(PutItemRequest.builder()
.tableName(tableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(accountUuid),
ATTR_CHALLENGE_TOKEN, AttributeValues.fromByteArray(challengeToken),
ATTR_TTL, AttributeValues.fromLong(getExpirationTimestamp(ttl))))
.conditionExpression("attribute_not_exists(#uuid)")
.expressionAttributeNames(UUID_NAME_MAP)
.build());
return true;
} catch (final ConditionalCheckFailedException e) {
return false;
@@ -84,11 +84,13 @@ public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
*/
public boolean remove(final UUID accountUuid, final byte[] challengeToken) {
try {
table.deleteItem(new DeleteItemSpec()
.withPrimaryKey(KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(accountUuid))
.withConditionExpression("#challenge = :challenge")
.withNameMap(CHALLENGE_TOKEN_NAME_MAP)
.withValueMap(Map.of(":challenge", challengeToken)));
db().deleteItem(DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(accountUuid)))
.conditionExpression("#challenge = :challenge")
.expressionAttributeNames(CHALLENGE_TOKEN_NAME_MAP)
.expressionAttributeValues(Map.of(":challenge", AttributeValues.fromByteArray(challengeToken)))
.build());
return true;
} catch (final ConditionalCheckFailedException e) {
return false;

View File

@@ -1,13 +1,14 @@
package org.whispersystems.textsecuregcm.storage;
import com.amazonaws.services.dynamodbv2.document.DeleteItemOutcome;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import java.time.Duration;
import java.time.Instant;
import java.util.Map;
public class ReportMessageDynamoDb {
@@ -16,33 +17,30 @@ public class ReportMessageDynamoDb {
static final Duration TIME_TO_LIVE = Duration.ofDays(7);
private final Table table;
private final DynamoDbClient db;
private final String tableName;
public ReportMessageDynamoDb(final DynamoDB dynamoDB, final String tableName) {
this.table = dynamoDB.getTable(tableName);
public ReportMessageDynamoDb(final DynamoDbClient dynamoDB, final String tableName) {
this.db = dynamoDB;
this.tableName = tableName;
}
public void store(byte[] hash) {
table.putItem(buildItemForHash(hash));
}
private Item buildItemForHash(byte[] hash) {
return new Item()
.withBinary(KEY_HASH, hash)
.withLong(ATTR_TTL, Instant.now().plus(TIME_TO_LIVE).getEpochSecond());
db.putItem(PutItemRequest.builder()
.tableName(tableName)
.item(Map.of(
KEY_HASH, AttributeValues.fromByteArray(hash),
ATTR_TTL, AttributeValues.fromLong(Instant.now().plus(TIME_TO_LIVE).getEpochSecond())
))
.build());
}
public boolean remove(byte[] hash) {
final DeleteItemSpec deleteItemSpec = new DeleteItemSpec()
.withPrimaryKey(KEY_HASH, hash)
.withReturnValues(ReturnValue.ALL_OLD);
final DeleteItemOutcome outcome = table.deleteItem(deleteItemSpec);
return outcome.getItem() != null;
final DeleteItemResponse deleteItemResponse = db.deleteItem(DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_HASH, AttributeValues.fromByteArray(hash)))
.returnValues(ReturnValue.ALL_OLD)
.build());
return !deleteItemResponse.attributes().isEmpty();
}
}

View File

@@ -7,7 +7,6 @@ package org.whispersystems.textsecuregcm.util;
import static com.codahale.metrics.MetricRegistry.name;
import com.amazonaws.services.s3.model.S3Object;
import com.google.common.annotations.VisibleForTesting;
import io.dropwizard.lifecycle.Managed;
import io.micrometer.core.instrument.Counter;
@@ -53,12 +52,6 @@ public class AsnManager implements Managed {
@Override
public void start() throws Exception {
try {
handleAsnTableChanged(asnTableMonitor.getObject());
} catch (final Exception e) {
log.warn("Failed to load initial IP-to-ASN map", e);
}
asnTableMonitor.start();
}
@@ -76,10 +69,10 @@ public class AsnManager implements Managed {
}
}
private void handleAsnTableChanged(final S3Object asnTableObject) {
private void handleAsnTableChanged(final InputStream asnTableObject) {
REFRESH_TIMER.record(() -> {
try {
handleAsnTableChanged(new GZIPInputStream(asnTableObject.getObjectContent()));
handleAsnTableChangedStream(new GZIPInputStream(asnTableObject));
} catch (final IOException e) {
log.error("Retrieved object was not a gzip archive", e);
}
@@ -87,7 +80,7 @@ public class AsnManager implements Managed {
}
@VisibleForTesting
void handleAsnTableChanged(final InputStream inputStream) {
void handleAsnTableChangedStream(final InputStream inputStream) {
try (final InputStreamReader reader = new InputStreamReader(inputStream)) {
asnTable.set(new AsnTable(reader));
} catch (final Exception e) {

View File

@@ -0,0 +1,89 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.util;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
/** AwsAV provides static helper methods for working with AWS AttributeValues. */
public class AttributeValues {
public static AttributeValue fromString(String value) {
return AttributeValue.builder().s(value).build();
}
public static AttributeValue fromLong(long value) {
return AttributeValue.builder().n(Long.toString(value)).build();
}
public static AttributeValue fromInt(int value) {
return AttributeValue.builder().n(Integer.toString(value)).build();
}
public static AttributeValue fromByteArray(byte[] value) {
return AttributeValues.fromSdkBytes(SdkBytes.fromByteArray(value));
}
public static AttributeValue fromByteBuffer(ByteBuffer value) {
return AttributeValues.fromSdkBytes(SdkBytes.fromByteBuffer(value));
}
public static AttributeValue fromUUID(UUID uuid) {
return AttributeValues.fromSdkBytes(SdkBytes.fromByteArrayUnsafe(UUIDUtil.toBytes(uuid)));
}
public static AttributeValue fromSdkBytes(SdkBytes value) {
return AttributeValue.builder().b(value).build();
}
private static int toInt(AttributeValue av) {
return Integer.parseInt(av.n());
}
private static long toLong(AttributeValue av) {
return Long.parseLong(av.n());
}
private static UUID toUUID(AttributeValue av) {
return UUIDUtil.fromBytes(av.b().asByteArrayUnsafe()); // We're guaranteed not to modify the byte array
}
private static byte[] toByteArray(AttributeValue av) {
return av.b().asByteArray();
}
private static String toString(AttributeValue av) {
return av.s();
}
public static Optional<AttributeValue> get(Map<String, AttributeValue> item, String key) {
return Optional.ofNullable(item.get(key));
}
public static int getInt(Map<String, AttributeValue> item, String key, int defaultValue) {
return AttributeValues.get(item, key).map(AttributeValues::toInt).orElse(defaultValue);
}
public static String getString(Map<String, AttributeValue> item, String key, String defaultValue) {
return AttributeValues.get(item, key).map(AttributeValues::toString).orElse(defaultValue);
}
public static long getLong(Map<String, AttributeValue> item, String key, long defaultValue) {
return AttributeValues.get(item, key).map(AttributeValues::toLong).orElse(defaultValue);
}
public static byte[] getByteArray(Map<String, AttributeValue> item, String key, byte[] defaultValue) {
return AttributeValues.get(item, key).map(AttributeValues::toByteArray).orElse(defaultValue);
}
public static UUID getUUID(Map<String, AttributeValue> item, String key, UUID defaultValue) {
return AttributeValues.get(item, key).map(AttributeValues::toUUID).orElse(defaultValue);
}
}

View File

@@ -0,0 +1,41 @@
package org.whispersystems.textsecuregcm.util;
import org.whispersystems.textsecuregcm.configuration.DynamoDbConfiguration;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClientBuilder;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import java.util.concurrent.Executor;
public class DynamoDbFromConfig {
private static ClientOverrideConfiguration clientOverrideConfiguration(DynamoDbConfiguration config) {
return ClientOverrideConfiguration.builder()
.apiCallTimeout(config.getClientExecutionTimeout())
.apiCallAttemptTimeout(config.getClientRequestTimeout())
.build();
}
public static DynamoDbClient client(DynamoDbConfiguration config, AwsCredentialsProvider credentialsProvider) {
return DynamoDbClient.builder()
.region(Region.of(config.getRegion()))
.credentialsProvider(credentialsProvider)
.overrideConfiguration(clientOverrideConfiguration(config))
.build();
}
public static DynamoDbAsyncClient asyncClient(DynamoDbConfiguration config, AwsCredentialsProvider credentialsProvider, Executor executor) {
DynamoDbAsyncClientBuilder builder = DynamoDbAsyncClient.builder()
.region(Region.of(config.getRegion()))
.credentialsProvider(credentialsProvider)
.overrideConfiguration(clientOverrideConfiguration(config));
if (executor != null) {
builder.asyncConfiguration(ClientAsyncConfiguration.builder()
.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR,
executor)
.build());
}
return builder.build();
}
}

View File

@@ -5,14 +5,10 @@
package org.whispersystems.textsecuregcm.util;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.google.common.annotations.VisibleForTesting;
import io.dropwizard.lifecycle.Managed;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
@@ -22,6 +18,14 @@ import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
/**
* An S3 object monitor watches a specific object in an S3 bucket and notifies a listener if that object changes.
@@ -36,11 +40,11 @@ public class S3ObjectMonitor implements Managed {
private final Duration refreshInterval;
private ScheduledFuture<?> refreshFuture;
private final Consumer<S3Object> changeListener;
private final Consumer<InputStream> changeListener;
private final AtomicReference<String> lastETag = new AtomicReference<>();
private final AmazonS3 s3Client;
private final S3Client s3Client;
private static final Logger log = LoggerFactory.getLogger(S3ObjectMonitor.class);
@@ -51,11 +55,11 @@ public class S3ObjectMonitor implements Managed {
final long maxObjectSize,
final ScheduledExecutorService refreshExecutorService,
final Duration refreshInterval,
final Consumer<S3Object> changeListener) {
final Consumer<InputStream> changeListener) {
this(AmazonS3ClientBuilder.standard()
.withCredentials(InstanceProfileCredentialsProvider.getInstance())
.withRegion(s3Region)
this(S3Client.builder()
.region(Region.of(s3Region))
.credentialsProvider(InstanceProfileCredentialsProvider.create())
.build(),
s3Bucket,
objectKey,
@@ -67,13 +71,13 @@ public class S3ObjectMonitor implements Managed {
@VisibleForTesting
S3ObjectMonitor(
final AmazonS3 s3Client,
final S3Client s3Client,
final String s3Bucket,
final String objectKey,
final long maxObjectSize,
final ScheduledExecutorService refreshExecutorService,
final Duration refreshInterval,
final Consumer<S3Object> changeListener) {
final Consumer<InputStream> changeListener) {
this.s3Client = s3Client;
this.s3Bucket = s3Bucket;
@@ -92,8 +96,13 @@ public class S3ObjectMonitor implements Managed {
throw new RuntimeException("S3 object manager already started");
}
// Run the first request immediately/blocking, then start subsequent calls.
log.info("Initial request for s3://{}/{}", s3Bucket, objectKey);
refresh();
refreshFuture = refreshExecutorService
.scheduleAtFixedRate(this::refresh, refreshInterval.toMillis(), refreshInterval.toMillis(), TimeUnit.MILLISECONDS);
.scheduleAtFixedRate(this::refresh, refreshInterval.toMillis(), refreshInterval.toMillis(),
TimeUnit.MILLISECONDS);
}
@Override
@@ -106,21 +115,24 @@ public class S3ObjectMonitor implements Managed {
/**
* Immediately returns the monitored S3 object regardless of whether it has changed since it was last retrieved.
*
* @return the current version of the monitored S3 object
*
* @return the current version of the monitored S3 object. Caller should close() this upon completion.
* @throws IOException if the retrieved S3 object is larger than the configured maximum size
*/
public S3Object getObject() throws IOException {
final S3Object s3Object = s3Client.getObject(s3Bucket, objectKey);
@VisibleForTesting
ResponseInputStream<GetObjectResponse> getObject() throws IOException {
ResponseInputStream<GetObjectResponse> response = s3Client.getObject(GetObjectRequest.builder()
.key(objectKey)
.bucket(s3Bucket)
.build());
lastETag.set(s3Object.getObjectMetadata().getETag());
lastETag.set(response.response().eTag());
if (s3Object.getObjectMetadata().getContentLength() <= maxObjectSize) {
return s3Object;
if (response.response().contentLength() <= maxObjectSize) {
return response;
} else {
log.warn("Object at s3://{}/{} has a size of {} bytes, which exceeds the maximum allowed size of {} bytes",
s3Bucket, objectKey, s3Object.getObjectMetadata().getContentLength(), maxObjectSize);
s3Bucket, objectKey, response.response().contentLength(), maxObjectSize);
response.abort();
throw new IOException("S3 object too large");
}
}
@@ -132,18 +144,20 @@ public class S3ObjectMonitor implements Managed {
@VisibleForTesting
void refresh() {
try {
final ObjectMetadata objectMetadata = s3Client.getObjectMetadata(s3Bucket, objectKey);
final HeadObjectResponse objectMetadata = s3Client.headObject(HeadObjectRequest.builder()
.bucket(s3Bucket)
.key(objectKey)
.build());
final String initialETag = lastETag.get();
final String refreshedETag = objectMetadata.getETag();
final String refreshedETag = objectMetadata.eTag();
if (!StringUtils.equals(initialETag, refreshedETag) && lastETag.compareAndSet(initialETag, refreshedETag)) {
final S3Object s3Object = getObject();
log.info("Object at s3://{}/{} has changed; new eTag is {} and object size is {} bytes",
s3Bucket, objectKey, s3Object.getObjectMetadata().getETag(), s3Object.getObjectMetadata().getContentLength());
changeListener.accept(s3Object);
try (final ResponseInputStream<GetObjectResponse> response = getObject()) {
log.info("Object at s3://{}/{} has changed; new eTag is {} and object size is {} bytes",
s3Bucket, objectKey, response.response().eTag(), response.response().contentLength());
changeListener.accept(response);
}
}
} catch (final Exception e) {
log.warn("Failed to refresh monitored object", e);

View File

@@ -7,14 +7,12 @@ package org.whispersystems.textsecuregcm.util;
import static com.codahale.metrics.MetricRegistry.name;
import com.amazonaws.services.s3.model.S3Object;
import com.google.common.annotations.VisibleForTesting;
import io.dropwizard.lifecycle.Managed;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Collections;
@@ -58,12 +56,6 @@ public class TorExitNodeManager implements Managed {
@Override
public synchronized void start() {
try {
handleExitListChanged(exitListMonitor.getObject());
} catch (final Exception e) {
log.warn("Failed to load initial Tor exit node list", e);
}
exitListMonitor.start();
}
@@ -76,12 +68,12 @@ public class TorExitNodeManager implements Managed {
return exitNodeAddresses.get().contains(address);
}
private void handleExitListChanged(final S3Object exitList) {
REFRESH_TIMER.record(() -> handleExitListChanged(exitList.getObjectContent()));
private void handleExitListChanged(final InputStream exitList) {
REFRESH_TIMER.record(() -> handleExitListChangedStream(exitList));
}
@VisibleForTesting
void handleExitListChanged(final InputStream inputStream) {
void handleExitListChangedStream(final InputStream inputStream) {
try (final BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
exitNodeAddresses.set(reader.lines().collect(Collectors.toSet()));
} catch (final Exception e) {

View File

@@ -5,6 +5,7 @@
package org.whispersystems.textsecuregcm.util;
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.util.UUID;
@@ -26,12 +27,15 @@ public class UUIDUtil {
}
public static UUID fromByteBuffer(final ByteBuffer byteBuffer) {
if (byteBuffer.array().length != 16) {
throw new IllegalArgumentException("unexpected byte array length; was " + byteBuffer.array().length + " but expected 16");
try {
final long mostSigBits = byteBuffer.getLong();
final long leastSigBits = byteBuffer.getLong();
if (byteBuffer.hasRemaining()) {
throw new IllegalArgumentException("unexpected byte array length; was greater than 16");
}
return new UUID(mostSigBits, leastSigBits);
} catch (BufferUnderflowException e) {
throw new IllegalArgumentException("unexpected byte array length; was less than 16");
}
final long mostSigBits = byteBuffer.getLong();
final long leastSigBits = byteBuffer.getLong();
return new UUID(mostSigBits, leastSigBits);
}
}

View File

@@ -171,7 +171,7 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
messagesManager.delete(account.getUuid(), device.getId(), storedMessageInfo.get().getGuid());
}
if (message.getType() != Envelope.Type.RECEIPT) {
if (message.getType() != Envelope.Type.SERVER_DELIVERY_RECEIPT) {
recordMessageDeliveryDuration(message.getTimestamp(), device);
sendDeliveryReceiptFor(message);
}

View File

@@ -0,0 +1,40 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.workers;
import io.dropwizard.cli.Command;
import io.dropwizard.setup.Bootstrap;
import java.nio.file.Files;
import java.nio.file.Path;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
public class CheckDynamicConfigurationCommand extends Command {
public CheckDynamicConfigurationCommand() {
super("check-dynamic-config", "Check validity of a dynamic configuration file");
}
@Override
public void configure(final Subparser subparser) {
subparser.addArgument("file")
.type(String.class)
.required(true)
.help("Dynamic configuration file to check");
}
@Override
public void run(final Bootstrap<?> bootstrap, final Namespace namespace) throws Exception {
final Path path = Path.of(namespace.getString("file"));
if (DynamicConfigurationManager.parseConfiguration(Files.readString(path)).isPresent()) {
System.out.println("Dynamic configuration file at " + path + " is valid");
} else {
System.err.println("Dynamic configuration file at " + path + " is not valid");
}
}
}

View File

@@ -9,11 +9,6 @@ import static com.codahale.metrics.MetricRegistry.name;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.fasterxml.jackson.databind.DeserializationFeature;
import io.dropwizard.Application;
import io.dropwizard.cli.EnvironmentCommand;
@@ -59,6 +54,9 @@ import org.whispersystems.textsecuregcm.storage.ReportMessageManager;
import org.whispersystems.textsecuregcm.storage.ReservedUsernames;
import org.whispersystems.textsecuregcm.storage.Usernames;
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfiguration> {
@@ -100,64 +98,20 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi, configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
ClientResources redisClusterClientResources = ClientResources.builder().build();
AmazonDynamoDBClientBuilder clientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(configuration.getMessageDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) configuration.getMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
AmazonDynamoDBClientBuilder keysDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(configuration.getKeysDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getKeysDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) configuration.getKeysDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
AmazonDynamoDBClientBuilder accountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(configuration.getAccountsDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) configuration.getAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
new LinkedBlockingDeque<>());
AmazonDynamoDBAsyncClientBuilder accountsDynamoDbAsyncClientBuilder = AmazonDynamoDBAsyncClientBuilder
.standard()
.withRegion(accountsDynamoDbClientBuilder.getRegion())
.withClientConfiguration(accountsDynamoDbClientBuilder.getClientConfiguration())
.withCredentials(accountsDynamoDbClientBuilder.getCredentials())
.withExecutorFactory(() -> accountsDynamoDbMigrationThreadPool);
AmazonDynamoDBClientBuilder migrationDeletedAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
AmazonDynamoDBClientBuilder migrationRetryAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(configuration.getMigrationRetryAccountsDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getMigrationRetryAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) configuration.getMigrationRetryAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
AmazonDynamoDBClientBuilder reportMessageDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
.standard()
.withRegion(configuration.getReportMessageDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getReportMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
.withRequestTimeout((int) configuration.getReportMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
DynamoDB messageDynamoDb = new DynamoDB(clientBuilder.build());
DynamoDB preKeysDynamoDb = new DynamoDB(keysDynamoDbClientBuilder.build());
DynamoDB reportMessagesDynamoDb = new DynamoDB(reportMessageDynamoDbClientBuilder.build());
AmazonDynamoDB accountsDynamoDbClient = accountsDynamoDbClientBuilder.build();
AmazonDynamoDBAsync accountsDynamoDbAsyncClient = accountsDynamoDbAsyncClientBuilder.build();
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig.client(configuration.getReportMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient messageDynamoDb = DynamoDbFromConfig.client(configuration.getMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient preKeysDynamoDb = DynamoDbFromConfig.client(configuration.getKeysDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(configuration.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(configuration.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
accountsDynamoDbMigrationThreadPool);
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", configuration.getCacheClusterConfiguration(), redisClusterClientResources);
@@ -173,14 +127,16 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(dynamicConfigurationManager);
DynamoDB migrationDeletedAccountsDynamoDb = new DynamoDB(migrationDeletedAccountsDynamoDbClientBuilder.build());
DynamoDB migrationRetryAccountsDynamoDb = new DynamoDB(migrationRetryAccountsDynamoDbClientBuilder.build());
DynamoDbClient migrationDeletedAccountsDynamoDb = DynamoDbFromConfig.client(configuration.getMigrationDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(configuration.getMigrationRetryAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(migrationDeletedAccountsDynamoDb, configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, configuration.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
Accounts accounts = new Accounts(accountDatabase);
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, new DynamoDB(accountsDynamoDbClient), configuration.getAccountsDynamoDbConfiguration().getTableName(), configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, configuration.getAccountsDynamoDbConfiguration().getTableName(), configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
Usernames usernames = new Usernames(accountDatabase);
Profiles profiles = new Profiles(accountDatabase);
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);

View File

@@ -1,43 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.workers;
import io.dropwizard.cli.ConfiguredCommand;
import io.dropwizard.setup.Bootstrap;
import io.lettuce.core.resource.ClientResources;
import net.sourceforge.argparse4j.inf.Namespace;
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import java.util.List;
public class GetRedisCommandStatsCommand extends ConfiguredCommand<WhisperServerConfiguration> {
public GetRedisCommandStatsCommand() {
super("rediscommandstats", "Dump Redis command stats");
}
@Override
protected void run(final Bootstrap<WhisperServerConfiguration> bootstrap, final Namespace namespace, final WhisperServerConfiguration config) {
final ClientResources redisClusterClientResources = ClientResources.builder().build();
final FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", config.getCacheClusterConfiguration(), redisClusterClientResources);
final FaultTolerantRedisCluster messagesCacheCluster = new FaultTolerantRedisCluster("messages_cluster", config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
final FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", config.getMetricsClusterConfiguration(), redisClusterClientResources);
for (final FaultTolerantRedisCluster cluster : List.of(cacheCluster, messagesCacheCluster, metricsCluster)) {
cluster.useCluster(connection -> connection.sync()
.upstream()
.commands()
.info("commandstats")
.asMap()
.forEach((node, commandStats) -> {
System.out.format("# %s - %s\n\n", cluster.getName(), node.getUri());
System.out.println(commandStats);
}));
}
}
}

View File

@@ -1,62 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.workers;
import io.dropwizard.cli.ConfiguredCommand;
import io.dropwizard.setup.Bootstrap;
import io.lettuce.core.resource.ClientResources;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class GetRedisSlowlogCommand extends ConfiguredCommand<WhisperServerConfiguration> {
public GetRedisSlowlogCommand() {
super("redisslowlog", "Dump a JSON blob describing slow Redis operations");
}
@Override
public void configure(final Subparser subparser) {
super.configure(subparser);
subparser.addArgument("-n", "--entries")
.dest("entries")
.type(Integer.class)
.required(false)
.setDefault(128)
.help("The maximum number of SLOWLOG entries to retrieve per cluster node");
}
@Override
protected void run(final Bootstrap<WhisperServerConfiguration> bootstrap, final Namespace namespace, final WhisperServerConfiguration config) throws Exception {
final int entries = namespace.getInt("entries");
final ClientResources redisClusterClientResources = ClientResources.builder().build();
final FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", config.getCacheClusterConfiguration(), redisClusterClientResources);
final FaultTolerantRedisCluster messagesCacheCluster = new FaultTolerantRedisCluster("messages_cluster", config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
final FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", config.getMetricsClusterConfiguration(), redisClusterClientResources);
final Map<String, List<Object>> slowlogsByUri = new HashMap<>();
for (final FaultTolerantRedisCluster cluster : List.of(cacheCluster, messagesCacheCluster, metricsCluster)) {
cluster.useCluster(connection -> connection.sync()
.upstream()
.commands()
.slowlogGet(entries)
.asMap()
.forEach((node, slowlogs) -> slowlogsByUri.put(node.getUri().toString(), slowlogs)));
}
SystemMapper.getMapper().writeValue(System.out, slowlogsByUri);
}
}

View File

@@ -0,0 +1,28 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.workers;
import io.dropwizard.cli.Command;
import io.dropwizard.setup.Bootstrap;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import org.whispersystems.textsecuregcm.WhisperServerVersion;
public class ServerVersionCommand extends Command {
public ServerVersionCommand() {
super("version", "Print the version of the service");
}
@Override
public void configure(final Subparser subparser) {
}
@Override
public void run(final Bootstrap<?> bootstrap, final Namespace namespace) throws Exception {
System.out.println(WhisperServerVersion.getServerVersion());
}
}

View File

@@ -25,8 +25,10 @@ message Envelope {
CIPHERTEXT = 1;
KEY_EXCHANGE = 2;
PREKEY_BUNDLE = 3;
RECEIPT = 5;
SERVER_DELIVERY_RECEIPT = 5;
UNIDENTIFIED_SENDER = 6;
reserved 7;
PLAINTEXT_CONTENT = 8; // for decryption error receipts
}
optional Type type = 1;

View File

@@ -31,8 +31,8 @@ class DynamicConfigurationTest {
void testParseExperimentConfig() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertFalse(emptyConfig.getExperimentEnrollmentConfiguration("test").isPresent());
}
@@ -51,8 +51,8 @@ class DynamicConfigurationTest {
" enrolledUuids:\n" +
" - 71618739-114c-4b1f-bb0d-6478a44eb600";
final DynamicConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(experimentConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration config =
DynamicConfigurationManager.parseConfiguration(experimentConfigYaml).orElseThrow();
assertFalse(config.getExperimentEnrollmentConfiguration("unconfigured").isPresent());
@@ -79,8 +79,8 @@ class DynamicConfigurationTest {
void testParsePreRegistrationExperiments() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertFalse(emptyConfig.getPreRegistrationEnrollmentConfiguration("test").isPresent());
}
@@ -105,8 +105,8 @@ class DynamicConfigurationTest {
" excludedCountryCodes:\n" +
" - 47";
final DynamicConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(experimentConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration config =
DynamicConfigurationManager.parseConfiguration(experimentConfigYaml).orElseThrow();
assertFalse(config.getPreRegistrationEnrollmentConfiguration("unconfigured").isPresent());
@@ -152,14 +152,14 @@ class DynamicConfigurationTest {
void testParseRemoteDeprecationConfig() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertNotNull(emptyConfig.getRemoteDeprecationConfiguration());
}
{
final String experimentConfigYaml =
final String remoteDeprecationConfig =
"remoteDeprecation:\n" +
" minimumVersions:\n" +
" IOS: 1.2.3\n" +
@@ -172,8 +172,9 @@ class DynamicConfigurationTest {
" DESKTOP:\n" +
" - 1.4.0-beta.2";
final DynamicConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(experimentConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration config =
DynamicConfigurationManager.parseConfiguration(remoteDeprecationConfig).orElseThrow();
final DynamicRemoteDeprecationConfiguration remoteDeprecationConfiguration = config
.getRemoteDeprecationConfiguration();
@@ -191,8 +192,8 @@ class DynamicConfigurationTest {
void testParseMessageRateConfiguration() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertFalse(emptyConfig.getMessageRateConfiguration().isEnforceUnsealedSenderRateLimit());
}
@@ -202,8 +203,8 @@ class DynamicConfigurationTest {
"messageRate:\n" +
" enforceUnsealedSenderRateLimit: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(messageRateConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(messageRateConfigYaml).orElseThrow();
assertTrue(emptyConfig.getMessageRateConfiguration().isEnforceUnsealedSenderRateLimit());
}
@@ -213,19 +214,19 @@ class DynamicConfigurationTest {
void testParseFeatureFlags() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertTrue(emptyConfig.getActiveFeatureFlags().isEmpty());
}
{
final String emptyConfigYaml =
final String featureFlagYaml =
"featureFlags:\n"
+ " - testFlag";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(featureFlagYaml).orElseThrow();
assertTrue(emptyConfig.getActiveFeatureFlags().contains("testFlag"));
}
@@ -235,22 +236,22 @@ class DynamicConfigurationTest {
void testParseTwilioConfiguration() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertTrue(emptyConfig.getTwilioConfiguration().getNumbers().isEmpty());
}
{
final String emptyConfigYaml =
final String twilioConfigYaml =
"twilio:\n"
+ " numbers:\n"
+ " - 2135551212\n"
+ " - 2135551313";
final DynamicTwilioConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class)
.getTwilioConfiguration();
final DynamicTwilioConfiguration config =
DynamicConfigurationManager.parseConfiguration(twilioConfigYaml).orElseThrow()
.getTwilioConfiguration();
assertEquals(List.of("2135551212", "2135551313"), config.getNumbers());
}
@@ -260,8 +261,8 @@ class DynamicConfigurationTest {
void testParsePaymentsConfiguration() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertTrue(emptyConfig.getPaymentsConfiguration().getAllowedCountryCodes().isEmpty());
}
@@ -272,9 +273,9 @@ class DynamicConfigurationTest {
+ " allowedCountryCodes:\n"
+ " - 44";
final DynamicPaymentsConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(paymentsConfigYaml, DynamicConfiguration.class)
.getPaymentsConfiguration();
final DynamicPaymentsConfiguration config =
DynamicConfigurationManager.parseConfiguration(paymentsConfigYaml).orElseThrow()
.getPaymentsConfiguration();
assertEquals(Set.of("44"), config.getAllowedCountryCodes());
}
@@ -284,8 +285,8 @@ class DynamicConfigurationTest {
void testParseSignupCaptchaConfiguration() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertTrue(emptyConfig.getSignupCaptchaConfiguration().getCountryCodes().isEmpty());
}
@@ -296,9 +297,9 @@ class DynamicConfigurationTest {
+ " countryCodes:\n"
+ " - 1";
final DynamicSignupCaptchaConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(signupCaptchaConfig, DynamicConfiguration.class)
.getSignupCaptchaConfiguration();
final DynamicSignupCaptchaConfiguration config =
DynamicConfigurationManager.parseConfiguration(signupCaptchaConfig).orElseThrow()
.getSignupCaptchaConfiguration();
assertEquals(Set.of("1"), config.getCountryCodes());
}
@@ -308,8 +309,8 @@ class DynamicConfigurationTest {
void testParseAccountsDynamoDbMigrationConfiguration() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isBackgroundMigrationEnabled());
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled());
@@ -326,9 +327,9 @@ class DynamicConfigurationTest {
+ " readEnabled: true\n"
+ " writeEnabled: true";
final DynamicAccountsDynamoDbMigrationConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(accountsDynamoDbMigrationConfig, DynamicConfiguration.class)
.getAccountsDynamoDbMigrationConfiguration();
final DynamicAccountsDynamoDbMigrationConfiguration config =
DynamicConfigurationManager.parseConfiguration(accountsDynamoDbMigrationConfig).orElseThrow()
.getAccountsDynamoDbMigrationConfiguration();
assertTrue(config.isBackgroundMigrationEnabled());
assertEquals(100, config.getBackgroundMigrationExecutorThreads());
@@ -342,8 +343,8 @@ class DynamicConfigurationTest {
void testParseLimits() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER.readValue(
emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertThat(emptyConfig.getLimits().getUnsealedSenderNumber().getMaxCardinality()).isEqualTo(100);
assertThat(emptyConfig.getLimits().getUnsealedSenderNumber().getTtl()).isEqualTo(Duration.ofDays(1));
@@ -355,9 +356,10 @@ class DynamicConfigurationTest {
+ " unsealedSenderNumber:\n"
+ " maxCardinality: 99\n"
+ " ttl: PT23H";
final CardinalityRateLimitConfiguration unsealedSenderNumber = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(limitsConfig, DynamicConfiguration.class)
.getLimits().getUnsealedSenderNumber();
final CardinalityRateLimitConfiguration unsealedSenderNumber =
DynamicConfigurationManager.parseConfiguration(limitsConfig).orElseThrow()
.getLimits().getUnsealedSenderNumber();
assertThat(unsealedSenderNumber.getMaxCardinality()).isEqualTo(99);
assertThat(unsealedSenderNumber.getTtl()).isEqualTo(Duration.ofHours(23));
@@ -368,8 +370,8 @@ class DynamicConfigurationTest {
void testParseRateLimitReset() throws JsonProcessingException {
{
final String emptyConfigYaml = "test: true";
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER.readValue(
emptyConfigYaml, DynamicConfiguration.class);
final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertThat(emptyConfig.getRateLimitChallengeConfiguration().getClientSupportedVersions()).isEmpty();
assertThat(emptyConfig.getRateLimitChallengeConfiguration().isPreKeyLimitEnforced()).isFalse();
@@ -384,9 +386,11 @@ class DynamicConfigurationTest {
+ " IOS: 5.1.0\n"
+ " ANDROID: 5.2.0\n"
+ " DESKTOP: 5.0.0";
DynamicRateLimitChallengeConfiguration rateLimitChallengeConfiguration = DynamicConfigurationManager.OBJECT_MAPPER
.readValue(rateLimitChallengeConfig, DynamicConfiguration.class)
.getRateLimitChallengeConfiguration();
DynamicRateLimitChallengeConfiguration rateLimitChallengeConfiguration =
DynamicConfigurationManager.parseConfiguration(rateLimitChallengeConfig).orElseThrow()
.getRateLimitChallengeConfiguration();
final Map<ClientPlatform, Semver> clientSupportedVersions = rateLimitChallengeConfiguration.getClientSupportedVersions();
assertThat(clientSupportedVersions.get(ClientPlatform.IOS)).isEqualTo(new Semver("5.1.0"));

View File

@@ -11,26 +11,12 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Page;
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.KeyType;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
@@ -46,7 +32,22 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
class AccountsDynamoDbTest {
@@ -59,49 +60,75 @@ class AccountsDynamoDbTest {
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName(ACCOUNTS_TABLE_NAME)
.hashKey(AccountsDynamoDb.KEY_ACCOUNT_UUID)
.attributeDefinition(new AttributeDefinition(AccountsDynamoDb.KEY_ACCOUNT_UUID, ScalarAttributeType.B))
.attributeDefinition(AttributeDefinition.builder()
.attributeName(AccountsDynamoDb.KEY_ACCOUNT_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.build();
private AccountsDynamoDb accountsDynamoDb;
private Table migrationDeletedAccountsTable;
private Table migrationRetryAccountsTable;
@BeforeEach
void setupAccountsDao() {
CreateTableRequest createNumbersTableRequest = CreateTableRequest.builder()
.tableName(NUMBERS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName(AccountsDynamoDb.ATTR_ACCOUNT_E164)
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName(AccountsDynamoDb.ATTR_ACCOUNT_E164)
.attributeType(ScalarAttributeType.S)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
CreateTableRequest createNumbersTableRequest = new CreateTableRequest()
.withTableName(NUMBERS_TABLE_NAME)
.withKeySchema(new KeySchemaElement(AccountsDynamoDb.ATTR_ACCOUNT_E164, KeyType.HASH))
.withAttributeDefinitions(new AttributeDefinition(AccountsDynamoDb.ATTR_ACCOUNT_E164, ScalarAttributeType.S))
.withProvisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT);
dynamoDbExtension.getDynamoDbClient().createTable(createNumbersTableRequest);
final Table numbersTable = dynamoDbExtension.getDynamoDB().createTable(createNumbersTableRequest);
final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder()
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName(MigrationDeletedAccounts.KEY_UUID)
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName(MigrationDeletedAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
final CreateTableRequest createMigrationDeletedAccountsTableRequest = new CreateTableRequest()
.withTableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.withKeySchema(new KeySchemaElement(MigrationDeletedAccounts.KEY_UUID, KeyType.HASH))
.withAttributeDefinitions(new AttributeDefinition(MigrationDeletedAccounts.KEY_UUID, ScalarAttributeType.B))
.withProvisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT);
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest);
migrationDeletedAccountsTable = dynamoDbExtension.getDynamoDB().createTable(createMigrationDeletedAccountsTableRequest);
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(
dynamoDbExtension.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME);
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDB(),
migrationDeletedAccountsTable.getTableName());
final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder()
.tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName(MigrationRetryAccounts.KEY_UUID)
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName(MigrationRetryAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
final CreateTableRequest createMigrationRetryAccountsTableRequest = new CreateTableRequest()
.withTableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
.withKeySchema(new KeySchemaElement(MigrationRetryAccounts.KEY_UUID, KeyType.HASH))
.withAttributeDefinitions(new AttributeDefinition(MigrationRetryAccounts.KEY_UUID, ScalarAttributeType.B))
.withProvisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT);
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest);
migrationRetryAccountsTable = dynamoDbExtension.getDynamoDB().createTable(createMigrationRetryAccountsTableRequest);
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts((dynamoDbExtension.getDynamoDbClient()),
MIGRATION_RETRY_ACCOUNTS_TABLE_NAME);
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts((dynamoDbExtension.getDynamoDB()),
migrationRetryAccountsTable.getTableName());
this.accountsDynamoDb = new AccountsDynamoDb(dynamoDbExtension.getClient(), dynamoDbExtension.getAsyncClient(), new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), dynamoDbExtension.getDynamoDB(), dynamoDbExtension.getTableName(), numbersTable.getTableName(),
migrationDeletedAccounts, migrationRetryAccounts);
this.accountsDynamoDb = new AccountsDynamoDb(
dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getDynamoDbAsyncClient(),
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
dynamoDbExtension.getTableName(),
NUMBERS_TABLE_NAME,
migrationDeletedAccounts,
migrationRetryAccounts);
}
@Test
@@ -262,8 +289,12 @@ class AccountsDynamoDbTest {
verifyRecentlyDeletedAccountsTableItemCount(1);
assertThat(migrationDeletedAccountsTable
.getItem(MigrationDeletedAccounts.primaryKey(deletedAccount.getUuid()))).isNotNull();
Map<String, AttributeValue> primaryKey = MigrationDeletedAccounts.primaryKey(deletedAccount.getUuid());
assertThat(dynamoDbExtension.getDynamoDbClient().getItem(GetItemRequest.builder()
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.key(Map.of(MigrationDeletedAccounts.KEY_UUID, primaryKey.get(MigrationDeletedAccounts.KEY_UUID)))
.build()))
.isNotNull();
accountsDynamoDb.deleteRecentlyDeletedUuids();
@@ -273,8 +304,10 @@ class AccountsDynamoDbTest {
private void verifyRecentlyDeletedAccountsTableItemCount(int expectedItemCount) {
int totalItems = 0;
for (Page<Item, ScanOutcome> page : migrationDeletedAccountsTable.scan(new ScanSpec()).pages()) {
for (Item ignored : page) {
for (ScanResponse page : dynamoDbExtension.getDynamoDbClient().scanPaginator(ScanRequest.builder()
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.build())) {
for (Map<String, AttributeValue> item : page.items()) {
totalItems++;
}
}
@@ -306,16 +339,15 @@ class AccountsDynamoDbTest {
configuration.setRingBufferSizeInClosedState(2);
configuration.setFailureRateThreshold(50);
final AmazonDynamoDB client = mock(AmazonDynamoDB.class);
final DynamoDB dynamoDB = new DynamoDB(client);
final DynamoDbClient client = mock(DynamoDbClient.class);
when(client.transactWriteItems(any()))
when(client.transactWriteItems(any(TransactWriteItemsRequest.class)))
.thenThrow(RuntimeException.class);
when(client.updateItem(any()))
when(client.updateItem(any(UpdateItemRequest.class)))
.thenThrow(RuntimeException.class);
AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(AmazonDynamoDBAsync.class), mock(ThreadPoolExecutor.class), dynamoDB, ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME, mock(
AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(DynamoDbAsyncClient.class), mock(ThreadPoolExecutor.class), ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME, mock(
MigrationDeletedAccounts.class), mock(MigrationRetryAccounts.class));
Account account = generateAccount("+14151112222", UUID.randomUUID());
@@ -408,20 +440,22 @@ class AccountsDynamoDbTest {
}
private void verifyStoredState(String number, UUID uuid, Account expecting) {
final Table accounts = dynamoDbExtension.getDynamoDB().getTable(dynamoDbExtension.getTableName());
final DynamoDbClient db = dynamoDbExtension.getDynamoDbClient();
Item item = accounts.getItem(new GetItemSpec()
.withPrimaryKey(AccountsDynamoDb.KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(uuid))
.withConsistentRead(true));
final GetItemResponse get = db.getItem(GetItemRequest.builder()
.tableName(dynamoDbExtension.getTableName())
.key(Map.of(AccountsDynamoDb.KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
.consistentRead(true)
.build());
if (item != null) {
String data = new String(item.getBinary(AccountsDynamoDb.ATTR_ACCOUNT_DATA), StandardCharsets.UTF_8);
if (get.hasItem()) {
String data = new String(get.item().get(AccountsDynamoDb.ATTR_ACCOUNT_DATA).b().asByteArray(), StandardCharsets.UTF_8);
assertThat(data).isNotEmpty();
assertThat(item.getNumber(AccountsDynamoDb.ATTR_MIGRATION_VERSION).intValue())
assertThat(AttributeValues.getInt(get.item(), AccountsDynamoDb.ATTR_MIGRATION_VERSION, -1))
.isEqualTo(expecting.getDynamoDbMigrationVersion());
Account result = AccountsDynamoDb.fromItem(item);
Account result = AccountsDynamoDb.fromItem(get.item());
verifyStoredState(number, uuid, result, expecting);
} else {
throw new AssertionError("No data");

View File

@@ -1,33 +1,35 @@
package org.whispersystems.textsecuregcm.storage;
import com.almworks.sqlite4java.SQLite;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndex;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import java.net.ServerSocket;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback {
static final String DEFAULT_TABLE_NAME = "test_table";
static final ProvisionedThroughput DEFAULT_PROVISIONED_THROUGHPUT = new ProvisionedThroughput(20L, 20L);
static final ProvisionedThroughput DEFAULT_PROVISIONED_THROUGHPUT = ProvisionedThroughput.builder()
.readCapacityUnits(20L)
.writeCapacityUnits(20L)
.build();
private DynamoDBProxyServer server;
private int port;
@@ -42,9 +44,8 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
private final long readCapacityUnits;
private final long writeCapacityUnits;
private AmazonDynamoDB client;
private AmazonDynamoDBAsync asyncClient;
private DynamoDB dynamoDB;
private DynamoDbClient dynamoDB2;
private DynamoDbAsyncClient dynamoAsyncDB2;
private DynamoDbExtension(String tableName, String hashKey, String rangeKey, List<AttributeDefinition> attributeDefinitions, List<GlobalSecondaryIndex> globalSecondaryIndexes, long readCapacityUnits,
long writeCapacityUnits) {
@@ -87,26 +88,33 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
KeySchemaElement[] keySchemaElements;
if (rangeKeyName == null) {
keySchemaElements = new KeySchemaElement[] {
new KeySchemaElement(hashKeyName, "HASH"),
KeySchemaElement.builder().attributeName(hashKeyName).keyType(KeyType.HASH).build(),
};
} else {
keySchemaElements = new KeySchemaElement[] {
new KeySchemaElement(hashKeyName, "HASH"),
new KeySchemaElement(rangeKeyName, "RANGE")
KeySchemaElement.builder().attributeName(hashKeyName).keyType(KeyType.HASH).build(),
KeySchemaElement.builder().attributeName(rangeKeyName).keyType(KeyType.RANGE).build(),
};
}
final CreateTableRequest createTableRequest = new CreateTableRequest()
.withTableName(tableName)
.withKeySchema(keySchemaElements)
.withAttributeDefinitions(attributeDefinitions.isEmpty() ? null : attributeDefinitions)
.withGlobalSecondaryIndexes(globalSecondaryIndexes.isEmpty() ? null : globalSecondaryIndexes)
.withProvisionedThroughput(new ProvisionedThroughput(readCapacityUnits, writeCapacityUnits));
final CreateTableRequest createTableRequest = CreateTableRequest.builder()
.tableName(tableName)
.keySchema(keySchemaElements)
.attributeDefinitions(attributeDefinitions.isEmpty() ? null : attributeDefinitions)
.globalSecondaryIndexes(globalSecondaryIndexes.isEmpty() ? null : globalSecondaryIndexes)
.provisionedThroughput(ProvisionedThroughput.builder()
.readCapacityUnits(readCapacityUnits)
.writeCapacityUnits(writeCapacityUnits)
.build())
.build();
getDynamoDB().createTable(createTableRequest);
getDynamoDbClient().createTable(createTableRequest);
}
private void startServer() throws Exception {
// Even though we're using AWS SDK v2, Dynamo's local implementation's canonical location
// is within v1 (https://github.com/aws/aws-sdk-java-v2/issues/982). This does support
// v2 clients, though.
SQLite.setLibraryPath("target/lib"); // if you see a library failed to load error, you need to run mvn test-compile at least once first
ServerSocket serverSocket = new ServerSocket(0);
serverSocket.setReuseAddress(false);
@@ -117,18 +125,18 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
}
private void initializeClient() {
AmazonDynamoDBClientBuilder clientBuilder = AmazonDynamoDBClientBuilder.standard()
.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")));
client = clientBuilder.build();
asyncClient = AmazonDynamoDBAsyncClientBuilder.standard()
.withEndpointConfiguration(clientBuilder.getEndpoint())
.withCredentials(clientBuilder.getCredentials())
.build();
dynamoDB = new DynamoDB(client);
dynamoDB2 = DynamoDbClient.builder()
.endpointOverride(URI.create("http://localhost:" + port))
.region(Region.of("local-test-region"))
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create("accessKey", "secretKey")))
.build();
dynamoAsyncDB2 = DynamoDbAsyncClient.builder()
.endpointOverride(URI.create("http://localhost:" + port))
.region(Region.of("local-test-region"))
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create("accessKey", "secretKey")))
.build();
}
static class DynamoDbExtensionBuilder {
@@ -140,8 +148,8 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
private List<AttributeDefinition> attributeDefinitions = new ArrayList<>();
private List<GlobalSecondaryIndex> globalSecondaryIndexes = new ArrayList<>();
private long readCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.getReadCapacityUnits();
private long writeCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.getWriteCapacityUnits();
private long readCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.readCapacityUnits();
private long writeCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.writeCapacityUnits();
private DynamoDbExtensionBuilder() {
@@ -178,16 +186,12 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
}
}
public AmazonDynamoDB getClient() {
return client;
public DynamoDbClient getDynamoDbClient() {
return dynamoDB2;
}
public AmazonDynamoDBAsync getAsyncClient() {
return asyncClient;
}
public DynamoDB getDynamoDB() {
return dynamoDB;
public DynamoDbAsyncClient getDynamoDbAsyncClient() {
return dynamoAsyncDB2;
}
public String getTableName() {

View File

@@ -5,13 +5,13 @@
package org.whispersystems.textsecuregcm.storage;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import org.whispersystems.textsecuregcm.tests.util.LocalDynamoDbRule;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
public class KeysDynamoDbRule extends LocalDynamoDbRule {
public static final String TABLE_NAME = "Signal_Keys_Test";
@@ -19,18 +19,22 @@ public class KeysDynamoDbRule extends LocalDynamoDbRule {
@Override
protected void before() throws Throwable {
super.before();
final DynamoDB dynamoDB = getDynamoDB();
final CreateTableRequest createTableRequest = new CreateTableRequest()
.withTableName(TABLE_NAME)
.withKeySchema(new KeySchemaElement(KeysDynamoDb.KEY_ACCOUNT_UUID, "HASH"),
new KeySchemaElement(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID, "RANGE"))
.withAttributeDefinitions(new AttributeDefinition(KeysDynamoDb.KEY_ACCOUNT_UUID, ScalarAttributeType.B),
new AttributeDefinition(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID, ScalarAttributeType.B))
.withProvisionedThroughput(new ProvisionedThroughput(20L, 20L));
dynamoDB.createTable(createTableRequest);
getDynamoDbClient().createTable(CreateTableRequest.builder()
.tableName(TABLE_NAME)
.keySchema(
KeySchemaElement.builder().attributeName(KeysDynamoDb.KEY_ACCOUNT_UUID).keyType(KeyType.HASH).build(),
KeySchemaElement.builder().attributeName(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID).keyType(KeyType.RANGE)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName(KeysDynamoDb.KEY_ACCOUNT_UUID)
.attributeType(ScalarAttributeType.B)
.build(),
AttributeDefinition.builder()
.attributeName(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID)
.attributeType(ScalarAttributeType.B)
.build())
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(20L).writeCapacityUnits(20L).build())
.build());
}
@Override

View File

@@ -9,6 +9,7 @@ import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.whispersystems.textsecuregcm.entities.PreKey;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import java.util.Collections;
import java.util.List;
@@ -17,6 +18,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -34,7 +36,7 @@ public class KeysDynamoDbTest {
@Before
public void setup() {
keysDynamoDb = new KeysDynamoDb(dynamoDbRule.getDynamoDB(), KeysDynamoDbRule.TABLE_NAME);
keysDynamoDb = new KeysDynamoDb(dynamoDbRule.getDynamoDbClient(), KeysDynamoDbRule.TABLE_NAME);
account = mock(Account.class);
when(account.getNumber()).thenReturn(ACCOUNT_NUMBER);
@@ -133,4 +135,10 @@ public class KeysDynamoDbTest {
assertEquals(0, keysDynamoDb.getCount(account, DEVICE_ID));
assertEquals(1, keysDynamoDb.getCount(account, DEVICE_ID + 1));
}
@Test
public void testSortKeyPrefix() {
AttributeValue got = KeysDynamoDb.getSortKeyPrefix(123);
assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 123}, got.b().asByteArray());
}
}

View File

@@ -9,12 +9,6 @@ import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemCollection;
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
import com.google.protobuf.ByteString;
import io.lettuce.core.cluster.SlotHash;
import java.nio.ByteBuffer;
@@ -22,6 +16,7 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
@@ -38,6 +33,10 @@ import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
@@ -62,7 +61,7 @@ public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
connection.sync().upstream().commands().configSet("notify-keyspace-events", "K$glz");
});
final MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
final MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDbClient(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
final AccountsManager accountsManager = mock(AccountsManager.class);
final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
@@ -142,17 +141,16 @@ public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
final List<MessageProtos.Envelope> persistedMessages = new ArrayList<>(messageCount);
DynamoDB dynamoDB = messagesDynamoDbRule.getDynamoDB();
Table table = dynamoDB.getTable(MessagesDynamoDbRule.TABLE_NAME);
final ItemCollection<ScanOutcome> scan = table.scan(new ScanSpec());
for (Item item : scan) {
persistedMessages.add(MessageProtos.Envelope.newBuilder()
.setServerGuid(convertBinaryToUuid(item.getBinary("U")).toString())
.setType(MessageProtos.Envelope.Type.valueOf(item.getInt("T")))
.setTimestamp(item.getLong("TS"))
.setServerTimestamp(extractServerTimestamp(item.getBinary("S")))
.setContent(ByteString.copyFrom(item.getBinary("C")))
.build());
DynamoDbClient dynamoDB = messagesDynamoDbRule.getDynamoDbClient();
for (Map<String, AttributeValue> item : dynamoDB
.scan(ScanRequest.builder().tableName(MessagesDynamoDbRule.TABLE_NAME).build()).items()) {
persistedMessages.add(MessageProtos.Envelope.newBuilder()
.setServerGuid(AttributeValues.getUUID(item, "U", null).toString())
.setType(MessageProtos.Envelope.Type.valueOf(AttributeValues.getInt(item, "T", -1)))
.setTimestamp(AttributeValues.getLong(item, "TS", -1))
.setServerTimestamp(extractServerTimestamp(AttributeValues.getByteArray(item, "S", null)))
.setContent(ByteString.copyFrom(AttributeValues.getByteArray(item, "C", null)))
.build());
}
assertEquals(expectedMessages, persistedMessages);

View File

@@ -4,10 +4,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import java.util.UUID;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class MigrationDeletedAccountsTest {
@@ -15,13 +15,16 @@ class MigrationDeletedAccountsTest {
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName("deleted_accounts_test")
.hashKey(MigrationDeletedAccounts.KEY_UUID)
.attributeDefinition(new AttributeDefinition(MigrationDeletedAccounts.KEY_UUID, ScalarAttributeType.B))
.attributeDefinition(AttributeDefinition.builder()
.attributeName(MigrationDeletedAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.build();
@Test
void test() {
final MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDB(),
final MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getTableName());
UUID firstUuid = UUID.randomUUID();

View File

@@ -2,12 +2,12 @@ package org.whispersystems.textsecuregcm.storage;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class MigrationRetryAccountsTest {
@@ -15,13 +15,16 @@ class MigrationRetryAccountsTest {
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName("account_migration_errors_test")
.hashKey(MigrationRetryAccounts.KEY_UUID)
.attributeDefinition(new AttributeDefinition(MigrationRetryAccounts.KEY_UUID, ScalarAttributeType.B))
.attributeDefinition(AttributeDefinition.builder()
.attributeName(MigrationRetryAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.build();
@Test
void test() {
final MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(dynamoDbExtension.getDynamoDB(),
final MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getTableName());
UUID firstUuid = UUID.randomUUID();

View File

@@ -9,8 +9,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
@@ -20,6 +18,8 @@ import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class PushChallengeDynamoDbTest {
@@ -34,12 +34,15 @@ class PushChallengeDynamoDbTest {
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName(TABLE_NAME)
.hashKey(PushChallengeDynamoDb.KEY_ACCOUNT_UUID)
.attributeDefinition(new AttributeDefinition(PushChallengeDynamoDb.KEY_ACCOUNT_UUID, ScalarAttributeType.B))
.attributeDefinition(AttributeDefinition.builder()
.attributeName(PushChallengeDynamoDb.KEY_ACCOUNT_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.build();
@BeforeEach
void setUp() {
this.pushChallengeDynamoDb = new PushChallengeDynamoDb(dynamoDbExtension.getDynamoDB(), TABLE_NAME, Clock.fixed(
this.pushChallengeDynamoDb = new PushChallengeDynamoDb(dynamoDbExtension.getDynamoDbClient(), TABLE_NAME, Clock.fixed(
Instant.ofEpochMilli(CURRENT_TIME_MILLIS), ZoneId.systemDefault()));
}

View File

@@ -4,13 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class ReportMessageDynamoDbTest {
@@ -22,13 +22,16 @@ class ReportMessageDynamoDbTest {
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName(TABLE_NAME)
.hashKey(ReportMessageDynamoDb.KEY_HASH)
.attributeDefinition(new AttributeDefinition(ReportMessageDynamoDb.KEY_HASH, ScalarAttributeType.B))
.attributeDefinition(AttributeDefinition.builder()
.attributeName(ReportMessageDynamoDb.KEY_HASH)
.attributeType(ScalarAttributeType.B)
.build())
.build();
@BeforeEach
void setUp() {
this.reportMessageDynamoDb = new ReportMessageDynamoDb(dynamoDbExtension.getDynamoDB(), TABLE_NAME);
this.reportMessageDynamoDb = new ReportMessageDynamoDb(dynamoDbExtension.getDynamoDbClient(), TABLE_NAME);
}
@Test

View File

@@ -266,6 +266,8 @@ class AccountControllerTest {
apnSender,
usernamesManager,
verifyExperimentEnrollmentManager);
clearInvocations(AuthHelper.DISABLED_DEVICE);
}
@Test
@@ -322,6 +324,51 @@ class AccountControllerTest {
assertThat(captor.getValue().getChallengeData().isPresent()).isTrue();
assertThat(captor.getValue().getChallengeData().get().length()).isEqualTo(32);
assertThat(captor.getValue().getMessage()).contains("\"challenge\" : \"" + captor.getValue().getChallengeData().get() + "\"");
assertThat(captor.getValue().isVoip()).isTrue();
verifyNoMoreInteractions(gcmSender);
}
@Test
void testGetApnPreauthExplicitVoip() throws Exception {
Response response = resources.getJerseyTest()
.target("/v1/accounts/apn/preauth/mytoken/+14152222222")
.queryParam("voip", "true")
.request()
.get();
assertThat(response.getStatus()).isEqualTo(200);
ArgumentCaptor<ApnMessage> captor = ArgumentCaptor.forClass(ApnMessage.class);
verify(apnSender, times(1)).sendMessage(captor.capture());
assertThat(captor.getValue().getApnId()).isEqualTo("mytoken");
assertThat(captor.getValue().getChallengeData().isPresent()).isTrue();
assertThat(captor.getValue().getChallengeData().get().length()).isEqualTo(32);
assertThat(captor.getValue().getMessage()).contains("\"challenge\" : \"" + captor.getValue().getChallengeData().get() + "\"");
assertThat(captor.getValue().isVoip()).isTrue();
verifyNoMoreInteractions(gcmSender);
}
@Test
void testGetApnPreauthExplicitNoVoip() throws Exception {
Response response = resources.getJerseyTest()
.target("/v1/accounts/apn/preauth/mytoken/+14152222222")
.queryParam("voip", "false")
.request()
.get();
assertThat(response.getStatus()).isEqualTo(200);
ArgumentCaptor<ApnMessage> captor = ArgumentCaptor.forClass(ApnMessage.class);
verify(apnSender, times(1)).sendMessage(captor.capture());
assertThat(captor.getValue().getApnId()).isEqualTo("mytoken");
assertThat(captor.getValue().getChallengeData().isPresent()).isTrue();
assertThat(captor.getValue().getChallengeData().get().length()).isEqualTo(32);
assertThat(captor.getValue().getMessage()).contains("\"challenge\" : \"" + captor.getValue().getChallengeData().get() + "\"");
assertThat(captor.getValue().isVoip()).isFalse();
verifyNoMoreInteractions(gcmSender);
}
@@ -1342,6 +1389,23 @@ class AccountControllerTest {
verify(directoryQueue, never()).refreshRegisteredUser(any(Account.class));
}
@Test
void testSetApnIdNoVoip() throws Exception {
Response response =
resources.getJerseyTest()
.target("/v1/accounts/apn/")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.DISABLED_NUMBER, AuthHelper.DISABLED_PASSWORD))
.put(Entity.json(new ApnRegistrationId("first", null)));
assertThat(response.getStatus()).isEqualTo(204);
verify(AuthHelper.DISABLED_DEVICE, times(1)).setApnId(eq("first"));
verify(AuthHelper.DISABLED_DEVICE, times(1)).setVoipApnId(null);
verify(accountsManager, times(1)).update(eq(AuthHelper.DISABLED_ACCOUNT));
verify(directoryQueue, never()).refreshRegisteredUser(any(Account.class));
}
@Test
void testSetApnIdByUuid() throws Exception {
Response response =

View File

@@ -42,6 +42,7 @@ import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.PendingDevicesManager;
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
@@ -114,6 +115,7 @@ public class DeviceControllerTest {
when(account.isEnabled()).thenReturn(false);
when(account.isGroupsV2Supported()).thenReturn(true);
when(account.isGv1MigrationSupported()).thenReturn(true);
when(account.isSenderKeySupported()).thenReturn(true);
when(pendingDevicesManager.getCodeForNumber(AuthHelper.VALID_NUMBER)).thenReturn(Optional.of(new StoredVerificationCode("5678901", System.currentTimeMillis(), null)));
when(pendingDevicesManager.getCodeForNumber(AuthHelper.VALID_NUMBER_TWO)).thenReturn(Optional.of(new StoredVerificationCode("1112223", System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(31), null)));
@@ -221,8 +223,7 @@ public class DeviceControllerTest {
@Test
@Parameters(method = "argumentsForDeviceDowngradeCapabilitiesTest")
public void deviceDowngradeCapabilitiesTest(final String userAgent, final boolean gv2, final boolean gv2_2, final boolean gv2_3, final int expectedStatus) throws Exception {
Device.DeviceCapabilities deviceCapabilities = new Device.DeviceCapabilities(gv2, gv2_2, gv2_3, true, false, true,
false);
DeviceCapabilities deviceCapabilities = new DeviceCapabilities(gv2, gv2_2, gv2_3, true, false, true, true);
AccountAttributes accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
Response response = resources.getJerseyTest()
.target("/v1/devices/5678901")
@@ -262,8 +263,7 @@ public class DeviceControllerTest {
@Test
public void deviceDowngradeGv1MigrationTest() {
Device.DeviceCapabilities deviceCapabilities = new Device.DeviceCapabilities(true, true, true, true, false, false,
false);
DeviceCapabilities deviceCapabilities = new DeviceCapabilities(true, true, true, true, false, false, true);
AccountAttributes accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
Response response = resources.getJerseyTest()
.target("/v1/devices/5678901")
@@ -274,7 +274,7 @@ public class DeviceControllerTest {
assertThat(response.getStatus()).isEqualTo(409);
deviceCapabilities = new Device.DeviceCapabilities(true, true, true, true, false, true, false);
deviceCapabilities = new DeviceCapabilities(true, true, true, true, false, true, true);
accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
response = resources.getJerseyTest()
.target("/v1/devices/5678901")
@@ -284,6 +284,31 @@ public class DeviceControllerTest {
.put(Entity.entity(accountAttributes, MediaType.APPLICATION_JSON_TYPE));
assertThat(response.getStatus()).isEqualTo(200);
}
@Test
public void deviceDowngradeSenderKeyTest() {
DeviceCapabilities deviceCapabilities = new DeviceCapabilities(true, true, true, true, true, true, false);
AccountAttributes accountAttributes =
new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
Response response = resources
.getJerseyTest()
.target("/v1/devices/5678901")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
.header("User-Agent", "Signal-Android/5.42.8675309 Android/30")
.put(Entity.entity(accountAttributes, MediaType.APPLICATION_JSON_TYPE));
assertThat(response.getStatus()).isEqualTo(409);
deviceCapabilities = new DeviceCapabilities(true, true, true, true, true, true, true);
accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
response = resources
.getJerseyTest()
.target("/v1/devices/5678901")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
.header("User-Agent", "Signal-Android/5.42.8675309 Android/30")
.put(Entity.entity(accountAttributes, MediaType.APPLICATION_JSON_TYPE));
assertThat(response.getStatus()).isEqualTo(200);
}
}

View File

@@ -478,7 +478,7 @@ class MessageControllerTest {
List<OutgoingMessageEntity> messages = new LinkedList<>() {{
add(new OutgoingMessageEntity(1L, false, messageGuidOne, Envelope.Type.CIPHERTEXT_VALUE, null, timestampOne, "+14152222222", sourceUuid, 2, "hi there".getBytes(), null, 0));
add(new OutgoingMessageEntity(2L, false, null, Envelope.Type.RECEIPT_VALUE, null, timestampTwo, "+14152222222", sourceUuid, 2, null, null, 0));
add(new OutgoingMessageEntity(2L, false, null, Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE, null, timestampTwo, "+14152222222", sourceUuid, 2, null, null, 0));
}};
OutgoingMessageEntityList messagesList = new OutgoingMessageEntityList(messages, false);
@@ -515,7 +515,7 @@ class MessageControllerTest {
List<OutgoingMessageEntity> messages = new LinkedList<OutgoingMessageEntity>() {{
add(new OutgoingMessageEntity(1L, false, UUID.randomUUID(), Envelope.Type.CIPHERTEXT_VALUE, null, timestampOne, "+14152222222", UUID.randomUUID(), 2, "hi there".getBytes(), null, 0));
add(new OutgoingMessageEntity(2L, false, UUID.randomUUID(), Envelope.Type.RECEIPT_VALUE, null, timestampTwo, "+14152222222", UUID.randomUUID(), 2, null, null, 0));
add(new OutgoingMessageEntity(2L, false, UUID.randomUUID(), Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE, null, timestampTwo, "+14152222222", UUID.randomUUID(), 2, null, null, 0));
}};
OutgoingMessageEntityList messagesList = new OutgoingMessageEntityList(messages, false);
@@ -546,7 +546,7 @@ class MessageControllerTest {
when(messagesManager.delete(AuthHelper.VALID_UUID, 1, "+14152222222", 31338))
.thenReturn(Optional.of(new OutgoingMessageEntity(31337L, true, null,
Envelope.Type.RECEIPT_VALUE,
Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE,
null, System.currentTimeMillis(),
"+14152222222", sourceUuid, 1, null, null, 0)));
@@ -583,37 +583,6 @@ class MessageControllerTest {
}
@ParameterizedTest
@MethodSource
void testOnlineMessage(final String fixture, final boolean expectedOnline) throws Exception {
final Response response =
resources.getJerseyTest()
.target(String.format("/v1/messages/%s", SINGLE_DEVICE_RECIPIENT))
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
.put(Entity.entity(mapper.readValue(jsonFixture(fixture), IncomingMessageList.class),
MediaType.APPLICATION_JSON_TYPE));
assertThat("Good Response", response.getStatus(), is(equalTo(200)));
verify(messageSender, times(1)).sendMessage(any(Account.class), any(Device.class), any(Envelope.class), eq(expectedOnline));
}
private static Stream<Arguments> testOnlineMessage() {
return Stream.of(
Arguments.of("fixtures/current_message_single_device.json", false), // default to `false` when absent
Arguments.of("fixtures/online_message_true.json", true),
Arguments.of("fixtures/online_message_false.json", false),
// iOS versions prior to 5.5.0.7 send `online` on IncomingMessageList.message, rather on the top-level entity.
// This causes some odd client behaviors, such as persisted typing indicators, so we have a temporary
// server-side adaptation.
Arguments.of("fixtures/online_message_true_nested_property.json", true),
Arguments.of("fixtures/online_message_false_nested_property.json", false)
);
}
@Test
void testReportMessage() {

View File

@@ -78,7 +78,7 @@ public class APNSenderTest {
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
assertThat(notification.getValue().getTopic()).isEqualTo("foo.voip");
@@ -112,7 +112,7 @@ public class APNSenderTest {
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NSE_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
assertThat(notification.getValue().getTopic()).isEqualTo("foo");
@@ -153,7 +153,7 @@ public class APNSenderTest {
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
assertThat(apnResult.getStatus()).isEqualTo(ApnResult.Status.NO_SUCH_USER);
@@ -256,7 +256,7 @@ public class APNSenderTest {
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
assertThat(apnResult.getStatus()).isEqualTo(ApnResult.Status.NO_SUCH_USER);
@@ -346,7 +346,7 @@ public class APNSenderTest {
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
assertThat(apnResult.getStatus()).isEqualTo(ApnResult.Status.GENERIC_FAILURE);
@@ -387,7 +387,7 @@ public class APNSenderTest {
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
verifyNoMoreInteractions(apnsClient);

View File

@@ -5,10 +5,11 @@
package org.whispersystems.textsecuregcm.tests.storage;
import org.junit.Before;
import org.junit.Test;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Device;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
@@ -16,11 +17,11 @@ import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.junit.Before;
import org.junit.Test;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
public class AccountTest {
@@ -38,6 +39,10 @@ public class AccountTest {
private final Device gv1MigrationIncapableDevice = mock(Device.class);
private final Device gv1MigrationIncapableExpiredDevice = mock(Device.class);
private final Device senderKeyCapableDevice = mock(Device.class);
private final Device senderKeyIncapableDevice = mock(Device.class);
private final Device senderKeyIncapableExpiredDevice = mock(Device.class);
@Before
public void setup() {
when(oldMasterDevice.getLastSeen()).thenReturn(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(366));
@@ -72,17 +77,29 @@ public class AccountTest {
when(gv2IncapableExpiredDevice.getLastSeen()).thenReturn(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(31));
when(gv2IncapableExpiredDevice.isEnabled()).thenReturn(false);
when(gv1MigrationCapableDevice.getCapabilities()).thenReturn(new Device.DeviceCapabilities(true, true, true, true, true, true,
false));
when(gv1MigrationCapableDevice.getCapabilities()).thenReturn(
new DeviceCapabilities(true, true, true, true, true, true, false));
when(gv1MigrationCapableDevice.isEnabled()).thenReturn(true);
when(gv1MigrationIncapableDevice.getCapabilities()).thenReturn(new Device.DeviceCapabilities(true, true, true, true, true, false,
false));
when(gv1MigrationIncapableDevice.getCapabilities()).thenReturn(
new DeviceCapabilities(true, true, true, true, true, false, false));
when(gv1MigrationIncapableDevice.isEnabled()).thenReturn(true);
when(gv1MigrationIncapableExpiredDevice.getCapabilities()).thenReturn(new Device.DeviceCapabilities(true, true, true, true, true, false,
false));
when(gv1MigrationIncapableExpiredDevice.getCapabilities()).thenReturn(
new DeviceCapabilities(true, true, true, true, true, false, false));
when(gv1MigrationIncapableExpiredDevice.isEnabled()).thenReturn(false);
when(senderKeyCapableDevice.getCapabilities()).thenReturn(
new DeviceCapabilities(true, true, true, true, true, true, true));
when(senderKeyCapableDevice.isEnabled()).thenReturn(true);
when(senderKeyIncapableDevice.getCapabilities()).thenReturn(
new DeviceCapabilities(true, true, true, true, true, true, false));
when(senderKeyIncapableDevice.isEnabled()).thenReturn(true);
when(senderKeyIncapableExpiredDevice.getCapabilities()).thenReturn(
new DeviceCapabilities(true, true, true, true, true, true, false));
when(senderKeyIncapableExpiredDevice.isEnabled()).thenReturn(false);
}
@Test
@@ -137,8 +154,8 @@ public class AccountTest {
final Device nonTransferCapableMasterDevice = mock(Device.class);
final Device transferCapableLinkedDevice = mock(Device.class);
final Device.DeviceCapabilities transferCapabilities = mock(Device.DeviceCapabilities.class);
final Device.DeviceCapabilities nonTransferCapabilities = mock(Device.DeviceCapabilities.class);
final DeviceCapabilities transferCapabilities = mock(DeviceCapabilities.class);
final DeviceCapabilities nonTransferCapabilities = mock(DeviceCapabilities.class);
when(transferCapableMasterDevice.getId()).thenReturn(1L);
when(transferCapableMasterDevice.isMaster()).thenReturn(true);
@@ -205,4 +222,15 @@ public class AccountTest {
assertFalse(new Account("+18005551234", UUID.randomUUID(), Set.of(gv1MigrationCapableDevice, gv1MigrationIncapableDevice), "1234".getBytes(StandardCharsets.UTF_8)).isGv1MigrationSupported());
assertTrue(new Account("+18005551234", UUID.randomUUID(), Set.of(gv1MigrationCapableDevice, gv1MigrationIncapableExpiredDevice), "1234".getBytes(StandardCharsets.UTF_8)).isGv1MigrationSupported());
}
@Test
public void isSenderKeySupported() {
assertThat(new Account("+18005551234", UUID.randomUUID(), Set.of(senderKeyCapableDevice),
"1234".getBytes(StandardCharsets.UTF_8)).isSenderKeySupported()).isTrue();
assertThat(new Account("+18005551234", UUID.randomUUID(), Set.of(senderKeyCapableDevice, senderKeyIncapableDevice),
"1234".getBytes(StandardCharsets.UTF_8)).isSenderKeySupported()).isFalse();
assertThat(new Account("+18005551234", UUID.randomUUID(),
Set.of(senderKeyCapableDevice, senderKeyIncapableExpiredDevice),
"1234".getBytes(StandardCharsets.UTF_8)).isSenderKeySupported()).isTrue();
}
}

View File

@@ -20,7 +20,6 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import io.lettuce.core.RedisException;
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
import java.util.HashSet;
@@ -32,6 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
@@ -41,12 +41,14 @@ import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper;
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
class AccountsManagerTest {
@@ -383,7 +385,7 @@ class AccountsManagerTest {
}
@Test
void testCompareAccounts() {
void testCompareAccounts() throws Exception {
RedisAdvancedClusterCommands<String, String> commands = mock(RedisAdvancedClusterCommands.class);
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
@@ -409,9 +411,58 @@ class AccountsManagerTest {
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
{
Device device1 = new Device();
device1.setId(1L);
a1.addDevice(device1);
assertEquals(Optional.of("devices"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
Device device2 = new Device();
device2.setId(1L);
a2.addDevice(device2);
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
device1.setLastSeen(1L);
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
device1.setName("name");
assertEquals(Optional.of("devices"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
device1.setName(null);
device1.setSignedPreKey(new SignedPreKey(1L, "123", "456"));
device2.setSignedPreKey(new SignedPreKey(2L, "123", "456"));
assertEquals(Optional.of("masterDeviceSignedPreKey"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
device1.setSignedPreKey(null);
device2.setSignedPreKey(null);
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
device1.setApnId("123");
Thread.sleep(5);
device2.setApnId("123");
assertEquals(Optional.of("masterDevicePushTimestamp"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
a1.removeDevice(1L);
a2.removeDevice(1L);
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
}
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
a1.setDynamoDbMigrationVersion(1);
assertEquals(Optional.of("migrationVersion"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
a2.setProfileName("name");

View File

@@ -67,7 +67,7 @@ public class MessagesDynamoDbTest {
@Before
public void setup() {
messagesDynamoDb = new MessagesDynamoDb(dynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
messagesDynamoDb = new MessagesDynamoDb(dynamoDbRule.getDynamoDbClient(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
}
@Test

View File

@@ -6,16 +6,16 @@
package org.whispersystems.textsecuregcm.tests.util;
import com.almworks.sqlite4java.SQLite;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
import org.junit.rules.ExternalResource;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import java.net.ServerSocket;
import java.net.URI;
public class LocalDynamoDbRule extends ExternalResource {
private DynamoDBProxyServer server;
@@ -43,11 +43,12 @@ public class LocalDynamoDbRule extends ExternalResource {
super.after();
}
public DynamoDB getDynamoDB() {
AmazonDynamoDBClientBuilder clientBuilder =
AmazonDynamoDBClientBuilder.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")));
return new DynamoDB(clientBuilder.build());
public DynamoDbClient getDynamoDbClient() {
return DynamoDbClient.builder()
.endpointOverride(URI.create("http://localhost:" + port))
.region(Region.of("local-test-region"))
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create("accessKey", "secretKey")))
.build();
}
}

View File

@@ -5,15 +5,15 @@
package org.whispersystems.textsecuregcm.tests.util;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.LocalSecondaryIndex;
import com.amazonaws.services.dynamodbv2.model.Projection;
import com.amazonaws.services.dynamodbv2.model.ProjectionType;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.LocalSecondaryIndex;
import software.amazon.awssdk.services.dynamodb.model.Projection;
import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
public class MessagesDynamoDbRule extends LocalDynamoDbRule {
@@ -22,20 +22,21 @@ public class MessagesDynamoDbRule extends LocalDynamoDbRule {
@Override
protected void before() throws Throwable {
super.before();
DynamoDB dynamoDB = getDynamoDB();
CreateTableRequest createTableRequest = new CreateTableRequest()
.withTableName(TABLE_NAME)
.withKeySchema(new KeySchemaElement("H", "HASH"),
new KeySchemaElement("S", "RANGE"))
.withAttributeDefinitions(new AttributeDefinition("H", ScalarAttributeType.B),
new AttributeDefinition("S", ScalarAttributeType.B),
new AttributeDefinition("U", ScalarAttributeType.B))
.withProvisionedThroughput(new ProvisionedThroughput(20L, 20L))
.withLocalSecondaryIndexes(new LocalSecondaryIndex().withIndexName("Message_UUID_Index")
.withKeySchema(new KeySchemaElement("H", "HASH"),
new KeySchemaElement("U", "RANGE"))
.withProjection(new Projection().withProjectionType(ProjectionType.KEYS_ONLY)));
dynamoDB.createTable(createTableRequest);
getDynamoDbClient().createTable(CreateTableRequest.builder()
.tableName(TABLE_NAME)
.keySchema(KeySchemaElement.builder().attributeName("H").keyType(KeyType.HASH).build(),
KeySchemaElement.builder().attributeName("S").keyType(KeyType.RANGE).build())
.attributeDefinitions(
AttributeDefinition.builder().attributeName("H").attributeType(ScalarAttributeType.B).build(),
AttributeDefinition.builder().attributeName("S").attributeType(ScalarAttributeType.B).build(),
AttributeDefinition.builder().attributeName("U").attributeType(ScalarAttributeType.B).build())
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(20L).writeCapacityUnits(20L).build())
.localSecondaryIndexes(LocalSecondaryIndex.builder().indexName("Message_UUID_Index")
.keySchema(KeySchemaElement.builder().attributeName("H").keyType(KeyType.HASH).build(),
KeySchemaElement.builder().attributeName("U").keyType(KeyType.RANGE).build())
.projection(Projection.builder().projectionType(ProjectionType.KEYS_ONLY).build())
.build())
.build());
}
@Override

View File

@@ -8,17 +8,11 @@ package org.whispersystems.textsecuregcm.util;
import org.junit.jupiter.api.Test;
import org.whispersystems.textsecuregcm.configuration.MonitoredS3ObjectConfiguration;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.Inet4Address;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mock;
@@ -34,7 +28,7 @@ class AsnManagerTest {
assertEquals(Optional.empty(), asnManager.getAsn("10.0.0.1"));
try (final InputStream tableInputStream = getClass().getResourceAsStream("ip2asn-test.tsv")) {
asnManager.handleAsnTableChanged(tableInputStream);
asnManager.handleAsnTableChangedStream(tableInputStream);
}
assertEquals(Optional.of(7922L), asnManager.getAsn("50.79.54.1"));

View File

@@ -12,7 +12,6 @@ import java.io.InputStreamReader;
import java.net.Inet4Address;
import java.net.UnknownHostException;
import java.util.Optional;
import java.util.zip.GZIPInputStream;
import static org.junit.jupiter.api.Assertions.*;

View File

@@ -0,0 +1,60 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.util;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.*;
public class AttributeValuesTest {
@Test
void testUUIDRoundTrip() {
UUID orig = UUID.randomUUID();
AttributeValue av = AttributeValues.fromUUID(orig);
UUID returned = AttributeValues.getUUID(Map.of("foo", av), "foo", null);
assertEquals(orig, returned);
}
@Test
void testLongRoundTrip() {
long orig = 12345;
AttributeValue av = AttributeValues.fromLong(orig);
long returned = AttributeValues.getLong(Map.of("foo", av), "foo", -1);
assertEquals(orig, returned);
}
@Test
void testIntRoundTrip() {
int orig = 12345;
AttributeValue av = AttributeValues.fromInt(orig);
int returned = AttributeValues.getInt(Map.of("foo", av), "foo", -1);
assertEquals(orig, returned);
}
@Test
void testByteBuffer() {
byte[] bytes = {1, 2, 3};
ByteBuffer bb = ByteBuffer.wrap(bytes);
AttributeValue av = AttributeValues.fromByteBuffer(bb);
byte[] returned = av.b().asByteArray();
assertArrayEquals(bytes, returned);
returned = AttributeValues.getByteArray(Map.of("foo", av), "foo", null);
assertArrayEquals(bytes, returned);
}
@Test
void testByteBuffer2() {
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
byteBuffer.putLong(123);
assertEquals(byteBuffer.remaining(), 0);
AttributeValue av = AttributeValues.fromByteBuffer(byteBuffer.flip());
assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 123}, AttributeValues.getByteArray(Map.of("foo", av), "foo", null));
}
}

View File

@@ -1,35 +1,39 @@
package org.whispersystems.textsecuregcm.util;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.time.Duration;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.function.Consumer;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.http.AbortableInputStream;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
class S3ObjectMonitorTest {
@Test
void refresh() {
final AmazonS3 s3Client = mock(AmazonS3.class);
final ObjectMetadata metadata = mock(ObjectMetadata.class);
final S3Object s3Object = mock(S3Object.class);
final S3Client s3Client = mock(S3Client.class);
final String bucket = "s3bucket";
final String objectKey = "greatest-smooth-jazz-hits-of-all-time.zip";
//noinspection unchecked
final Consumer<S3Object> listener = mock(Consumer.class);
final Consumer<InputStream> listener = mock(Consumer.class);
final S3ObjectMonitor objectMonitor = new S3ObjectMonitor(
s3Client,
@@ -40,28 +44,27 @@ class S3ObjectMonitorTest {
Duration.ofMinutes(1),
listener);
when(metadata.getETag()).thenReturn(UUID.randomUUID().toString());
when(s3Object.getObjectMetadata()).thenReturn(metadata);
when(s3Client.getObjectMetadata(bucket, objectKey)).thenReturn(metadata);
when(s3Client.getObject(bucket, objectKey)).thenReturn(s3Object);
String uuid = UUID.randomUUID().toString();
when(s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(
HeadObjectResponse.builder().eTag(uuid).build());
ResponseInputStream<GetObjectResponse> ris = responseInputStreamFromString("abc", uuid);
when(s3Client.getObject(GetObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(ris);
objectMonitor.refresh();
objectMonitor.refresh();
verify(listener).accept(s3Object);
verify(listener).accept(ris);
}
@Test
void refreshAfterGet() throws IOException {
final AmazonS3 s3Client = mock(AmazonS3.class);
final ObjectMetadata metadata = mock(ObjectMetadata.class);
final S3Object s3Object = mock(S3Object.class);
final S3Client s3Client = mock(S3Client.class);
final String bucket = "s3bucket";
final String objectKey = "greatest-smooth-jazz-hits-of-all-time.zip";
//noinspection unchecked
final Consumer<S3Object> listener = mock(Consumer.class);
final Consumer<InputStream> listener = mock(Consumer.class);
final S3ObjectMonitor objectMonitor = new S3ObjectMonitor(
s3Client,
@@ -72,29 +75,34 @@ class S3ObjectMonitorTest {
Duration.ofMinutes(1),
listener);
when(metadata.getETag()).thenReturn(UUID.randomUUID().toString());
when(s3Object.getObjectMetadata()).thenReturn(metadata);
when(s3Client.getObjectMetadata(bucket, objectKey)).thenReturn(metadata);
when(s3Client.getObject(bucket, objectKey)).thenReturn(s3Object);
String uuid = UUID.randomUUID().toString();
when(s3Client.headObject(HeadObjectRequest.builder().key(objectKey).bucket(bucket).build()))
.thenReturn(HeadObjectResponse.builder().eTag(uuid).build());
ResponseInputStream<GetObjectResponse> responseInputStream = responseInputStreamFromString("abc", uuid);
when(s3Client.getObject(GetObjectRequest.builder().key(objectKey).bucket(bucket).build())).thenReturn(responseInputStream);
objectMonitor.getObject();
objectMonitor.refresh();
verify(listener, never()).accept(s3Object);
verify(listener, never()).accept(responseInputStream);
}
private ResponseInputStream<GetObjectResponse> responseInputStreamFromString(String s, String etag) {
byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
AbortableInputStream ais = AbortableInputStream.create(new ByteArrayInputStream(bytes));
return new ResponseInputStream<>(GetObjectResponse.builder().contentLength((long) bytes.length).eTag(etag).build(), ais);
}
@Test
void refreshOversizedObject() {
final AmazonS3 s3Client = mock(AmazonS3.class);
final ObjectMetadata metadata = mock(ObjectMetadata.class);
final S3Object s3Object = mock(S3Object.class);
final S3Client s3Client = mock(S3Client.class);
final String bucket = "s3bucket";
final String objectKey = "greatest-smooth-jazz-hits-of-all-time.zip";
final long maxObjectSize = 16 * 1024 * 1024;
//noinspection unchecked
final Consumer<S3Object> listener = mock(Consumer.class);
final Consumer<InputStream> listener = mock(Consumer.class);
final S3ObjectMonitor objectMonitor = new S3ObjectMonitor(
s3Client,
@@ -105,11 +113,11 @@ class S3ObjectMonitorTest {
Duration.ofMinutes(1),
listener);
when(metadata.getETag()).thenReturn(UUID.randomUUID().toString());
when(metadata.getContentLength()).thenReturn(maxObjectSize + 1);
when(s3Object.getObjectMetadata()).thenReturn(metadata);
when(s3Client.getObjectMetadata(bucket, objectKey)).thenReturn(metadata);
when(s3Client.getObject(bucket, objectKey)).thenReturn(s3Object);
String uuid = UUID.randomUUID().toString();
when(s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(
HeadObjectResponse.builder().eTag(uuid).contentLength(maxObjectSize+1).build());
ResponseInputStream<GetObjectResponse> ris = responseInputStreamFromString("a".repeat((int) maxObjectSize+1), uuid);
when(s3Client.getObject(GetObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(ris);
objectMonitor.refresh();

View File

@@ -29,7 +29,7 @@ public class TorExitNodeManagerTest extends AbstractRedisClusterTest {
assertFalse(torExitNodeManager.isTorExitNode("10.0.0.1"));
assertFalse(torExitNodeManager.isTorExitNode("10.0.0.2"));
torExitNodeManager.handleExitListChanged(
torExitNodeManager.handleExitListChangedStream(
new ByteArrayInputStream("10.0.0.1\n10.0.0.2".getBytes(StandardCharsets.UTF_8)));
assertTrue(torExitNodeManager.isTorExitNode("10.0.0.1"));

View File

@@ -79,7 +79,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest
executorService = Executors.newSingleThreadExecutor();
messagesCache = new MessagesCache(getRedisCluster(), getRedisCluster(), executorService);
messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDbClient(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
reportMessageManager = mock(ReportMessageManager.class);
account = mock(Account.class);
device = mock(Device.class);

View File

@@ -823,7 +823,7 @@ public class WebSocketConnectionTest {
}
private OutgoingMessageEntity createMessage(long id, boolean cached, String sender, UUID senderUuid, long timestamp, boolean receipt, String content) {
return new OutgoingMessageEntity(id, cached, UUID.randomUUID(), receipt ? Envelope.Type.RECEIPT_VALUE : Envelope.Type.CIPHERTEXT_VALUE,
return new OutgoingMessageEntity(id, cached, UUID.randomUUID(), receipt ? Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE : Envelope.Type.CIPHERTEXT_VALUE,
null, timestamp, sender, senderUuid, 1, content.getBytes(), null, 0);
}

View File

@@ -1,11 +0,0 @@
{
"online": false,
"messages": [
{
"type": 1,
"destinationDeviceId": 1,
"body": "Zm9vYmFyego",
"timestamp": 1234
}
]
}

View File

@@ -1,11 +0,0 @@
{
"messages": [
{
"type": 1,
"destinationDeviceId": 1,
"body": "Zm9vYmFyego",
"timestamp": 1234,
"online": false
}
]
}

View File

@@ -1,11 +0,0 @@
{
"online": true,
"messages": [
{
"type": 1,
"destinationDeviceId": 1,
"body": "Zm9vYmFyego",
"timestamp": 1234
}
]
}

View File

@@ -1,11 +0,0 @@
{
"messages": [
{
"type": 1,
"destinationDeviceId": 1,
"body": "Zm9vYmFyego",
"timestamp": 1234,
"online": true
}
]
}

View File

@@ -5,12 +5,10 @@
<parent>
<artifactId>TextSecureServer</artifactId>
<groupId>org.whispersystems.textsecure</groupId>
<version>1.0</version>
<version>JGITVER</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>websocket-resources</artifactId>
<version>${TextSecureServer.version}</version>
<dependencies>
<dependency>
@@ -85,7 +83,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.6.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -103,7 +100,6 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
</dependencies>