mirror of
https://github.com/signalapp/Signal-Server.git
synced 2025-12-15 02:00:48 +00:00
Compare commits
42 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b41ed9d810 | ||
|
|
58d3a12eff | ||
|
|
88c4b2be97 | ||
|
|
6cbd57f19f | ||
|
|
5522376584 | ||
|
|
5089c37d28 | ||
|
|
1ccf24e68c | ||
|
|
411f7298f2 | ||
|
|
5b0214c6f2 | ||
|
|
735573e61b | ||
|
|
c545cff1b3 | ||
|
|
cbd9681e3e | ||
|
|
ca876e40ca | ||
|
|
76f5a71727 | ||
|
|
117de2382d | ||
|
|
25e7036451 | ||
|
|
3131bd3dd9 | ||
|
|
1cf9397bbd | ||
|
|
c97be15e79 | ||
|
|
164fc40990 | ||
|
|
6456af6284 | ||
|
|
81212cc13a | ||
|
|
6f0750790c | ||
|
|
3e61b5c49d | ||
|
|
50c4df4f45 | ||
|
|
1eb946f5fe | ||
|
|
7bd402b48d | ||
|
|
90444d5b91 | ||
|
|
5ee093f87c | ||
|
|
623743286c | ||
|
|
67067f1d2d | ||
|
|
07f9bb112e | ||
|
|
417d48c452 | ||
|
|
358412c78a | ||
|
|
215621a9b0 | ||
|
|
c3f53c4dd9 | ||
|
|
01514f83a0 | ||
|
|
c10b64c367 | ||
|
|
722055c8b5 | ||
|
|
680e501f83 | ||
|
|
f13f7a5ff4 | ||
|
|
5290656c3b |
11
.gitignore
vendored
11
.gitignore
vendored
@@ -9,10 +9,13 @@ config/production.yml
|
||||
config/federated.yml
|
||||
config/staging.yml
|
||||
config/testing.yml
|
||||
service/config/production.yml
|
||||
service/config/federated.yml
|
||||
service/config/staging.yml
|
||||
service/config/testing.yml
|
||||
config/deploy.properties
|
||||
/service/config/production.yml
|
||||
/service/config/federated.yml
|
||||
/service/config/staging.yml
|
||||
/service/config/testing.yml
|
||||
/service/config/deploy.properties
|
||||
/service/dependency-reduced-pom.xml
|
||||
.opsmanage
|
||||
put.sh
|
||||
deployer-staging.properties
|
||||
|
||||
9
.mvn/extensions.xml
Normal file
9
.mvn/extensions.xml
Normal file
@@ -0,0 +1,9 @@
|
||||
<extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
|
||||
<extension>
|
||||
<groupId>fr.brouillard.oss</groupId>
|
||||
<artifactId>jgitver-maven-plugin</artifactId>
|
||||
<version>1.7.1</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
14
.mvn/jgitver.config.xml
Normal file
14
.mvn/jgitver.config.xml
Normal file
@@ -0,0 +1,14 @@
|
||||
<configuration xmlns="http://jgitver.github.io/maven/configuration/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://jgitver.github.io/maven/configuration/1.1.0 https://jgitver.github.io/maven/configuration/jgitver-configuration-v1_1_0.xsd">
|
||||
<useDirty>true</useDirty>
|
||||
<useDefaultBranchingPolicy>false</useDefaultBranchingPolicy>
|
||||
<branchPolicies>
|
||||
<branchPolicy>
|
||||
<pattern>(.*)</pattern>
|
||||
<transformations>
|
||||
<transformation>IGNORE</transformation>
|
||||
</transformations>
|
||||
</branchPolicy>
|
||||
</branchPolicies>
|
||||
</configuration>
|
||||
@@ -5,18 +5,15 @@
|
||||
<parent>
|
||||
<artifactId>TextSecureServer</artifactId>
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<version>1.0</version>
|
||||
<version>JGITVER</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>gcm-sender-async</artifactId>
|
||||
<version>${TextSecureServer.version}</version>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.github.resilience4j</groupId>
|
||||
<artifactId>resilience4j-retry</artifactId>
|
||||
<version>${resilience4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
@@ -44,6 +41,11 @@
|
||||
<artifactId>jcl-over-slf4j</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-nop</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
|
||||
220
pom.xml
220
pom.xml
@@ -4,9 +4,6 @@
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<packaging>pom</packaging>
|
||||
<prerequisites>
|
||||
<maven>3.0.0</maven>
|
||||
</prerequisites>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
@@ -32,20 +29,36 @@
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<dropwizard.version>2.0.21</dropwizard.version>
|
||||
<aws.sdk.version>1.11.939</aws.sdk.version>
|
||||
<mockito.version>2.25.1</mockito.version>
|
||||
<aws.sdk2.version>2.16.66</aws.sdk2.version>
|
||||
<commons-codec.version>1.15</commons-codec.version>
|
||||
<commons-csv.version>1.8</commons-csv.version>
|
||||
<commons-io.version>2.9.0</commons-io.version>
|
||||
<dropwizard.version>2.0.22</dropwizard.version>
|
||||
<guava.version>30.1.1-jre</guava.version>
|
||||
<jaxb.version>2.3.1</jaxb.version>
|
||||
<jedis.version>2.9.0</jedis.version>
|
||||
<lettuce.version>6.0.4.RELEASE</lettuce.version>
|
||||
<libphonenumber.version>8.12.23</libphonenumber.version>
|
||||
<logstash.logback.version>6.6</logstash.logback.version>
|
||||
<micrometer.version>1.5.3</micrometer.version>
|
||||
<mockito.version>2.25.1</mockito.version>
|
||||
<netty.version>4.1.65.Final</netty.version>
|
||||
<netty.tcnative-boringssl-static.version>2.0.39.Final</netty.tcnative-boringssl-static.version>
|
||||
<opentest4j.version>1.2.0</opentest4j.version>
|
||||
<postgresql.version>9.4-1201-jdbc41</postgresql.version>
|
||||
<protobuf.version>3.17.1</protobuf.version>
|
||||
<pushy.version>0.14.2</pushy.version>
|
||||
<resilience4j.version>1.5.0</resilience4j.version>
|
||||
<semver4j.version>3.1.0</semver4j.version>
|
||||
<slf4j.version>1.7.30</slf4j.version>
|
||||
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<TextSecureServer.version>5.91</TextSecureServer.version>
|
||||
</properties>
|
||||
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<artifactId>TextSecureServer</artifactId>
|
||||
<version>1.0</version>
|
||||
<version>JGITVER</version>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
@@ -56,6 +69,13 @@
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-bom</artifactId>
|
||||
<version>${netty.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-bom</artifactId>
|
||||
@@ -63,6 +83,13 @@
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>bom</artifactId>
|
||||
<version>${aws.sdk2.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.github.resilience4j</groupId>
|
||||
<artifactId>resilience4j-bom</artifactId>
|
||||
@@ -77,6 +104,119 @@
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.eatthepath</groupId>
|
||||
<artifactId>pushy</artifactId>
|
||||
<version>${pushy.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.eatthepath</groupId>
|
||||
<artifactId>pushy-dropwizard-metrics-listener</artifactId>
|
||||
<version>${pushy.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>${guava.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>${protobuf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.googlecode.libphonenumber</groupId>
|
||||
<artifactId>libphonenumber</artifactId>
|
||||
<version>${libphonenumber.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.vdurmont</groupId>
|
||||
<artifactId>semver4j</artifactId>
|
||||
<version>${semver4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>${commons-codec.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>${commons-io.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.lettuce</groupId>
|
||||
<artifactId>lettuce-core</artifactId>
|
||||
<version>${lettuce.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-tcnative-boringssl-static</artifactId>
|
||||
<version>${netty.tcnative-boringssl-static.version}</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.xml.bind</groupId>
|
||||
<artifactId>jaxb-api</artifactId>
|
||||
<version>${jaxb.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.logstash.logback</groupId>
|
||||
<artifactId>logstash-logback-encoder</artifactId>
|
||||
<version>${logstash.logback.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-csv</artifactId>
|
||||
<version>${commons-csv.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jaxb</groupId>
|
||||
<artifactId>jaxb-runtime</artifactId>
|
||||
<version>${jaxb.version}</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<version>${mockito.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-inline</artifactId>
|
||||
<version>${mockito.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.opentest4j</groupId>
|
||||
<artifactId>opentest4j</artifactId>
|
||||
<version>${opentest4j.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>${postgresql.version}</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-nop</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>redis.clients</groupId>
|
||||
<artifactId>jedis</artifactId>
|
||||
<version>${jedis.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
@@ -123,20 +263,46 @@
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<extensions>
|
||||
<extension>
|
||||
<groupId>kr.motd.maven</groupId>
|
||||
<artifactId>os-maven-plugin</artifactId>
|
||||
<version>1.7.0</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
<plugins>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.xolstice.maven.plugins</groupId>
|
||||
<artifactId>protobuf-maven-plugin</artifactId>
|
||||
<version>0.6.1</version>
|
||||
<configuration>
|
||||
<protocArtifact>com.google.protobuf:protoc:3.17.2:exe:${os.detected.classifier}</protocArtifact>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
<goal>test-compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.8.0</version>
|
||||
<version>3.8.1</version>
|
||||
<configuration>
|
||||
<source>11</source>
|
||||
<target>11</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.1.1</version>
|
||||
<version>3.2.0</version>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
@@ -183,11 +349,39 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
<version>1.4.1</version>
|
||||
<version>3.0.0-M3</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>enforce</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<dependencyConvergence/>
|
||||
<requireMavenVersion>
|
||||
<version>3.0.0</version>
|
||||
</requireMavenVersion>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-install-plugin</artifactId>
|
||||
<version>3.0.0-M1</version>
|
||||
<configuration>
|
||||
<rules>
|
||||
<dependencyConvergence/>
|
||||
</rules>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<version>3.0.0-M1</version>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
|
||||
@@ -5,12 +5,10 @@
|
||||
<parent>
|
||||
<artifactId>TextSecureServer</artifactId>
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<version>1.0</version>
|
||||
<version>JGITVER</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>redis-dispatch</artifactId>
|
||||
<version>${TextSecureServer.version}</version>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
|
||||
<id>bin</id>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<formats>
|
||||
@@ -18,8 +18,8 @@
|
||||
<directory>${project.build.directory}</directory>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<includes>
|
||||
<include>${parent.artifactId}-${TextSecureServer.version}.jar</include>
|
||||
<include>${parent.artifactId}-${project.version}.jar</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
</assembly>
|
||||
</assembly>
|
||||
|
||||
@@ -21,9 +21,6 @@ twilio: # Twilio gateway configuration
|
||||
push:
|
||||
queueSize: # Size of push pending queue
|
||||
|
||||
redphone:
|
||||
authKey: # Deprecated
|
||||
|
||||
turn: # TURN server configuration
|
||||
secret: # TURN server secret
|
||||
uris:
|
||||
@@ -36,6 +33,23 @@ cacheCluster: # Redis server configuration for cache cluster
|
||||
urls:
|
||||
- redis://redis.example.com:6379/
|
||||
|
||||
clientPresenceCluster: # Redis server configuration for client presence cluster
|
||||
urls:
|
||||
- redis://redis.example.com:6379/
|
||||
|
||||
pubsub: # Redis server configuration for pubsub cluster
|
||||
url: redis://redis.example.com:6379/
|
||||
replicaUrls:
|
||||
- redis://redis.example.com:6379/
|
||||
|
||||
pushSchedulerCluster: # Redis server configuration for push scheduler cluster
|
||||
urls:
|
||||
- redis://redis.example.com:6379/
|
||||
|
||||
rateLimitersCluster: # Redis server configuration for rate limiters cluster
|
||||
urls:
|
||||
- redis://redis.example.com:6379/
|
||||
|
||||
directory:
|
||||
client: # Configuration for interfacing with Contact Discovery Service cluster
|
||||
userAuthenticationTokenSharedSecret: # hex-encoded secret shared with CDS used to generate auth tokens for Signal users
|
||||
@@ -43,13 +57,13 @@ directory:
|
||||
sqs:
|
||||
accessKey: # AWS SQS accessKey
|
||||
accessSecret: # AWS SQS accessSecret
|
||||
queueUrl: # AWS SQS queue url
|
||||
server:
|
||||
replicationUrl: # CDS replication endpoint base url
|
||||
replicationPassword: # CDS replication endpoint password
|
||||
replicationCaCertificate: # CDS replication endpoint TLS certificate trust root
|
||||
reconciliationChunkSize: # CDS reconciliation chunk size
|
||||
reconciliationChunkIntervalMs: # CDS reconciliation chunk interval, in milliseconds
|
||||
queueUrls: # AWS SQS queue urls
|
||||
- https://sqs.example.com/directory.fifo
|
||||
server: # One or more CDS servers
|
||||
- replicationName: # CDS replication name
|
||||
replicationUrl: # CDS replication endpoint base url
|
||||
replicationPassword: # CDS replication endpoint password
|
||||
replicationCaCertificate: # CDS replication endpoint TLS certificate trust root
|
||||
|
||||
messageCache: # Redis server configuration for message store cache
|
||||
persistDelayMinutes:
|
||||
@@ -58,16 +72,39 @@ messageCache: # Redis server configuration for message store cache
|
||||
urls:
|
||||
- redis://redis.example.com:6379/
|
||||
|
||||
messageStore: # Postgresql database configuration for message store
|
||||
driverClass: org.postgresql.Driver
|
||||
user:
|
||||
password:
|
||||
url:
|
||||
|
||||
metricsCluster:
|
||||
urls:
|
||||
- redis://redis.example.com:6379/
|
||||
|
||||
messageDynamoDb: # DynamoDB table configuration
|
||||
region:
|
||||
tableName:
|
||||
|
||||
keysDynamoDb: # DynamoDB table configuration
|
||||
region:
|
||||
tableName:
|
||||
|
||||
accountsDynamoDb: # DynamoDB table configuration
|
||||
region:
|
||||
tableName:
|
||||
phoneNumberTableName:
|
||||
|
||||
migrationDeletedAccountsDynamoDb: # DynamoDB table configuration
|
||||
region:
|
||||
tableName:
|
||||
|
||||
migrationRetryAccountsDynamoDb: # DynamoDB table configuration
|
||||
region:
|
||||
tableName:
|
||||
|
||||
pushChallengeDynamoDb: # DynamoDB table configuration
|
||||
region:
|
||||
tableName:
|
||||
|
||||
reportMessageDynamoDb: # DynamoDB table configuration
|
||||
region:
|
||||
tableName:
|
||||
|
||||
awsAttachments: # AWS S3 configuration
|
||||
accessKey:
|
||||
accessSecret:
|
||||
@@ -81,18 +118,22 @@ gcpAttachments: # GCP Storage configuration
|
||||
pathPrefix:
|
||||
rsaSigningKey:
|
||||
|
||||
profiles: # AWS S3 configuration
|
||||
accessKey:
|
||||
accessSecret:
|
||||
bucket:
|
||||
region:
|
||||
|
||||
database: # Postgresql database configuration
|
||||
abuseDatabase: # Postgresql database configuration
|
||||
driverClass: org.postgresql.Driver
|
||||
user:
|
||||
password:
|
||||
url:
|
||||
|
||||
accountsDatabase: # Postgresql database configuration
|
||||
driverClass: org.postgresql.Driver
|
||||
user:
|
||||
password:
|
||||
url:
|
||||
|
||||
accountDatabaseCrawler:
|
||||
chunkSize: # accounts per run
|
||||
chunkIntervalMs: # time per run
|
||||
|
||||
apn: # Apple Push Notifications configuration
|
||||
sandbox: true
|
||||
bundleId:
|
||||
@@ -104,11 +145,52 @@ gcm: # GCM Configuration
|
||||
senderId:
|
||||
apiKey:
|
||||
|
||||
micrometer: # Micrometer metrics config
|
||||
- name: "example"
|
||||
- uri: "https://metrics.example.com/"
|
||||
- apiKey:
|
||||
- accountId:
|
||||
cdn:
|
||||
accessKey: # AWS Access Key ID
|
||||
accessSecret: # AWS Access Secret
|
||||
bucket: # S3 Bucket name
|
||||
region: # AWS region
|
||||
|
||||
wavefront: # Wavefront micrometer metrics config
|
||||
uri: # Wavefront proxy endpoint
|
||||
batchSize: # Number of measurements to send per request
|
||||
|
||||
datadog:
|
||||
apiKey:
|
||||
environment:
|
||||
|
||||
unidentifiedDelivery:
|
||||
certificate:
|
||||
privateKey:
|
||||
expiresDays:
|
||||
|
||||
voiceVerification:
|
||||
url: https://cdn-ca.signal.org/verification/
|
||||
locales:
|
||||
- en
|
||||
|
||||
recaptcha:
|
||||
secret:
|
||||
|
||||
storageService:
|
||||
uri:
|
||||
userAuthenticationTokenSharedSecret:
|
||||
storageCaCertificate:
|
||||
|
||||
backupService:
|
||||
uri:
|
||||
userAuthenticationTokenSharedSecret:
|
||||
backupCaCertificate:
|
||||
|
||||
zkConfig:
|
||||
serverPublic:
|
||||
serverSecret:
|
||||
enabled:
|
||||
|
||||
appConfig:
|
||||
application:
|
||||
environment:
|
||||
configuration:
|
||||
|
||||
remoteConfig:
|
||||
authorizedTokens:
|
||||
@@ -118,9 +200,22 @@ remoteConfig:
|
||||
- # Nth authorized token
|
||||
globalConfig: # keys and values that are given to clients on GET /v1/config
|
||||
|
||||
paymentService:
|
||||
|
||||
paymentsService:
|
||||
userAuthenticationTokenSharedSecret: # hex-encoded 32-byte secret shared with MobileCoin services used to generate auth tokens for Signal users
|
||||
|
||||
torExitNodeList:
|
||||
s3Region:
|
||||
s3Bucket:
|
||||
objectKey:
|
||||
maxSize:
|
||||
|
||||
asnTable:
|
||||
s3Region:
|
||||
s3Bucket:
|
||||
objectKey:
|
||||
maxSize:
|
||||
|
||||
donation:
|
||||
uri: # value
|
||||
apiKey: # value
|
||||
|
||||
142
service/pom.xml
142
service/pom.xml
@@ -5,16 +5,10 @@
|
||||
<parent>
|
||||
<artifactId>TextSecureServer</artifactId>
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<version>1.0</version>
|
||||
<version>JGITVER</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>service</artifactId>
|
||||
<version>${TextSecureServer.version}</version>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -33,29 +27,28 @@
|
||||
<dependency>
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<artifactId>redis-dispatch</artifactId>
|
||||
<version>${TextSecureServer.version}</version>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<artifactId>websocket-resources</artifactId>
|
||||
<version>${TextSecureServer.version}</version>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<artifactId>gcm-sender-async</artifactId>
|
||||
<version>${TextSecureServer.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.signal</groupId>
|
||||
<artifactId>zkgroup-java</artifactId>
|
||||
<version>0.7.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.whispersystems</groupId>
|
||||
<artifactId>curve25519-java</artifactId>
|
||||
<version>0.5.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.dropwizard</groupId>
|
||||
@@ -134,7 +127,6 @@
|
||||
<dependency>
|
||||
<groupId>net.logstash.logback</groupId>
|
||||
<artifactId>logstash-logback-encoder</artifactId>
|
||||
<version>6.6</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -178,8 +170,6 @@
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jaxb</groupId>
|
||||
<artifactId>jaxb-runtime</artifactId>
|
||||
<version>2.3.1</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -204,17 +194,10 @@
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.13</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-csv</artifactId>
|
||||
<version>1.8</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.vavr</groupId>
|
||||
<artifactId>vavr</artifactId>
|
||||
<version>0.10.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -225,12 +208,6 @@
|
||||
<dependency>
|
||||
<groupId>io.github.resilience4j</groupId>
|
||||
<artifactId>resilience4j-circuitbreaker</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.github.resilience4j</groupId>
|
||||
@@ -245,6 +222,10 @@
|
||||
<groupId>io.micrometer</groupId>
|
||||
<artifactId>micrometer-registry-wavefront</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.micrometer</groupId>
|
||||
<artifactId>micrometer-registry-datadog</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
@@ -271,6 +252,14 @@
|
||||
<artifactId>jackson-jaxrs-json-provider</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>dynamodb</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-core</artifactId>
|
||||
@@ -287,84 +276,56 @@
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-appconfig</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-dynamodb</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>redis.clients</groupId>
|
||||
<artifactId>jedis</artifactId>
|
||||
<version>2.9.0</version>
|
||||
<type>jar</type>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.lettuce</groupId>
|
||||
<artifactId>lettuce-core</artifactId>
|
||||
<version>6.0.4.RELEASE</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>9.4-1201-jdbc41</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.whispersystems</groupId>
|
||||
<artifactId>curve25519-java</artifactId>
|
||||
<version>0.5.0</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.eatthepath</groupId>
|
||||
<artifactId>pushy</artifactId>
|
||||
<version>${pushy.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.eatthepath</groupId>
|
||||
<artifactId>pushy-dropwizard-metrics-listener</artifactId>
|
||||
<version>${pushy.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>io.dropwizard.metrics</groupId>
|
||||
<artifactId>metrics-core</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-tcnative-boringssl-static</artifactId>
|
||||
<version>2.0.34.Final</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.vdurmont</groupId>
|
||||
<artifactId>semver4j</artifactId>
|
||||
<version>3.1.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>30.1.1-jre</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>2.6.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.googlecode.libphonenumber</groupId>
|
||||
<artifactId>libphonenumber</artifactId>
|
||||
<version>8.12.21</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -466,12 +427,12 @@
|
||||
|
||||
|
||||
<build>
|
||||
<finalName>${parent.artifactId}-${TextSecureServer.version}</finalName>
|
||||
<finalName>${project.parent.artifactId}-${project.version}</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>1.6</version>
|
||||
<version>3.2.4</version>
|
||||
<configuration>
|
||||
<createDependencyReducedPom>true</createDependencyReducedPom>
|
||||
<filters>
|
||||
@@ -504,8 +465,9 @@
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4</version>
|
||||
<version>3.3.0</version>
|
||||
<configuration>
|
||||
<descriptors>
|
||||
<descriptor>assembly.xml</descriptor>
|
||||
@@ -521,8 +483,58 @@
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>properties-maven-plugin</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>read-deploy-configuration</id>
|
||||
<phase>deploy</phase>
|
||||
<goals>
|
||||
<goal>read-project-properties</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<files>${project.basedir}/config/deploy.properties</files>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>com.bazaarvoice.maven.plugins</groupId>
|
||||
<artifactId>s3-upload-maven-plugin</artifactId>
|
||||
<version>1.5</version>
|
||||
<configuration>
|
||||
<source>${project.build.directory}/${project.build.finalName}-bin.tar.gz</source>
|
||||
<bucketName>${deploy.bucketName}</bucketName>
|
||||
<destination>${project.build.finalName}-bin.tar.gz</destination>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>deploy-to-s3</id>
|
||||
<phase>deploy</phase>
|
||||
<goals>
|
||||
<goal>s3-upload</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>templating-maven-plugin</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>filter-src</id>
|
||||
<goals>
|
||||
<goal>filter-sources</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
|
||||
all:
|
||||
protoc --java_out=../src/main/java/ TextSecure.proto PubSubMessage.proto
|
||||
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
* Copyright 2013-2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm;
|
||||
|
||||
public class WhisperServerVersion {
|
||||
|
||||
private static final String VERSION = "${project.version}";
|
||||
|
||||
public static String getServerVersion() {
|
||||
return VERSION;
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import org.whispersystems.textsecuregcm.configuration.AppConfigConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.AwsAttachmentsConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.DirectoryConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.DonationConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.DynamoDbConfiguration;
|
||||
@@ -29,7 +30,7 @@ import org.whispersystems.textsecuregcm.configuration.GcpAttachmentsConfiguratio
|
||||
import org.whispersystems.textsecuregcm.configuration.MaxDeviceConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.MessageCacheConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.MessageDynamoDbConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.MicrometerConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.WavefrontConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.PaymentsServiceConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.PushConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration;
|
||||
@@ -79,7 +80,12 @@ public class WhisperServerConfiguration extends Configuration {
|
||||
@NotNull
|
||||
@Valid
|
||||
@JsonProperty
|
||||
private MicrometerConfiguration micrometer;
|
||||
private WavefrontConfiguration wavefront;
|
||||
|
||||
@NotNull
|
||||
@Valid
|
||||
@JsonProperty
|
||||
private DatadogConfiguration datadog;
|
||||
|
||||
@NotNull
|
||||
@Valid
|
||||
@@ -393,8 +399,12 @@ public class WhisperServerConfiguration extends Configuration {
|
||||
return cdn;
|
||||
}
|
||||
|
||||
public MicrometerConfiguration getMicrometerConfiguration() {
|
||||
return micrometer;
|
||||
public WavefrontConfiguration getWavefrontConfiguration() {
|
||||
return wavefront;
|
||||
}
|
||||
|
||||
public DatadogConfiguration getDatadogConfiguration() {
|
||||
return datadog;
|
||||
}
|
||||
|
||||
public UnidentifiedDeliveryConfiguration getDeliveryCertificate() {
|
||||
|
||||
@@ -6,17 +6,10 @@ package org.whispersystems.textsecuregcm;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.codahale.metrics.SharedMetricRegistries;
|
||||
@@ -39,16 +32,24 @@ import io.dropwizard.setup.Bootstrap;
|
||||
import io.dropwizard.setup.Environment;
|
||||
import io.lettuce.core.resource.ClientResources;
|
||||
import io.micrometer.core.instrument.Clock;
|
||||
import io.micrometer.core.instrument.Meter.Id;
|
||||
import io.micrometer.core.instrument.Metrics;
|
||||
import io.micrometer.core.instrument.Tags;
|
||||
import io.micrometer.core.instrument.config.MeterFilter;
|
||||
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
|
||||
import io.micrometer.datadog.DatadogConfig;
|
||||
import io.micrometer.datadog.DatadogMeterRegistry;
|
||||
import io.micrometer.wavefront.WavefrontConfig;
|
||||
import io.micrometer.wavefront.WavefrontMeterRegistry;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.net.http.HttpClient;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
@@ -65,6 +66,8 @@ import org.jdbi.v3.core.Jdbi;
|
||||
import org.signal.zkgroup.ServerSecretParams;
|
||||
import org.signal.zkgroup.auth.ServerZkAuthOperations;
|
||||
import org.signal.zkgroup.profiles.ServerZkProfileOperations;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.whispersystems.dispatch.DispatchManager;
|
||||
import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
|
||||
import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
|
||||
@@ -119,6 +122,7 @@ import org.whispersystems.textsecuregcm.metrics.FreeMemoryGauge;
|
||||
import org.whispersystems.textsecuregcm.metrics.GarbageCollectionGauges;
|
||||
import org.whispersystems.textsecuregcm.metrics.MaxFileDescriptorGauge;
|
||||
import org.whispersystems.textsecuregcm.metrics.MetricsApplicationEventListener;
|
||||
import org.whispersystems.textsecuregcm.metrics.MetricsRequestEventListener;
|
||||
import org.whispersystems.textsecuregcm.metrics.NetworkReceivedGauge;
|
||||
import org.whispersystems.textsecuregcm.metrics.NetworkSentGauge;
|
||||
import org.whispersystems.textsecuregcm.metrics.NstatCounters;
|
||||
@@ -188,32 +192,37 @@ import org.whispersystems.textsecuregcm.storage.Usernames;
|
||||
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
||||
import org.whispersystems.textsecuregcm.util.AsnManager;
|
||||
import org.whispersystems.textsecuregcm.util.Constants;
|
||||
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
|
||||
import org.whispersystems.textsecuregcm.util.TorExitNodeManager;
|
||||
import org.whispersystems.textsecuregcm.websocket.AuthenticatedConnectListener;
|
||||
import org.whispersystems.textsecuregcm.websocket.DeadLetterHandler;
|
||||
import org.whispersystems.textsecuregcm.websocket.ProvisioningConnectListener;
|
||||
import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator;
|
||||
import org.whispersystems.textsecuregcm.workers.CertificateCommand;
|
||||
import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand;
|
||||
import org.whispersystems.textsecuregcm.workers.DeleteUserCommand;
|
||||
import org.whispersystems.textsecuregcm.workers.GetRedisCommandStatsCommand;
|
||||
import org.whispersystems.textsecuregcm.workers.GetRedisSlowlogCommand;
|
||||
import org.whispersystems.textsecuregcm.workers.ServerVersionCommand;
|
||||
import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask;
|
||||
import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask;
|
||||
import org.whispersystems.textsecuregcm.workers.VacuumCommand;
|
||||
import org.whispersystems.textsecuregcm.workers.ZkParamsCommand;
|
||||
import org.whispersystems.websocket.WebSocketResourceProviderFactory;
|
||||
import org.whispersystems.websocket.setup.WebSocketEnvironment;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
|
||||
public class WhisperServerService extends Application<WhisperServerConfiguration> {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(WhisperServerService.class);
|
||||
|
||||
@Override
|
||||
public void initialize(Bootstrap<WhisperServerConfiguration> bootstrap) {
|
||||
bootstrap.addCommand(new VacuumCommand());
|
||||
bootstrap.addCommand(new DeleteUserCommand());
|
||||
bootstrap.addCommand(new CertificateCommand());
|
||||
bootstrap.addCommand(new ZkParamsCommand());
|
||||
bootstrap.addCommand(new GetRedisSlowlogCommand());
|
||||
bootstrap.addCommand(new GetRedisCommandStatsCommand());
|
||||
bootstrap.addCommand(new ServerVersionCommand());
|
||||
bootstrap.addCommand(new CheckDynamicConfigurationCommand());
|
||||
|
||||
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("accountdb", "accountsdb.xml") {
|
||||
@Override
|
||||
@@ -239,8 +248,13 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||
@Override
|
||||
public void run(WhisperServerConfiguration config, Environment environment)
|
||||
throws Exception {
|
||||
|
||||
SharedMetricRegistries.add(Constants.METRICS_NAME, environment.metrics());
|
||||
|
||||
final DistributionStatisticConfig defaultDistributionStatisticConfig = DistributionStatisticConfig.builder()
|
||||
.percentiles(.75, .95, .99, .999)
|
||||
.build();
|
||||
|
||||
final WavefrontConfig wavefrontConfig = new WavefrontConfig() {
|
||||
@Override
|
||||
public String get(final String key) {
|
||||
@@ -249,25 +263,80 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||
|
||||
@Override
|
||||
public String uri() {
|
||||
return config.getMicrometerConfiguration().getUri();
|
||||
return config.getWavefrontConfiguration().getUri();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int batchSize() {
|
||||
return config.getMicrometerConfiguration().getBatchSize();
|
||||
return config.getWavefrontConfiguration().getBatchSize();
|
||||
}
|
||||
};
|
||||
|
||||
Metrics.addRegistry(new WavefrontMeterRegistry(wavefrontConfig, Clock.SYSTEM) {
|
||||
@Override
|
||||
protected DistributionStatisticConfig defaultHistogramConfig() {
|
||||
return DistributionStatisticConfig.builder()
|
||||
.percentiles(.75, .95, .99, .999)
|
||||
.build()
|
||||
.merge(super.defaultHistogramConfig());
|
||||
return defaultDistributionStatisticConfig.merge(super.defaultHistogramConfig());
|
||||
}
|
||||
});
|
||||
|
||||
{
|
||||
final String hostname;
|
||||
|
||||
{
|
||||
String localHostName = "unknown";
|
||||
|
||||
try {
|
||||
localHostName = InetAddress.getLocalHost().getHostName().toLowerCase(Locale.US);
|
||||
log.info("Setting host tag to {}", localHostName);
|
||||
} catch (final UnknownHostException e) {
|
||||
log.warn("Failed to get hostname", e);
|
||||
}
|
||||
|
||||
hostname = localHostName;
|
||||
}
|
||||
|
||||
final DatadogMeterRegistry datadogMeterRegistry = new DatadogMeterRegistry(new DatadogConfig() {
|
||||
@Override
|
||||
public String get(final String key) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String apiKey() {
|
||||
return config.getDatadogConfiguration().getApiKey();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Duration step() {
|
||||
return config.getDatadogConfiguration().getStep();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String hostTag() {
|
||||
return "host";
|
||||
}
|
||||
}, Clock.SYSTEM);
|
||||
|
||||
datadogMeterRegistry.config().commonTags(
|
||||
Tags.of(
|
||||
"service", "chat",
|
||||
"host", hostname,
|
||||
"version", WhisperServerVersion.getServerVersion(),
|
||||
"env", config.getDatadogConfiguration().getEnvironment()))
|
||||
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.REQUEST_COUNTER_NAME))
|
||||
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.ANDROID_REQUEST_COUNTER_NAME))
|
||||
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.DESKTOP_REQUEST_COUNTER_NAME))
|
||||
.meterFilter(MeterFilter.denyNameStartsWith(MetricsRequestEventListener.IOS_REQUEST_COUNTER_NAME))
|
||||
.meterFilter(new MeterFilter() {
|
||||
@Override
|
||||
public DistributionStatisticConfig configure(final Id id, final DistributionStatisticConfig config) {
|
||||
return defaultDistributionStatisticConfig.merge(config);
|
||||
}
|
||||
});
|
||||
|
||||
Metrics.addRegistry(datadogMeterRegistry);
|
||||
}
|
||||
|
||||
environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
environment.getObjectMapper().setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
|
||||
environment.getObjectMapper().setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
|
||||
@@ -279,80 +348,40 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("accounts_database", accountJdbi, config.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
|
||||
FaultTolerantDatabase abuseDatabase = new FaultTolerantDatabase("abuse_database", abuseJdbi, config.getAbuseDatabaseConfiguration().getCircuitBreakerConfiguration());
|
||||
|
||||
AmazonDynamoDBClientBuilder messageDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(config.getMessageDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) config.getMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
DynamoDbClient messageDynamoDb = DynamoDbFromConfig.client(config.getMessageDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
AmazonDynamoDBClientBuilder keysDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(config.getKeysDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getKeysDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) config.getKeysDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
DynamoDbClient preKeyDynamoDb = DynamoDbFromConfig.client(config.getKeysDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
AmazonDynamoDBClientBuilder accountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(config.getAccountsDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) config.getAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(config.getAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
// The thread pool core & max sizes are set via dynamic configuration within AccountsDynamoDb
|
||||
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
|
||||
new LinkedBlockingDeque<>());
|
||||
|
||||
AmazonDynamoDBAsyncClientBuilder accountsDynamoDbAsyncClientBuilder = AmazonDynamoDBAsyncClientBuilder
|
||||
.standard()
|
||||
.withRegion(accountsDynamoDbClientBuilder.getRegion())
|
||||
.withClientConfiguration(accountsDynamoDbClientBuilder.getClientConfiguration())
|
||||
.withCredentials(accountsDynamoDbClientBuilder.getCredentials())
|
||||
.withExecutorFactory(() -> accountsDynamoDbMigrationThreadPool);
|
||||
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(config.getAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
|
||||
accountsDynamoDbMigrationThreadPool);
|
||||
|
||||
AmazonDynamoDBClientBuilder migrationDeletedAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(config.getMigrationDeletedAccountsDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getMigrationDeletedAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) config.getMigrationDeletedAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
DynamoDbClient recentlyDeletedAccountsDynamoDb = DynamoDbFromConfig.client(config.getMigrationDeletedAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
AmazonDynamoDBClientBuilder migrationRetryAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(config.getMigrationRetryAccountsDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getMigrationRetryAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) config.getMigrationRetryAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
DynamoDbClient pushChallengeDynamoDbClient = DynamoDbFromConfig.client(config.getPushChallengeDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
AmazonDynamoDBClientBuilder pushChallengeDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(config.getPushChallengeDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getPushChallengeDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) config.getPushChallengeDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
DynamoDbClient reportMessageDynamoDbClient = DynamoDbFromConfig.client(config.getReportMessageDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
AmazonDynamoDBClientBuilder reportMessageDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(config.getReportMessageDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) config.getReportMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) config.getReportMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
|
||||
DynamoDB messageDynamoDb = new DynamoDB(messageDynamoDbClientBuilder.build());
|
||||
DynamoDB preKeyDynamoDb = new DynamoDB(keysDynamoDbClientBuilder.build());
|
||||
|
||||
AmazonDynamoDB accountsDynamoDbClient = accountsDynamoDbClientBuilder.build();
|
||||
AmazonDynamoDBAsync accountsDynamodbAsyncClient = accountsDynamoDbAsyncClientBuilder.build();
|
||||
|
||||
DynamoDB recentlyDeletedAccountsDynamoDb = new DynamoDB(migrationDeletedAccountsDynamoDbClientBuilder.build());
|
||||
DynamoDB migrationRetryAccountsDynamoDb = new DynamoDB(migrationRetryAccountsDynamoDbClientBuilder.build());
|
||||
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(config.getMigrationRetryAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(recentlyDeletedAccountsDynamoDb, config.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
|
||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, config.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
|
||||
|
||||
Accounts accounts = new Accounts(accountDatabase);
|
||||
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamodbAsyncClient, accountsDynamoDbMigrationThreadPool, new DynamoDB(accountsDynamoDbClient), config.getAccountsDynamoDbConfiguration().getTableName(), config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
|
||||
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, config.getAccountsDynamoDbConfiguration().getTableName(), config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
|
||||
PendingAccounts pendingAccounts = new PendingAccounts(accountDatabase);
|
||||
PendingDevices pendingDevices = new PendingDevices (accountDatabase);
|
||||
Usernames usernames = new Usernames(accountDatabase);
|
||||
@@ -362,8 +391,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb, config.getMessageDynamoDbConfiguration().getTableName(), config.getMessageDynamoDbConfiguration().getTimeToLive());
|
||||
AbusiveHostRules abusiveHostRules = new AbusiveHostRules(abuseDatabase);
|
||||
RemoteConfigs remoteConfigs = new RemoteConfigs(accountDatabase);
|
||||
PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(new DynamoDB(pushChallengeDynamoDbClientBuilder.build()), config.getPushChallengeDynamoDbConfiguration().getTableName());
|
||||
ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(new DynamoDB(reportMessageDynamoDbClientBuilder.build()), config.getReportMessageDynamoDbConfiguration().getTableName());
|
||||
PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(pushChallengeDynamoDbClient, config.getPushChallengeDynamoDbConfiguration().getTableName());
|
||||
ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(reportMessageDynamoDbClient, config.getReportMessageDynamoDbConfiguration().getTableName());
|
||||
|
||||
RedisClientFactory pubSubClientFactory = new RedisClientFactory("pubsub_cache", config.getPubsubCacheConfiguration().getUrl(), config.getPubsubCacheConfiguration().getReplicaUrls(), config.getPubsubCacheConfiguration().getCircuitBreakerConfiguration());
|
||||
ReplicatedJedisPool pubsubClient = pubSubClientFactory.getRedisClientPool();
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2013-2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.configuration;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import javax.validation.constraints.NotBlank;
|
||||
import javax.validation.constraints.NotNull;
|
||||
import java.time.Duration;
|
||||
|
||||
public class DatadogConfiguration {
|
||||
|
||||
@JsonProperty
|
||||
@NotBlank
|
||||
private String apiKey;
|
||||
|
||||
@JsonProperty
|
||||
@NotNull
|
||||
private Duration step = Duration.ofSeconds(10);
|
||||
|
||||
@JsonProperty
|
||||
@NotBlank
|
||||
private String environment;
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public Duration getStep() {
|
||||
return step;
|
||||
}
|
||||
|
||||
public String getEnvironment() {
|
||||
return environment;
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import javax.validation.constraints.Positive;
|
||||
|
||||
public class MicrometerConfiguration {
|
||||
public class WavefrontConfiguration {
|
||||
|
||||
@JsonProperty
|
||||
private String uri;
|
||||
@@ -170,7 +170,8 @@ public class AccountController {
|
||||
@Path("/{type}/preauth/{token}/{number}")
|
||||
public Response getPreAuth(@PathParam("type") String pushType,
|
||||
@PathParam("token") String pushToken,
|
||||
@PathParam("number") String number)
|
||||
@PathParam("number") String number,
|
||||
@QueryParam("voip") Optional<Boolean> useVoip)
|
||||
{
|
||||
if (!"apn".equals(pushType) && !"fcm".equals(pushType)) {
|
||||
return Response.status(400).build();
|
||||
@@ -190,7 +191,7 @@ public class AccountController {
|
||||
if ("fcm".equals(pushType)) {
|
||||
gcmSender.sendMessage(new GcmMessage(pushToken, number, 0, GcmMessage.Type.CHALLENGE, Optional.of(storedVerificationCode.getPushCode())));
|
||||
} else if ("apn".equals(pushType)) {
|
||||
apnSender.sendMessage(new ApnMessage(pushToken, number, 0, true, ApnMessage.Type.CHALLENGE, Optional.of(storedVerificationCode.getPushCode())));
|
||||
apnSender.sendMessage(new ApnMessage(pushToken, number, 0, useVoip.orElse(true), ApnMessage.Type.CHALLENGE, Optional.of(storedVerificationCode.getPushCode())));
|
||||
} else {
|
||||
throw new AssertionError();
|
||||
}
|
||||
|
||||
@@ -234,6 +234,10 @@ public class DeviceController {
|
||||
private boolean isCapabilityDowngrade(Account account, DeviceCapabilities capabilities, String userAgent) {
|
||||
boolean isDowngrade = false;
|
||||
|
||||
if (account.isSenderKeySupported() && !capabilities.isSenderKey()) {
|
||||
isDowngrade = true;
|
||||
}
|
||||
|
||||
if (account.isGv1MigrationSupported() && !capabilities.isGv1Migration()) {
|
||||
isDowngrade = true;
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ import org.whispersystems.textsecuregcm.entities.MultiRecipientMessage.Recipient
|
||||
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntity;
|
||||
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntityList;
|
||||
import org.whispersystems.textsecuregcm.entities.SendMessageResponse;
|
||||
import org.whispersystems.textsecuregcm.entities.SendMultiRecipientMessageResponse;
|
||||
import org.whispersystems.textsecuregcm.entities.StaleDevices;
|
||||
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeException;
|
||||
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
|
||||
@@ -133,6 +134,7 @@ public class MessageController {
|
||||
|
||||
private final ClusterLuaScript recordInternationalUnsealedSenderMetricsScript;
|
||||
|
||||
private static final String LEGACY_MESSAGE_SENT_COUNTER = name(MessageController.class, "legacyMessageSent");
|
||||
private static final String SENT_MESSAGE_COUNTER_NAME = name(MessageController.class, "sentMessages");
|
||||
private static final String REJECT_UNSEALED_SENDER_COUNTER_NAME = name(MessageController.class, "rejectUnsealedSenderLimit");
|
||||
private static final String INTERNATIONAL_UNSEALED_SENDER_COUNTER_NAME = name(MessageController.class, "internationalUnsealedSender");
|
||||
@@ -298,17 +300,8 @@ public class MessageController {
|
||||
validateCompleteDeviceList(destination.get(), messages.getMessages(), isSyncMessage);
|
||||
validateRegistrationIds(destination.get(), messages.getMessages());
|
||||
|
||||
// iOS versions prior to 5.5.0.7 send `online` on IncomingMessageList.message, rather on the top-level entity.
|
||||
// This causes some odd client behaviors, such as persisted typing indicators, so we have a temporary
|
||||
// server-side adaptation.
|
||||
final boolean online = messages.getMessages()
|
||||
.stream()
|
||||
.findFirst()
|
||||
.map(IncomingMessage::isOnline)
|
||||
.orElse(messages.isOnline());
|
||||
|
||||
final List<Tag> tags = List.of(UserAgentTagUtil.getPlatformTag(userAgent),
|
||||
Tag.of(EPHEMERAL_TAG_NAME, String.valueOf(online)),
|
||||
Tag.of(EPHEMERAL_TAG_NAME, String.valueOf(messages.isOnline())),
|
||||
Tag.of(SENDER_TYPE_TAG_NAME, senderType));
|
||||
|
||||
for (IncomingMessage incomingMessage : messages.getMessages()) {
|
||||
@@ -316,7 +309,7 @@ public class MessageController {
|
||||
|
||||
if (destinationDevice.isPresent()) {
|
||||
Metrics.counter(SENT_MESSAGE_COUNTER_NAME, tags).increment();
|
||||
sendMessage(source, destination.get(), destinationDevice.get(), messages.getTimestamp(), online, incomingMessage);
|
||||
sendMessage(source, destination.get(), destinationDevice.get(), messages.getTimestamp(), messages.isOnline(), incomingMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -426,7 +419,7 @@ public class MessageController {
|
||||
uuids404.add(destinationAccount.getUuid());
|
||||
}
|
||||
}
|
||||
return Response.ok(new SendMessageResponse(uuids404)).build();
|
||||
return Response.ok(new SendMultiRecipientMessageResponse(uuids404)).build();
|
||||
}
|
||||
|
||||
private void checkAccessKeys(CombinedUnidentifiedSenderAccessKeys accessKeys, Map<UUID, Account> uuidToAccountMap) {
|
||||
@@ -556,7 +549,7 @@ public class MessageController {
|
||||
account.getAuthenticatedDevice().get().getId(),
|
||||
source, timestamp);
|
||||
|
||||
if (message.isPresent() && message.get().getType() != Envelope.Type.RECEIPT_VALUE) {
|
||||
if (message.isPresent() && message.get().getType() != Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE) {
|
||||
receiptSender.sendReceipt(account,
|
||||
message.get().getSource(),
|
||||
message.get().getTimestamp());
|
||||
@@ -578,7 +571,7 @@ public class MessageController {
|
||||
|
||||
if (message.isPresent()) {
|
||||
WebSocketConnection.recordMessageDeliveryDuration(message.get().getTimestamp(), account.getAuthenticatedDevice().get());
|
||||
if (!Util.isEmpty(message.get().getSource()) && message.get().getType() != Envelope.Type.RECEIPT_VALUE) {
|
||||
if (!Util.isEmpty(message.get().getSource()) && message.get().getType() != Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE) {
|
||||
receiptSender.sendReceipt(account, message.get().getSource(), message.get().getTimestamp());
|
||||
}
|
||||
}
|
||||
@@ -612,7 +605,7 @@ public class MessageController {
|
||||
Optional<byte[]> messageContent = getMessageContent(incomingMessage);
|
||||
Envelope.Builder messageBuilder = Envelope.newBuilder();
|
||||
|
||||
messageBuilder.setType(Envelope.Type.valueOf(incomingMessage.getType()))
|
||||
messageBuilder.setType(Envelope.Type.forNumber(incomingMessage.getType()))
|
||||
.setTimestamp(timestamp == 0 ? System.currentTimeMillis() : timestamp)
|
||||
.setServerTimestamp(System.currentTimeMillis());
|
||||
|
||||
@@ -623,6 +616,7 @@ public class MessageController {
|
||||
}
|
||||
|
||||
if (messageBody.isPresent()) {
|
||||
Metrics.counter(LEGACY_MESSAGE_SENT_COUNTER).increment();
|
||||
messageBuilder.setLegacyMessage(ByteString.copyFrom(messageBody.get()));
|
||||
}
|
||||
|
||||
@@ -647,7 +641,7 @@ public class MessageController {
|
||||
byte[] payload = new byte[1 + recipientKeyMaterial.length + commonPayload.length];
|
||||
payload[0] = MultiRecipientMessageProvider.VERSION;
|
||||
System.arraycopy(recipientKeyMaterial, 0, payload, 1, recipientKeyMaterial.length);
|
||||
System.arraycopy(commonPayload, 0, payload, 1 + recipientKeyMaterial.length, payload.length);
|
||||
System.arraycopy(commonPayload, 0, payload, 1 + recipientKeyMaterial.length, commonPayload.length);
|
||||
|
||||
messageBuilder
|
||||
.setType(Type.UNIDENTIFIED_SENDER)
|
||||
|
||||
@@ -32,9 +32,6 @@ public class IncomingMessage {
|
||||
@JsonProperty
|
||||
private long timestamp; // deprecated
|
||||
|
||||
@JsonProperty
|
||||
private Boolean online; // use IncomingMessageList.online - this is a temporary adaptation for older clients
|
||||
|
||||
public String getDestination() {
|
||||
return destination;
|
||||
}
|
||||
@@ -62,8 +59,4 @@ public class IncomingMessage {
|
||||
public String getContent() {
|
||||
return content;
|
||||
}
|
||||
|
||||
public Boolean isOnline() {
|
||||
return online;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
*/
|
||||
package org.whispersystems.textsecuregcm.entities;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude.Include;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import javax.validation.Valid;
|
||||
@@ -15,6 +17,7 @@ public class IncomingMessageList {
|
||||
@JsonProperty
|
||||
@NotNull
|
||||
@Valid
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
private List<IncomingMessage> messages;
|
||||
|
||||
@JsonProperty
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,24 +6,15 @@
|
||||
package org.whispersystems.textsecuregcm.entities;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public class SendMessageResponse {
|
||||
|
||||
@JsonProperty
|
||||
private boolean needsSync;
|
||||
|
||||
@JsonProperty
|
||||
private List<UUID> uuids404;
|
||||
|
||||
public SendMessageResponse() {}
|
||||
|
||||
public SendMessageResponse(boolean needsSync) {
|
||||
this.needsSync = needsSync;
|
||||
}
|
||||
|
||||
public SendMessageResponse(List<UUID> uuids404) {
|
||||
this.uuids404 = uuids404;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright 2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.entities;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public class SendMultiRecipientMessageResponse {
|
||||
@JsonProperty
|
||||
private List<UUID> uuids404;
|
||||
|
||||
public SendMultiRecipientMessageResponse() {
|
||||
}
|
||||
|
||||
public SendMultiRecipientMessageResponse(final List<UUID> uuids404) {
|
||||
this.uuids404 = uuids404;
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import java.time.Duration;
|
||||
import javax.validation.constraints.NotEmpty;
|
||||
import net.logstash.logback.appender.LogstashTcpSocketAppender;
|
||||
import net.logstash.logback.encoder.LogstashEncoder;
|
||||
import org.whispersystems.textsecuregcm.WhisperServerVersion;
|
||||
|
||||
@JsonTypeName("logstashtcpsocket")
|
||||
public class LogstashTcpSocketAppenderFactory extends AbstractAppenderFactory<ILoggingEvent> {
|
||||
@@ -83,7 +84,8 @@ public class LogstashTcpSocketAppenderFactory extends AbstractAppenderFactory<IL
|
||||
}
|
||||
customFieldsNode.set("service", TextNode.valueOf("chat"));
|
||||
customFieldsNode.set("ddsource", TextNode.valueOf("logstash"));
|
||||
customFieldsNode.set("ddtags", TextNode.valueOf("env:" + environment));
|
||||
customFieldsNode.set("ddtags", TextNode.valueOf("env:" + environment + ",version:" + WhisperServerVersion.getServerVersion()));
|
||||
|
||||
encoder.setCustomFields(customFieldsNode.toString());
|
||||
final LayoutWrappingEncoder<ILoggingEvent> prefix = new LayoutWrappingEncoder<>();
|
||||
final PatternLayout layout = new PatternLayout();
|
||||
|
||||
@@ -29,18 +29,18 @@ import java.util.regex.Pattern;
|
||||
/**
|
||||
* Gathers and reports request-level metrics.
|
||||
*/
|
||||
class MetricsRequestEventListener implements RequestEventListener {
|
||||
public class MetricsRequestEventListener implements RequestEventListener {
|
||||
|
||||
static final String REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "request");
|
||||
static final String PATH_TAG = "path";
|
||||
static final String STATUS_CODE_TAG = "status";
|
||||
static final String TRAFFIC_SOURCE_TAG = "trafficSource";
|
||||
public static final String REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "request");
|
||||
public static final String ANDROID_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "androidRequest");
|
||||
public static final String DESKTOP_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "desktopRequest");
|
||||
public static final String IOS_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "iosRequest");
|
||||
|
||||
static final String ANDROID_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "androidRequest");
|
||||
static final String DESKTOP_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "desktopRequest");
|
||||
static final String IOS_REQUEST_COUNTER_NAME = MetricRegistry.name(MetricsRequestEventListener.class, "iosRequest");
|
||||
static final String OS_TAG = "os";
|
||||
static final String SDK_TAG = "sdkVersion";
|
||||
static final String PATH_TAG = "path";
|
||||
static final String STATUS_CODE_TAG = "status";
|
||||
static final String TRAFFIC_SOURCE_TAG = "trafficSource";
|
||||
static final String OS_TAG = "os";
|
||||
static final String SDK_TAG = "sdkVersion";
|
||||
|
||||
private static final Set<String> ACCEPTABLE_DESKTOP_OS_STRINGS = Set.of("linux", "macos", "windows");
|
||||
|
||||
|
||||
@@ -16,7 +16,8 @@ public class ApnMessage {
|
||||
NOTIFICATION, CHALLENGE, RATE_LIMIT_CHALLENGE
|
||||
}
|
||||
|
||||
public static final String APN_NOTIFICATION_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}}";
|
||||
public static final String APN_VOIP_NOTIFICATION_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}}";
|
||||
public static final String APN_NSE_NOTIFICATION_PAYLOAD = "{\"aps\":{\"mutable-content\":1,\"alert\":{\"loc-key\":\"APN_Message\"}}}";
|
||||
public static final String APN_CHALLENGE_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}, \"challenge\" : \"%s\"}";
|
||||
public static final String APN_RATE_LIMIT_CHALLENGE_PAYLOAD = "{\"aps\":{\"sound\":\"default\",\"alert\":{\"loc-key\":\"APN_Message\"}}, \"rateLimitChallenge\" : \"%s\"}";
|
||||
public static final long MAX_EXPIRATION = Integer.MAX_VALUE * 1000L;
|
||||
@@ -48,7 +49,7 @@ public class ApnMessage {
|
||||
public String getMessage() {
|
||||
switch (type) {
|
||||
case NOTIFICATION:
|
||||
return APN_NOTIFICATION_PAYLOAD;
|
||||
return this.isVoip() ? APN_VOIP_NOTIFICATION_PAYLOAD : APN_NSE_NOTIFICATION_PAYLOAD;
|
||||
|
||||
case CHALLENGE:
|
||||
return String.format(APN_CHALLENGE_PAYLOAD, challengeData.orElseThrow(AssertionError::new));
|
||||
|
||||
@@ -43,7 +43,7 @@ public class ReceiptSender {
|
||||
.setSourceUuid(source.getUuid().toString())
|
||||
.setSourceDevice((int) source.getAuthenticatedDevice().get().getId())
|
||||
.setTimestamp(messageId)
|
||||
.setType(Envelope.Type.RECEIPT);
|
||||
.setType(Envelope.Type.SERVER_DELIVERY_RECEIPT);
|
||||
|
||||
if (source.getRelay().isPresent()) {
|
||||
message.setRelay(source.getRelay().get());
|
||||
|
||||
@@ -5,21 +5,18 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.Page;
|
||||
import com.amazonaws.services.dynamodbv2.document.QueryOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
@@ -29,7 +26,7 @@ import static io.micrometer.core.instrument.Metrics.timer;
|
||||
|
||||
public class AbstractDynamoDbStore {
|
||||
|
||||
private final DynamoDB dynamoDb;
|
||||
private final DynamoDbClient dynamoDbClient;
|
||||
|
||||
private final Timer batchWriteItemsFirstPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "true");
|
||||
private final Timer batchWriteItemsRetryPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "false");
|
||||
@@ -41,44 +38,31 @@ public class AbstractDynamoDbStore {
|
||||
public static final int DYNAMO_DB_MAX_BATCH_SIZE = 25; // This limit comes from Amazon Dynamo DB itself. It will reject batch writes larger than this.
|
||||
public static final int RESULT_SET_CHUNK_SIZE = 100;
|
||||
|
||||
public AbstractDynamoDbStore(final DynamoDB dynamoDb) {
|
||||
this.dynamoDb = dynamoDb;
|
||||
public AbstractDynamoDbStore(final DynamoDbClient dynamoDbClient) {
|
||||
this.dynamoDbClient = dynamoDbClient;
|
||||
}
|
||||
|
||||
protected DynamoDB getDynamoDb() {
|
||||
return dynamoDb;
|
||||
protected DynamoDbClient db() {
|
||||
return dynamoDbClient;
|
||||
}
|
||||
|
||||
protected void executeTableWriteItemsUntilComplete(final TableWriteItems items) {
|
||||
AtomicReference<BatchWriteItemOutcome> outcome = new AtomicReference<>();
|
||||
batchWriteItemsFirstPass.record(() -> outcome.set(dynamoDb.batchWriteItem(items)));
|
||||
protected void executeTableWriteItemsUntilComplete(final Map<String,List<WriteRequest>> items) {
|
||||
AtomicReference<BatchWriteItemResponse> outcome = new AtomicReference<>();
|
||||
batchWriteItemsFirstPass.record(() -> outcome.set(dynamoDbClient.batchWriteItem(BatchWriteItemRequest.builder().requestItems(items).build())));
|
||||
int attemptCount = 0;
|
||||
while (!outcome.get().getUnprocessedItems().isEmpty() && attemptCount < MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE) {
|
||||
batchWriteItemsRetryPass.record(() -> outcome.set(dynamoDb.batchWriteItemUnprocessed(outcome.get().getUnprocessedItems())));
|
||||
while (!outcome.get().unprocessedItems().isEmpty() && attemptCount < MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE) {
|
||||
batchWriteItemsRetryPass.record(() -> outcome.set(dynamoDbClient.batchWriteItem(BatchWriteItemRequest.builder()
|
||||
.requestItems(outcome.get().unprocessedItems())
|
||||
.build())));
|
||||
++attemptCount;
|
||||
}
|
||||
if (!outcome.get().getUnprocessedItems().isEmpty()) {
|
||||
logger.error("Attempt count ({}) reached max ({}}) before applying all batch writes to dynamo. {} unprocessed items remain.", attemptCount, MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE, outcome.get().getUnprocessedItems().size());
|
||||
batchWriteItemsUnprocessed.increment(outcome.get().getUnprocessedItems().size());
|
||||
if (!outcome.get().unprocessedItems().isEmpty()) {
|
||||
int totalItems = outcome.get().unprocessedItems().values().stream().mapToInt(List::size).sum();
|
||||
logger.error("Attempt count ({}) reached max ({}}) before applying all batch writes to dynamo. {} unprocessed items remain.", attemptCount, MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE, totalItems);
|
||||
batchWriteItemsUnprocessed.increment(totalItems);
|
||||
}
|
||||
}
|
||||
|
||||
protected long countItemsMatchingQuery(final Table table, final QuerySpec querySpec) {
|
||||
// This is very confusing, but does appear to be the intended behavior. See:
|
||||
//
|
||||
// - https://github.com/aws/aws-sdk-java/issues/693
|
||||
// - https://github.com/aws/aws-sdk-java/issues/915
|
||||
// - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.Count
|
||||
|
||||
long matchingItems = 0;
|
||||
|
||||
for (final Page<Item, QueryOutcome> page : table.query(querySpec).pages()) {
|
||||
matchingItems += page.getLowLevelResult().getQueryResult().getCount();
|
||||
}
|
||||
|
||||
return matchingItems;
|
||||
}
|
||||
|
||||
static <T> void writeInBatches(final Iterable<T> items, final Consumer<List<T>> action) {
|
||||
final List<T> batch = new ArrayList<>(DYNAMO_DB_MAX_BATCH_SIZE);
|
||||
|
||||
|
||||
@@ -2,25 +2,6 @@ package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
||||
import com.amazonaws.handlers.AsyncHandler;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
|
||||
import com.amazonaws.services.dynamodbv2.model.CancellationReason;
|
||||
import com.amazonaws.services.dynamodbv2.model.Delete;
|
||||
import com.amazonaws.services.dynamodbv2.model.GetItemResult;
|
||||
import com.amazonaws.services.dynamodbv2.model.Put;
|
||||
import com.amazonaws.services.dynamodbv2.model.ReturnValuesOnConditionCheckFailure;
|
||||
import com.amazonaws.services.dynamodbv2.model.TransactWriteItem;
|
||||
import com.amazonaws.services.dynamodbv2.model.TransactWriteItemsRequest;
|
||||
import com.amazonaws.services.dynamodbv2.model.TransactWriteItemsResult;
|
||||
import com.amazonaws.services.dynamodbv2.model.TransactionCanceledException;
|
||||
import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
@@ -33,12 +14,27 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.stream.Collectors;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.CancellationReason;
|
||||
import software.amazon.awssdk.services.dynamodb.model.Delete;
|
||||
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.Put;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ReturnValuesOnConditionCheckFailure;
|
||||
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItem;
|
||||
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.TransactionCanceledException;
|
||||
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
|
||||
|
||||
public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountStore {
|
||||
|
||||
@@ -51,9 +47,8 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
|
||||
static final String ATTR_MIGRATION_VERSION = "V";
|
||||
|
||||
private final AmazonDynamoDB client;
|
||||
private final Table accountsTable;
|
||||
private final AmazonDynamoDBAsync asyncClient;
|
||||
private final DynamoDbClient client;
|
||||
private final DynamoDbAsyncClient asyncClient;
|
||||
|
||||
private final ThreadPoolExecutor migrationThreadPool;
|
||||
|
||||
@@ -61,6 +56,7 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
private final MigrationRetryAccounts migrationRetryAccounts;
|
||||
|
||||
private final String phoneNumbersTableName;
|
||||
private final String accountsTableName;
|
||||
|
||||
private static final Timer CREATE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "create"));
|
||||
private static final Timer UPDATE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "update"));
|
||||
@@ -70,18 +66,17 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(AccountsDynamoDb.class);
|
||||
|
||||
public AccountsDynamoDb(AmazonDynamoDB client, AmazonDynamoDBAsync asyncClient,
|
||||
ThreadPoolExecutor migrationThreadPool, DynamoDB dynamoDb, String accountsTableName, String phoneNumbersTableName,
|
||||
public AccountsDynamoDb(DynamoDbClient client, DynamoDbAsyncClient asyncClient,
|
||||
ThreadPoolExecutor migrationThreadPool, String accountsTableName, String phoneNumbersTableName,
|
||||
MigrationDeletedAccounts migrationDeletedAccounts,
|
||||
MigrationRetryAccounts accountsMigrationErrors) {
|
||||
|
||||
super(dynamoDb);
|
||||
super(client);
|
||||
|
||||
this.client = client;
|
||||
this.accountsTable = dynamoDb.getTable(accountsTableName);
|
||||
this.phoneNumbersTableName = phoneNumbersTableName;
|
||||
|
||||
this.asyncClient = asyncClient;
|
||||
this.phoneNumbersTableName = phoneNumbersTableName;
|
||||
this.accountsTableName = accountsTableName;
|
||||
this.migrationThreadPool = migrationThreadPool;
|
||||
|
||||
this.migrationDeletedAccounts = migrationDeletedAccounts;
|
||||
@@ -90,32 +85,34 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
|
||||
@Override
|
||||
public boolean create(Account account) {
|
||||
|
||||
return CREATE_TIMER.record(() -> {
|
||||
|
||||
try {
|
||||
TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid());
|
||||
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder()
|
||||
.conditionExpression("attribute_not_exists(#number) OR #number = :number")
|
||||
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
|
||||
.expressionAttributeValues(Map.of(":number", AttributeValues.fromString(account.getNumber()))));
|
||||
|
||||
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid());
|
||||
|
||||
final TransactWriteItemsRequest request = new TransactWriteItemsRequest()
|
||||
.withTransactItems(phoneNumberConstraintPut, accountPut);
|
||||
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
|
||||
.transactItems(phoneNumberConstraintPut, accountPut)
|
||||
.build();
|
||||
|
||||
try {
|
||||
client.transactWriteItems(request);
|
||||
} catch (TransactionCanceledException e) {
|
||||
|
||||
final CancellationReason accountCancellationReason = e.getCancellationReasons().get(1);
|
||||
final CancellationReason accountCancellationReason = e.cancellationReasons().get(1);
|
||||
|
||||
if ("ConditionalCheckFailed".equals(accountCancellationReason.getCode())) {
|
||||
if ("ConditionalCheckFailed".equals(accountCancellationReason.code())) {
|
||||
throw new IllegalArgumentException("uuid present with different phone number");
|
||||
}
|
||||
|
||||
final CancellationReason phoneNumberConstraintCancellationReason = e.getCancellationReasons().get(0);
|
||||
final CancellationReason phoneNumberConstraintCancellationReason = e.cancellationReasons().get(0);
|
||||
|
||||
if ("ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.getCode())) {
|
||||
if ("ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.code())) {
|
||||
|
||||
ByteBuffer actualAccountUuid = phoneNumberConstraintCancellationReason.getItem().get(KEY_ACCOUNT_UUID).getB();
|
||||
ByteBuffer actualAccountUuid = phoneNumberConstraintCancellationReason.item().get(KEY_ACCOUNT_UUID).b().asByteBuffer();
|
||||
account.setUuid(UUIDUtil.fromByteBuffer(actualAccountUuid));
|
||||
|
||||
update(account);
|
||||
@@ -134,39 +131,37 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
});
|
||||
}
|
||||
|
||||
private TransactWriteItem buildPutWriteItemForAccount(Account account, UUID uuid) throws JsonProcessingException {
|
||||
return new TransactWriteItem()
|
||||
.withPut(
|
||||
new Put()
|
||||
.withTableName(accountsTable.getTableName())
|
||||
.withItem(Map.of(
|
||||
KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid)),
|
||||
ATTR_ACCOUNT_E164, new AttributeValue(account.getNumber()),
|
||||
ATTR_ACCOUNT_DATA, new AttributeValue()
|
||||
.withB(ByteBuffer.wrap(SystemMapper.getMapper().writeValueAsBytes(account))),
|
||||
ATTR_MIGRATION_VERSION, new AttributeValue().withN(
|
||||
String.valueOf(account.getDynamoDbMigrationVersion()))))
|
||||
.withConditionExpression("attribute_not_exists(#number) OR #number = :number")
|
||||
.withExpressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
|
||||
.withExpressionAttributeValues(Map.of(":number", new AttributeValue(account.getNumber()))));
|
||||
private TransactWriteItem buildPutWriteItemForAccount(Account account, UUID uuid, Put.Builder putBuilder) throws JsonProcessingException {
|
||||
return TransactWriteItem.builder()
|
||||
.put(putBuilder
|
||||
.tableName(accountsTableName)
|
||||
.item(Map.of(
|
||||
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid),
|
||||
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
|
||||
ATTR_ACCOUNT_DATA, AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
|
||||
ATTR_MIGRATION_VERSION, AttributeValues.fromInt(account.getDynamoDbMigrationVersion())))
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
private TransactWriteItem buildPutWriteItemForPhoneNumberConstraint(Account account, UUID uuid) {
|
||||
return new TransactWriteItem()
|
||||
.withPut(
|
||||
new Put()
|
||||
.withTableName(phoneNumbersTableName)
|
||||
.withItem(Map.of(
|
||||
ATTR_ACCOUNT_E164, new AttributeValue(account.getNumber()),
|
||||
KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid))))
|
||||
.withConditionExpression(
|
||||
return TransactWriteItem.builder()
|
||||
.put(
|
||||
Put.builder()
|
||||
.tableName(phoneNumbersTableName)
|
||||
.item(Map.of(
|
||||
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
|
||||
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
|
||||
.conditionExpression(
|
||||
"attribute_not_exists(#number) OR (attribute_exists(#number) AND #uuid = :uuid)")
|
||||
.withExpressionAttributeNames(
|
||||
.expressionAttributeNames(
|
||||
Map.of("#uuid", KEY_ACCOUNT_UUID,
|
||||
"#number", ATTR_ACCOUNT_E164))
|
||||
.withExpressionAttributeValues(
|
||||
Map.of(":uuid", new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid))))
|
||||
.withReturnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD));
|
||||
.expressionAttributeValues(
|
||||
Map.of(":uuid", AttributeValues.fromUUID(uuid)))
|
||||
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -174,16 +169,18 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
UPDATE_TIMER.record(() -> {
|
||||
UpdateItemRequest updateItemRequest;
|
||||
try {
|
||||
updateItemRequest = new UpdateItemRequest()
|
||||
.withTableName(accountsTable.getTableName())
|
||||
.withKey(Map.of(KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(account.getUuid()))))
|
||||
.withUpdateExpression("SET #data = :data, #version = :version")
|
||||
.withConditionExpression("attribute_exists(#number)")
|
||||
.withExpressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
|
||||
updateItemRequest = UpdateItemRequest.builder()
|
||||
.tableName(accountsTableName)
|
||||
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
|
||||
.updateExpression("SET #data = :data, #version = :version")
|
||||
.conditionExpression("attribute_exists(#number)")
|
||||
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
|
||||
"#data", ATTR_ACCOUNT_DATA,
|
||||
"#version", ATTR_MIGRATION_VERSION))
|
||||
.withExpressionAttributeValues(Map.of(":data", new AttributeValue().withB(ByteBuffer.wrap(SystemMapper.getMapper().writeValueAsBytes(account))),
|
||||
":version", new AttributeValue().withN(String.valueOf(account.getDynamoDbMigrationVersion()))));
|
||||
.expressionAttributeValues(Map.of(
|
||||
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
|
||||
":version", AttributeValues.fromInt(account.getDynamoDbMigrationVersion())))
|
||||
.build();
|
||||
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
@@ -193,37 +190,42 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Optional<Account> get(String number) {
|
||||
|
||||
return GET_BY_NUMBER_TIMER.record(() -> {
|
||||
|
||||
final GetItemResult phoneNumberAndUuid = client.getItem(phoneNumbersTableName,
|
||||
Map.of(ATTR_ACCOUNT_E164, new AttributeValue(number)), true);
|
||||
final GetItemResponse response = client.getItem(GetItemRequest.builder()
|
||||
.tableName(phoneNumbersTableName)
|
||||
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(number)))
|
||||
.build());
|
||||
|
||||
return Optional.ofNullable(phoneNumberAndUuid.getItem())
|
||||
.map(item -> item.get(KEY_ACCOUNT_UUID).getB())
|
||||
.map(uuid -> accountsTable.getItem(new GetItemSpec()
|
||||
.withPrimaryKey(KEY_ACCOUNT_UUID, uuid.array())
|
||||
.withConsistentRead(true)))
|
||||
return Optional.ofNullable(response.item())
|
||||
.map(item -> item.get(KEY_ACCOUNT_UUID))
|
||||
.map(uuid -> accountByUuid(uuid))
|
||||
.map(AccountsDynamoDb::fromItem);
|
||||
});
|
||||
}
|
||||
|
||||
private Map<String, AttributeValue> accountByUuid(AttributeValue uuid) {
|
||||
GetItemResponse r = client.getItem(GetItemRequest.builder()
|
||||
.tableName(accountsTableName)
|
||||
.key(Map.of(KEY_ACCOUNT_UUID, uuid))
|
||||
.consistentRead(true)
|
||||
.build());
|
||||
return r.item().isEmpty() ? null : r.item();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Account> get(UUID uuid) {
|
||||
Optional<Item> maybeItem = GET_BY_UUID_TIMER.record(() ->
|
||||
Optional.ofNullable(accountsTable.getItem(new GetItemSpec().
|
||||
withPrimaryKey(new PrimaryKey(KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(uuid)))
|
||||
.withConsistentRead(true))));
|
||||
|
||||
return maybeItem.map(AccountsDynamoDb::fromItem);
|
||||
return GET_BY_UUID_TIMER.record(() ->
|
||||
Optional.ofNullable(accountByUuid(AttributeValues.fromUUID(uuid)))
|
||||
.map(AccountsDynamoDb::fromItem));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(UUID uuid) {
|
||||
DELETE_TIMER.record(() -> {
|
||||
|
||||
delete(uuid, true);
|
||||
});
|
||||
}
|
||||
@@ -238,18 +240,22 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
|
||||
maybeAccount.ifPresent(account -> {
|
||||
|
||||
TransactWriteItem phoneNumberDelete = new TransactWriteItem()
|
||||
.withDelete(new Delete()
|
||||
.withTableName(phoneNumbersTableName)
|
||||
.withKey(Map.of(ATTR_ACCOUNT_E164, new AttributeValue(account.getNumber()))));
|
||||
TransactWriteItem phoneNumberDelete = TransactWriteItem.builder()
|
||||
.delete(Delete.builder()
|
||||
.tableName(phoneNumbersTableName)
|
||||
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber())))
|
||||
.build())
|
||||
.build();
|
||||
|
||||
TransactWriteItem accountDelete = new TransactWriteItem().withDelete(
|
||||
new Delete()
|
||||
.withTableName(accountsTable.getTableName())
|
||||
.withKey(Map.of(KEY_ACCOUNT_UUID, new AttributeValue().withB(UUIDUtil.toByteBuffer(uuid)))));
|
||||
TransactWriteItem accountDelete = TransactWriteItem.builder()
|
||||
.delete(Delete.builder()
|
||||
.tableName(accountsTableName)
|
||||
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
|
||||
.build())
|
||||
.build();
|
||||
|
||||
TransactWriteItemsRequest request = new TransactWriteItemsRequest()
|
||||
.withTransactItems(phoneNumberDelete, accountDelete);
|
||||
TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
|
||||
.transactItems(phoneNumberDelete, accountDelete).build();
|
||||
|
||||
client.transactWriteItems(request);
|
||||
});
|
||||
@@ -299,64 +305,62 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||
try {
|
||||
TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid());
|
||||
|
||||
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid());
|
||||
accountPut.getPut()
|
||||
.setConditionExpression("attribute_not_exists(#uuid) OR (attribute_exists(#uuid) AND #version < :version)");
|
||||
accountPut.getPut()
|
||||
.setExpressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID,
|
||||
"#version", ATTR_MIGRATION_VERSION));
|
||||
accountPut.getPut()
|
||||
.setExpressionAttributeValues(
|
||||
Map.of(":version", new AttributeValue().withN(String.valueOf(account.getDynamoDbMigrationVersion()))));
|
||||
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder()
|
||||
.conditionExpression("attribute_not_exists(#uuid) OR (attribute_exists(#uuid) AND #version < :version)")
|
||||
.expressionAttributeNames(Map.of(
|
||||
"#uuid", KEY_ACCOUNT_UUID,
|
||||
"#version", ATTR_MIGRATION_VERSION))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":version", AttributeValues.fromInt(account.getDynamoDbMigrationVersion()))));
|
||||
|
||||
final TransactWriteItemsRequest request = new TransactWriteItemsRequest()
|
||||
.withTransactItems(phoneNumberConstraintPut, accountPut);
|
||||
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
|
||||
.transactItems(phoneNumberConstraintPut, accountPut).build();
|
||||
|
||||
final CompletableFuture<Boolean> resultFuture = new CompletableFuture<>();
|
||||
|
||||
asyncClient.transactWriteItemsAsync(request,
|
||||
new AsyncHandler<>() {
|
||||
@Override
|
||||
public void onError(Exception exception) {
|
||||
if (exception instanceof TransactionCanceledException) {
|
||||
// account is already migrated
|
||||
resultFuture.complete(false);
|
||||
} else {
|
||||
try {
|
||||
migrationRetryAccounts.put(account.getUuid());
|
||||
} catch (final Exception e) {
|
||||
logger.error("Could not store account {}", account.getUuid());
|
||||
}
|
||||
resultFuture.completeExceptionally(exception);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onSuccess(TransactWriteItemsRequest request, TransactWriteItemsResult transactWriteItemsResult) {
|
||||
resultFuture.complete(true);
|
||||
}
|
||||
});
|
||||
|
||||
asyncClient.transactWriteItems(request).whenCompleteAsync((result, exception) -> {
|
||||
if (result != null) {
|
||||
resultFuture.complete(true);
|
||||
return;
|
||||
}
|
||||
if (exception instanceof CompletionException) {
|
||||
// whenCompleteAsync can wrap exceptions in a CompletionException; unwrap it to get to the root cause.
|
||||
exception = exception.getCause();
|
||||
}
|
||||
if (exception instanceof TransactionCanceledException) {
|
||||
// account is already migrated
|
||||
resultFuture.complete(false);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
migrationRetryAccounts.put(account.getUuid());
|
||||
} catch (final Exception e) {
|
||||
logger.error("Could not store account {}", account.getUuid());
|
||||
}
|
||||
resultFuture.completeExceptionally(exception);
|
||||
});
|
||||
return resultFuture;
|
||||
|
||||
} catch (Exception e) {
|
||||
return CompletableFuture.failedFuture(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static String extractCancellationReasonCodes(final TransactionCanceledException exception) {
|
||||
return exception.getCancellationReasons().stream()
|
||||
.map(CancellationReason::getCode)
|
||||
return exception.cancellationReasons().stream()
|
||||
.map(CancellationReason::code)
|
||||
.collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Account fromItem(Item item) {
|
||||
static Account fromItem(Map<String, AttributeValue> item) {
|
||||
if (!item.containsKey(ATTR_ACCOUNT_DATA) ||
|
||||
!item.containsKey(ATTR_ACCOUNT_E164) ||
|
||||
!item.containsKey(KEY_ACCOUNT_UUID)) {
|
||||
throw new RuntimeException("item missing values");
|
||||
}
|
||||
try {
|
||||
Account account = SystemMapper.getMapper().readValue(item.getBinary(ATTR_ACCOUNT_DATA), Account.class);
|
||||
|
||||
account.setNumber(item.getString(ATTR_ACCOUNT_E164));
|
||||
account.setUuid(UUIDUtil.fromByteBuffer(item.getByteBuffer(KEY_ACCOUNT_UUID)));
|
||||
Account account = SystemMapper.getMapper().readValue(item.get(ATTR_ACCOUNT_DATA).b().asByteArray(), Account.class);
|
||||
account.setNumber(item.get(ATTR_ACCOUNT_E164).s());
|
||||
account.setUuid(UUIDUtil.fromByteBuffer(item.get(KEY_ACCOUNT_UUID).b().asByteBuffer()));
|
||||
|
||||
return account;
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
|
||||
import com.codahale.metrics.MetricRegistry;
|
||||
import com.codahale.metrics.SharedMetricRegistries;
|
||||
import com.codahale.metrics.Timer;
|
||||
@@ -21,12 +20,16 @@ import io.micrometer.core.instrument.Metrics;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.stream.Collectors;
|
||||
import net.logstash.logback.argument.StructuredArguments;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.whispersystems.textsecuregcm.auth.AmbiguousIdentifier;
|
||||
@@ -38,6 +41,7 @@ import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
||||
import org.whispersystems.textsecuregcm.util.Constants;
|
||||
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
||||
import org.whispersystems.textsecuregcm.util.Util;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
|
||||
|
||||
public class AccountsManager {
|
||||
|
||||
@@ -112,6 +116,7 @@ public class AccountsManager {
|
||||
|
||||
this.migrationComparisonMapper = mapper.copy();
|
||||
migrationComparisonMapper.addMixIn(Account.class, AccountComparisonMixin.class);
|
||||
migrationComparisonMapper.addMixIn(Device.class, DeviceComparisonMixin.class);
|
||||
|
||||
this.dynamicConfigurationManager = dynamicConfigurationManager;
|
||||
this.experimentEnrollmentManager = experimentEnrollmentManager;
|
||||
@@ -459,8 +464,14 @@ public class AccountsManager {
|
||||
}
|
||||
|
||||
try {
|
||||
if (!serializedEquals(databaseAccount.getRegistrationLock(), dynamoAccount.getRegistrationLock())) {
|
||||
return Optional.of("registrationLock");
|
||||
if (databaseAccount.getMasterDevice().isPresent() && dynamoAccount.getMasterDevice().isPresent()) {
|
||||
if (!Objects.equals(databaseAccount.getMasterDevice().get().getSignedPreKey(), dynamoAccount.getMasterDevice().get().getSignedPreKey())) {
|
||||
return Optional.of("masterDeviceSignedPreKey");
|
||||
}
|
||||
|
||||
if (!Objects.equals(databaseAccount.getMasterDevice().get().getPushTimestamp(), dynamoAccount.getMasterDevice().get().getPushTimestamp())) {
|
||||
return Optional.of("masterDevicePushTimestamp");
|
||||
}
|
||||
}
|
||||
|
||||
if (!serializedEquals(databaseAccount.getDevices(), dynamoAccount.getDevices())) {
|
||||
@@ -475,10 +486,6 @@ public class AccountsManager {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
if (databaseAccount.getDynamoDbMigrationVersion() != dynamoAccount.getDynamoDbMigrationVersion()) {
|
||||
return Optional.of("migrationVersion");
|
||||
}
|
||||
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@@ -521,11 +528,26 @@ public class AccountsManager {
|
||||
|
||||
if (maybeUUid.isPresent()
|
||||
&& dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isLogMismatches()) {
|
||||
logger.info("Mismatched {} for {}", mismatchDescription, maybeUUid.get());
|
||||
|
||||
final String abbreviatedCallChain = getAbbreviatedCallChain(new RuntimeException().getStackTrace());
|
||||
|
||||
logger.info("Mismatched account data: {}", StructuredArguments.entries(Map.of(
|
||||
"type", mismatchDescription,
|
||||
"uuid", maybeUUid.get(),
|
||||
"callChain", abbreviatedCallChain
|
||||
)));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private String getAbbreviatedCallChain(final StackTraceElement[] stackTrace) {
|
||||
return Arrays.stream(stackTrace)
|
||||
.filter(stackTraceElement -> stackTraceElement.getClassName().contains("org.whispersystems"))
|
||||
.filter(stackTraceElement -> !(stackTraceElement.getClassName().endsWith("AccountsManager") && stackTraceElement.getMethodName().contains("compare")))
|
||||
.map(stackTraceElement -> StringUtils.substringAfterLast(stackTraceElement.getClassName(), ".") + ":" + stackTraceElement.getMethodName())
|
||||
.collect(Collectors.joining(" -> "));
|
||||
}
|
||||
|
||||
private static abstract class AccountComparisonMixin extends Account {
|
||||
|
||||
@JsonIgnore
|
||||
@@ -533,6 +555,13 @@ public class AccountsManager {
|
||||
|
||||
}
|
||||
|
||||
private static abstract class DeviceComparisonMixin extends Device {
|
||||
|
||||
@JsonIgnore
|
||||
private long lastSeen;
|
||||
|
||||
}
|
||||
|
||||
private boolean serializedEquals(final Object database, final Object dynamo) throws JsonProcessingException {
|
||||
final byte[] databaseSerialized = migrationComparisonMapper.writeValueAsBytes(database);
|
||||
final byte[] dynamoSerialized = migrationComparisonMapper.writeValueAsBytes(dynamo);
|
||||
|
||||
@@ -18,8 +18,12 @@ import org.slf4j.LoggerFactory;
|
||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
||||
import org.whispersystems.textsecuregcm.util.Util;
|
||||
|
||||
import javax.validation.ConstraintViolation;
|
||||
import javax.validation.Validation;
|
||||
import javax.validation.Validator;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
@@ -32,16 +36,19 @@ public class DynamicConfigurationManager {
|
||||
private final AmazonAppConfig appConfigClient;
|
||||
|
||||
private final AtomicReference<DynamicConfiguration> configuration = new AtomicReference<>();
|
||||
private final Logger logger = LoggerFactory.getLogger(DynamicConfigurationManager.class);
|
||||
|
||||
private GetConfigurationResult lastConfigResult;
|
||||
|
||||
private boolean initialized = false;
|
||||
|
||||
public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory())
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory())
|
||||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
||||
.registerModule(new JavaTimeModule());
|
||||
|
||||
private static final Validator VALIDATOR = Validation.buildDefaultValidatorFactory().getValidator();
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DynamicConfigurationManager.class);
|
||||
|
||||
public DynamicConfigurationManager(String application, String environment, String configurationName) {
|
||||
this(AmazonAppConfigClient.builder()
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(10000).withRequestTimeout(10000))
|
||||
@@ -104,7 +111,9 @@ public class DynamicConfigurationManager {
|
||||
|
||||
if (!StringUtils.equals(lastConfigResult.getConfigurationVersion(), previousVersion)) {
|
||||
logger.info("Received new config version: {}", lastConfigResult.getConfigurationVersion());
|
||||
maybeDynamicConfiguration = Optional.of(OBJECT_MAPPER.readValue(StandardCharsets.UTF_8.decode(lastConfigResult.getContent().asReadOnlyBuffer()).toString(), DynamicConfiguration.class));
|
||||
|
||||
maybeDynamicConfiguration =
|
||||
parseConfiguration(StandardCharsets.UTF_8.decode(lastConfigResult.getContent().asReadOnlyBuffer()).toString());
|
||||
} else {
|
||||
// No change since last version
|
||||
maybeDynamicConfiguration = Optional.empty();
|
||||
@@ -113,6 +122,23 @@ public class DynamicConfigurationManager {
|
||||
return maybeDynamicConfiguration;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static Optional<DynamicConfiguration> parseConfiguration(final String configurationYaml) throws JsonProcessingException {
|
||||
final DynamicConfiguration configuration = OBJECT_MAPPER.readValue(configurationYaml, DynamicConfiguration.class);
|
||||
final Set<ConstraintViolation<DynamicConfiguration>> violations = VALIDATOR.validate(configuration);
|
||||
|
||||
final Optional<DynamicConfiguration> maybeDynamicConfiguration;
|
||||
|
||||
if (violations.isEmpty()) {
|
||||
maybeDynamicConfiguration = Optional.of(configuration);
|
||||
} else {
|
||||
logger.warn("Failed to validate configuration: {}", violations);
|
||||
maybeDynamicConfiguration = Optional.empty();
|
||||
}
|
||||
|
||||
return maybeDynamicConfiguration;
|
||||
}
|
||||
|
||||
private DynamicConfiguration retrieveInitialDynamicConfiguration() {
|
||||
for (;;) {
|
||||
try {
|
||||
|
||||
@@ -7,195 +7,230 @@ package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DeleteItemOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
|
||||
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
|
||||
import com.amazonaws.services.dynamodbv2.model.Select;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.micrometer.core.instrument.DistributionSummary;
|
||||
import io.micrometer.core.instrument.Metrics;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.whispersystems.textsecuregcm.entities.PreKey;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.PutRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.QueryResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.Select;
|
||||
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
|
||||
|
||||
public class KeysDynamoDb extends AbstractDynamoDbStore {
|
||||
|
||||
private final Table table;
|
||||
private final String tableName;
|
||||
|
||||
static final String KEY_ACCOUNT_UUID = "U";
|
||||
static final String KEY_DEVICE_ID_KEY_ID = "DK";
|
||||
static final String KEY_PUBLIC_KEY = "P";
|
||||
static final String KEY_ACCOUNT_UUID = "U";
|
||||
static final String KEY_DEVICE_ID_KEY_ID = "DK";
|
||||
static final String KEY_PUBLIC_KEY = "P";
|
||||
|
||||
private static final Timer STORE_KEYS_TIMER = Metrics.timer(name(KeysDynamoDb.class, "storeKeys"));
|
||||
private static final Timer TAKE_KEY_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForDevice"));
|
||||
private static final Timer TAKE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForAccount"));
|
||||
private static final Timer GET_KEY_COUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "getKeyCount"));
|
||||
private static final Timer DELETE_KEYS_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForDevice"));
|
||||
private static final Timer DELETE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForAccount"));
|
||||
private static final DistributionSummary CONTESTED_KEY_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "contestedKeys"));
|
||||
private static final DistributionSummary KEY_COUNT_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "keyCount"));
|
||||
private static final Timer STORE_KEYS_TIMER = Metrics.timer(name(KeysDynamoDb.class, "storeKeys"));
|
||||
private static final Timer TAKE_KEY_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForDevice"));
|
||||
private static final Timer TAKE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForAccount"));
|
||||
private static final Timer GET_KEY_COUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "getKeyCount"));
|
||||
private static final Timer DELETE_KEYS_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForDevice"));
|
||||
private static final Timer DELETE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForAccount"));
|
||||
private static final DistributionSummary CONTESTED_KEY_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "contestedKeys"));
|
||||
private static final DistributionSummary KEY_COUNT_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "keyCount"));
|
||||
|
||||
public KeysDynamoDb(final DynamoDB dynamoDB, final String tableName) {
|
||||
super(dynamoDB);
|
||||
public KeysDynamoDb(final DynamoDbClient dynamoDB, final String tableName) {
|
||||
super(dynamoDB);
|
||||
this.tableName = tableName;
|
||||
}
|
||||
|
||||
this.table = dynamoDB.getTable(tableName);
|
||||
}
|
||||
public void store(final Account account, final long deviceId, final List<PreKey> keys) {
|
||||
STORE_KEYS_TIMER.record(() -> {
|
||||
delete(account, deviceId);
|
||||
|
||||
public void store(final Account account, final long deviceId, final List<PreKey> keys) {
|
||||
STORE_KEYS_TIMER.record(() -> {
|
||||
delete(account, deviceId);
|
||||
writeInBatches(keys, batch -> {
|
||||
List<WriteRequest> items = new ArrayList<>();
|
||||
for (final PreKey preKey : batch) {
|
||||
items.add(WriteRequest.builder()
|
||||
.putRequest(PutRequest.builder()
|
||||
.item(getItemFromPreKey(account.getUuid(), deviceId, preKey))
|
||||
.build())
|
||||
.build());
|
||||
}
|
||||
executeTableWriteItemsUntilComplete(Map.of(tableName, items));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
writeInBatches(keys, batch -> {
|
||||
final TableWriteItems items = new TableWriteItems(table.getTableName());
|
||||
public Optional<PreKey> take(final Account account, final long deviceId) {
|
||||
return TAKE_KEY_FOR_DEVICE_TIMER.record(() -> {
|
||||
final AttributeValue partitionKey = getPartitionKey(account.getUuid());
|
||||
QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
|
||||
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":uuid", partitionKey,
|
||||
":sortprefix", getSortKeyPrefix(deviceId)))
|
||||
.projectionExpression(KEY_DEVICE_ID_KEY_ID)
|
||||
.consistentRead(false)
|
||||
.build();
|
||||
|
||||
for (final PreKey preKey : batch) {
|
||||
items.addItemToPut(getItemFromPreKey(account.getUuid(), deviceId, preKey));
|
||||
}
|
||||
int contestedKeys = 0;
|
||||
|
||||
executeTableWriteItemsUntilComplete(items);
|
||||
});
|
||||
});
|
||||
}
|
||||
try {
|
||||
QueryResponse response = db().query(queryRequest);
|
||||
for (Map<String, AttributeValue> candidate : response.items()) {
|
||||
DeleteItemRequest deleteItemRequest = DeleteItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.key(Map.of(
|
||||
KEY_ACCOUNT_UUID, partitionKey,
|
||||
KEY_DEVICE_ID_KEY_ID, candidate.get(KEY_DEVICE_ID_KEY_ID)))
|
||||
.returnValues(ReturnValue.ALL_OLD)
|
||||
.build();
|
||||
DeleteItemResponse deleteItemResponse = db().deleteItem(deleteItemRequest);
|
||||
if (deleteItemResponse.hasAttributes()) {
|
||||
return Optional.of(getPreKeyFromItem(deleteItemResponse.attributes()));
|
||||
}
|
||||
|
||||
public Optional<PreKey> take(final Account account, final long deviceId) {
|
||||
return TAKE_KEY_FOR_DEVICE_TIMER.record(() -> {
|
||||
final byte[] partitionKey = getPartitionKey(account.getUuid());
|
||||
contestedKeys++;
|
||||
}
|
||||
|
||||
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
|
||||
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
|
||||
.withValueMap(Map.of(":uuid", partitionKey,
|
||||
":sortprefix", getSortKeyPrefix(deviceId)))
|
||||
.withProjectionExpression(KEY_DEVICE_ID_KEY_ID)
|
||||
.withConsistentRead(false);
|
||||
return Optional.empty();
|
||||
} finally {
|
||||
CONTESTED_KEY_DISTRIBUTION.record(contestedKeys);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
int contestedKeys = 0;
|
||||
public Map<Long, PreKey> take(final Account account) {
|
||||
return TAKE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
|
||||
final Map<Long, PreKey> preKeysByDeviceId = new HashMap<>();
|
||||
|
||||
try {
|
||||
for (final Item candidate : table.query(querySpec)) {
|
||||
final DeleteItemSpec deleteItemSpec = new DeleteItemSpec().withPrimaryKey(KEY_ACCOUNT_UUID, partitionKey, KEY_DEVICE_ID_KEY_ID, candidate.getBinary(KEY_DEVICE_ID_KEY_ID))
|
||||
.withReturnValues(ReturnValue.ALL_OLD);
|
||||
for (final Device device : account.getDevices()) {
|
||||
take(account, device.getId()).ifPresent(preKey -> preKeysByDeviceId.put(device.getId(), preKey));
|
||||
}
|
||||
|
||||
final DeleteItemOutcome outcome = table.deleteItem(deleteItemSpec);
|
||||
return preKeysByDeviceId;
|
||||
});
|
||||
}
|
||||
|
||||
if (outcome.getItem() != null) {
|
||||
return Optional.of(getPreKeyFromItem(outcome.getItem()));
|
||||
}
|
||||
public int getCount(final Account account, final long deviceId) {
|
||||
return GET_KEY_COUNT_TIMER.record(() -> {
|
||||
QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
|
||||
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":uuid", getPartitionKey(account.getUuid()),
|
||||
":sortprefix", getSortKeyPrefix(deviceId)))
|
||||
.select(Select.COUNT)
|
||||
.consistentRead(false)
|
||||
.build();
|
||||
|
||||
contestedKeys++;
|
||||
}
|
||||
int keyCount = 0;
|
||||
// This is very confusing, but does appear to be the intended behavior. See:
|
||||
//
|
||||
// - https://github.com/aws/aws-sdk-java/issues/693
|
||||
// - https://github.com/aws/aws-sdk-java/issues/915
|
||||
// - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.Count
|
||||
for (final QueryResponse page : db().queryPaginator(queryRequest)) {
|
||||
keyCount += page.count();
|
||||
}
|
||||
KEY_COUNT_DISTRIBUTION.record(keyCount);
|
||||
return keyCount;
|
||||
});
|
||||
}
|
||||
|
||||
return Optional.empty();
|
||||
} finally {
|
||||
CONTESTED_KEY_DISTRIBUTION.record(contestedKeys);
|
||||
}
|
||||
});
|
||||
}
|
||||
public void delete(final Account account) {
|
||||
DELETE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
|
||||
final QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.keyConditionExpression("#uuid = :uuid")
|
||||
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":uuid", getPartitionKey(account.getUuid())))
|
||||
.projectionExpression(KEY_DEVICE_ID_KEY_ID)
|
||||
.consistentRead(true)
|
||||
.build();
|
||||
|
||||
public Map<Long, PreKey> take(final Account account) {
|
||||
return TAKE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
|
||||
final Map<Long, PreKey> preKeysByDeviceId = new HashMap<>();
|
||||
deleteItemsForAccountMatchingQuery(account, queryRequest);
|
||||
});
|
||||
}
|
||||
|
||||
for (final Device device : account.getDevices()) {
|
||||
take(account, device.getId()).ifPresent(preKey -> preKeysByDeviceId.put(device.getId(), preKey));
|
||||
}
|
||||
@VisibleForTesting
|
||||
void delete(final Account account, final long deviceId) {
|
||||
DELETE_KEYS_FOR_DEVICE_TIMER.record(() -> {
|
||||
final QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
|
||||
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":uuid", getPartitionKey(account.getUuid()),
|
||||
":sortprefix", getSortKeyPrefix(deviceId)))
|
||||
.projectionExpression(KEY_DEVICE_ID_KEY_ID)
|
||||
.consistentRead(true)
|
||||
.build();
|
||||
|
||||
return preKeysByDeviceId;
|
||||
});
|
||||
}
|
||||
deleteItemsForAccountMatchingQuery(account, queryRequest);
|
||||
});
|
||||
}
|
||||
|
||||
public int getCount(final Account account, final long deviceId) {
|
||||
return GET_KEY_COUNT_TIMER.record(() -> {
|
||||
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
|
||||
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
|
||||
.withValueMap(Map.of(":uuid", getPartitionKey(account.getUuid()),
|
||||
":sortprefix", getSortKeyPrefix(deviceId)))
|
||||
.withSelect(Select.COUNT)
|
||||
.withConsistentRead(false);
|
||||
private void deleteItemsForAccountMatchingQuery(final Account account, final QueryRequest querySpec) {
|
||||
final AttributeValue partitionKey = getPartitionKey(account.getUuid());
|
||||
|
||||
final int keyCount = (int)countItemsMatchingQuery(table, querySpec);
|
||||
writeInBatches(db().query(querySpec).items(), batch -> {
|
||||
List<WriteRequest> deletes = new ArrayList<>();
|
||||
for (final Map<String, AttributeValue> item : batch) {
|
||||
deletes.add(WriteRequest.builder()
|
||||
.deleteRequest(DeleteRequest.builder()
|
||||
.key(Map.of(
|
||||
KEY_ACCOUNT_UUID, partitionKey,
|
||||
KEY_DEVICE_ID_KEY_ID, item.get(KEY_DEVICE_ID_KEY_ID)))
|
||||
.build())
|
||||
.build());
|
||||
}
|
||||
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
|
||||
});
|
||||
}
|
||||
|
||||
KEY_COUNT_DISTRIBUTION.record(keyCount);
|
||||
return keyCount;
|
||||
});
|
||||
}
|
||||
private static AttributeValue getPartitionKey(final UUID accountUuid) {
|
||||
return AttributeValues.fromUUID(accountUuid);
|
||||
}
|
||||
|
||||
public void delete(final Account account) {
|
||||
DELETE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
|
||||
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid")
|
||||
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID))
|
||||
.withValueMap(Map.of(":uuid", getPartitionKey(account.getUuid())))
|
||||
.withProjectionExpression(KEY_DEVICE_ID_KEY_ID)
|
||||
.withConsistentRead(true);
|
||||
private static AttributeValue getSortKey(final long deviceId, final long keyId) {
|
||||
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]);
|
||||
byteBuffer.putLong(deviceId);
|
||||
byteBuffer.putLong(keyId);
|
||||
return AttributeValues.fromByteBuffer(byteBuffer.flip());
|
||||
}
|
||||
|
||||
deleteItemsForAccountMatchingQuery(account, querySpec);
|
||||
});
|
||||
}
|
||||
@VisibleForTesting
|
||||
static AttributeValue getSortKeyPrefix(final long deviceId) {
|
||||
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
|
||||
byteBuffer.putLong(deviceId);
|
||||
return AttributeValues.fromByteBuffer(byteBuffer.flip());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void delete(final Account account, final long deviceId) {
|
||||
DELETE_KEYS_FOR_DEVICE_TIMER.record(() -> {
|
||||
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
|
||||
.withNameMap(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
|
||||
.withValueMap(Map.of(":uuid", getPartitionKey(account.getUuid()),
|
||||
":sortprefix", getSortKeyPrefix(deviceId)))
|
||||
.withProjectionExpression(KEY_DEVICE_ID_KEY_ID)
|
||||
.withConsistentRead(true);
|
||||
private Map<String, AttributeValue> getItemFromPreKey(final UUID accountUuid, final long deviceId, final PreKey preKey) {
|
||||
return Map.of(
|
||||
KEY_ACCOUNT_UUID, getPartitionKey(accountUuid),
|
||||
KEY_DEVICE_ID_KEY_ID, getSortKey(deviceId, preKey.getKeyId()),
|
||||
KEY_PUBLIC_KEY, AttributeValues.fromString(preKey.getPublicKey()));
|
||||
}
|
||||
|
||||
deleteItemsForAccountMatchingQuery(account, querySpec);
|
||||
});
|
||||
}
|
||||
|
||||
private void deleteItemsForAccountMatchingQuery(final Account account, final QuerySpec querySpec) {
|
||||
final byte[] partitionKey = getPartitionKey(account.getUuid());
|
||||
|
||||
writeInBatches(table.query(querySpec), batch -> {
|
||||
final TableWriteItems writeItems = new TableWriteItems(table.getTableName());
|
||||
|
||||
for (final Item item : batch) {
|
||||
writeItems.addPrimaryKeyToDelete(new PrimaryKey(KEY_ACCOUNT_UUID, partitionKey, KEY_DEVICE_ID_KEY_ID, item.getBinary(KEY_DEVICE_ID_KEY_ID)));
|
||||
}
|
||||
|
||||
executeTableWriteItemsUntilComplete(writeItems);
|
||||
});
|
||||
}
|
||||
|
||||
private static byte[] getPartitionKey(final UUID accountUuid) {
|
||||
return UUIDUtil.toBytes(accountUuid);
|
||||
}
|
||||
|
||||
private static byte[] getSortKey(final long deviceId, final long keyId) {
|
||||
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]);
|
||||
byteBuffer.putLong(deviceId);
|
||||
byteBuffer.putLong(keyId);
|
||||
return byteBuffer.array();
|
||||
}
|
||||
|
||||
private static byte[] getSortKeyPrefix(final long deviceId) {
|
||||
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
|
||||
byteBuffer.putLong(deviceId);
|
||||
return byteBuffer.array();
|
||||
}
|
||||
|
||||
private Item getItemFromPreKey(final UUID accountUuid, final long deviceId, final PreKey preKey) {
|
||||
return new Item().withBinary(KEY_ACCOUNT_UUID, getPartitionKey(accountUuid))
|
||||
.withBinary(KEY_DEVICE_ID_KEY_ID, getSortKey(deviceId, preKey.getKeyId()))
|
||||
.withString(KEY_PUBLIC_KEY, preKey.getPublicKey());
|
||||
}
|
||||
|
||||
private PreKey getPreKeyFromItem(final Item item) {
|
||||
final long keyId = ByteBuffer.wrap(item.getBinary(KEY_DEVICE_ID_KEY_ID)).getLong(8);
|
||||
return new PreKey(keyId, item.getString(KEY_PUBLIC_KEY));
|
||||
}
|
||||
private PreKey getPreKeyFromItem(Map<String, AttributeValue> item) {
|
||||
final long keyId = item.get(KEY_DEVICE_ID_KEY_ID).b().asByteBuffer().getLong(8);
|
||||
return new PreKey(keyId, item.get(KEY_PUBLIC_KEY).s());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,17 +8,7 @@ package org.whispersystems.textsecuregcm.storage;
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
import static io.micrometer.core.instrument.Metrics.timer;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DeleteItemOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Index;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
|
||||
import com.amazonaws.services.dynamodbv2.document.api.QueryApi;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
|
||||
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.time.Duration;
|
||||
@@ -27,11 +17,22 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.annotation.Nonnull;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.whispersystems.textsecuregcm.entities.MessageProtos;
|
||||
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntity;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.PutRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
|
||||
|
||||
public class MessagesDynamoDb extends AbstractDynamoDbStore {
|
||||
|
||||
@@ -60,7 +61,7 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
|
||||
private final String tableName;
|
||||
private final Duration timeToLive;
|
||||
|
||||
public MessagesDynamoDb(DynamoDB dynamoDb, String tableName, Duration timeToLive) {
|
||||
public MessagesDynamoDb(DynamoDbClient dynamoDb, String tableName, Duration timeToLive) {
|
||||
super(dynamoDb);
|
||||
|
||||
this.tableName = tableName;
|
||||
@@ -76,54 +77,61 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
|
||||
throw new IllegalArgumentException("Maximum batch size of " + DYNAMO_DB_MAX_BATCH_SIZE + " execeeded with " + messages.size() + " messages");
|
||||
}
|
||||
|
||||
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
TableWriteItems items = new TableWriteItems(tableName);
|
||||
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
List<WriteRequest> writeItems = new ArrayList<>();
|
||||
for (MessageProtos.Envelope message : messages) {
|
||||
final UUID messageUuid = UUID.fromString(message.getServerGuid());
|
||||
final Item item = new Item().withBinary(KEY_PARTITION, partitionKey)
|
||||
.withBinary(KEY_SORT, convertSortKey(destinationDeviceId, message.getServerTimestamp(), messageUuid))
|
||||
.withBinary(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT, convertLocalIndexMessageUuidSortKey(messageUuid))
|
||||
.withInt(KEY_TYPE, message.getType().getNumber())
|
||||
.withLong(KEY_TIMESTAMP, message.getTimestamp())
|
||||
.withLong(KEY_TTL, getTtlForMessage(message));
|
||||
final ImmutableMap.Builder<String, AttributeValue> item = ImmutableMap.<String, AttributeValue>builder()
|
||||
.put(KEY_PARTITION, partitionKey)
|
||||
.put(KEY_SORT, convertSortKey(destinationDeviceId, message.getServerTimestamp(), messageUuid))
|
||||
.put(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT, convertLocalIndexMessageUuidSortKey(messageUuid))
|
||||
.put(KEY_TYPE, AttributeValues.fromInt(message.getType().getNumber()))
|
||||
.put(KEY_TIMESTAMP, AttributeValues.fromLong(message.getTimestamp()))
|
||||
.put(KEY_TTL, AttributeValues.fromLong(getTtlForMessage(message)));
|
||||
if (message.hasRelay() && message.getRelay().length() > 0) {
|
||||
item.withString(KEY_RELAY, message.getRelay());
|
||||
item.put(KEY_RELAY, AttributeValues.fromString(message.getRelay()));
|
||||
}
|
||||
if (message.hasSource()) {
|
||||
item.withString(KEY_SOURCE, message.getSource());
|
||||
item.put(KEY_SOURCE, AttributeValues.fromString(message.getSource()));
|
||||
}
|
||||
if (message.hasSourceUuid()) {
|
||||
item.withBinary(KEY_SOURCE_UUID, UUIDUtil.toBytes(UUID.fromString(message.getSourceUuid())));
|
||||
item.put(KEY_SOURCE_UUID, AttributeValues.fromUUID(UUID.fromString(message.getSourceUuid())));
|
||||
}
|
||||
if (message.hasSourceDevice()) {
|
||||
item.withInt(KEY_SOURCE_DEVICE, message.getSourceDevice());
|
||||
item.put(KEY_SOURCE_DEVICE, AttributeValues.fromInt(message.getSourceDevice()));
|
||||
}
|
||||
if (message.hasLegacyMessage()) {
|
||||
item.withBinary(KEY_MESSAGE, message.getLegacyMessage().toByteArray());
|
||||
item.put(KEY_MESSAGE, AttributeValues.fromByteArray(message.getLegacyMessage().toByteArray()));
|
||||
}
|
||||
if (message.hasContent()) {
|
||||
item.withBinary(KEY_CONTENT, message.getContent().toByteArray());
|
||||
item.put(KEY_CONTENT, AttributeValues.fromByteArray(message.getContent().toByteArray()));
|
||||
}
|
||||
items.addItemToPut(item);
|
||||
writeItems.add(WriteRequest.builder().putRequest(PutRequest.builder()
|
||||
.item(item.build())
|
||||
.build()).build());
|
||||
}
|
||||
|
||||
executeTableWriteItemsUntilComplete(items);
|
||||
executeTableWriteItemsUntilComplete(Map.of(tableName, writeItems));
|
||||
}
|
||||
|
||||
public List<OutgoingMessageEntity> load(final UUID destinationAccountUuid, final long destinationDeviceId, final int requestedNumberOfMessagesToFetch) {
|
||||
return loadTimer.record(() -> {
|
||||
final int numberOfMessagesToFetch = Math.min(requestedNumberOfMessagesToFetch, RESULT_SET_CHUNK_SIZE);
|
||||
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QuerySpec querySpec = new QuerySpec().withConsistentRead(true)
|
||||
.withKeyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
|
||||
.withNameMap(Map.of("#part", KEY_PARTITION,
|
||||
"#sort", KEY_SORT))
|
||||
.withValueMap(Map.of(":part", partitionKey,
|
||||
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
|
||||
.withMaxResultSize(numberOfMessagesToFetch);
|
||||
final Table table = getDynamoDb().getTable(tableName);
|
||||
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.consistentRead(true)
|
||||
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
|
||||
.expressionAttributeNames(Map.of(
|
||||
"#part", KEY_PARTITION,
|
||||
"#sort", KEY_SORT))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":part", partitionKey,
|
||||
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
|
||||
.limit(numberOfMessagesToFetch)
|
||||
.build();
|
||||
List<OutgoingMessageEntity> messageEntities = new ArrayList<>(numberOfMessagesToFetch);
|
||||
for (Item message : table.query(querySpec)) {
|
||||
for (Map<String, AttributeValue> message : db().query(queryRequest).items()) {
|
||||
messageEntities.add(convertItemToOutgoingMessageEntity(message));
|
||||
}
|
||||
return messageEntities;
|
||||
@@ -136,53 +144,63 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
|
||||
throw new IllegalArgumentException("must specify a source");
|
||||
}
|
||||
|
||||
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QuerySpec querySpec = new QuerySpec().withProjectionExpression(KEY_SORT)
|
||||
.withConsistentRead(true)
|
||||
.withKeyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
|
||||
.withFilterExpression("#source = :source AND #timestamp = :timestamp")
|
||||
.withNameMap(Map.of("#part", KEY_PARTITION,
|
||||
"#sort", KEY_SORT,
|
||||
"#source", KEY_SOURCE,
|
||||
"#timestamp", KEY_TIMESTAMP))
|
||||
.withValueMap(Map.of(":part", partitionKey,
|
||||
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId),
|
||||
":source", source,
|
||||
":timestamp", timestamp));
|
||||
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.projectionExpression(KEY_SORT)
|
||||
.consistentRead(true)
|
||||
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
|
||||
.filterExpression("#source = :source AND #timestamp = :timestamp")
|
||||
.expressionAttributeNames(Map.of(
|
||||
"#part", KEY_PARTITION,
|
||||
"#sort", KEY_SORT,
|
||||
"#source", KEY_SOURCE,
|
||||
"#timestamp", KEY_TIMESTAMP))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":part", partitionKey,
|
||||
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId),
|
||||
":source", AttributeValues.fromString(source),
|
||||
":timestamp", AttributeValues.fromLong(timestamp)))
|
||||
.build();
|
||||
|
||||
final Table table = getDynamoDb().getTable(tableName);
|
||||
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(table, partitionKey, querySpec, table);
|
||||
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(partitionKey, queryRequest);
|
||||
});
|
||||
}
|
||||
|
||||
public Optional<OutgoingMessageEntity> deleteMessageByDestinationAndGuid(final UUID destinationAccountUuid, final long destinationDeviceId, final UUID messageUuid) {
|
||||
return deleteByGuid.record(() -> {
|
||||
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QuerySpec querySpec = new QuerySpec().withProjectionExpression(KEY_SORT)
|
||||
.withConsistentRead(true)
|
||||
.withKeyConditionExpression("#part = :part AND #uuid = :uuid")
|
||||
.withNameMap(Map.of("#part", KEY_PARTITION,
|
||||
"#uuid", LOCAL_INDEX_MESSAGE_UUID_KEY_SORT))
|
||||
.withValueMap(Map.of(":part", partitionKey,
|
||||
":uuid", convertLocalIndexMessageUuidSortKey(messageUuid)));
|
||||
final Table table = getDynamoDb().getTable(tableName);
|
||||
final Index index = table.getIndex(LOCAL_INDEX_MESSAGE_UUID_NAME);
|
||||
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(table, partitionKey, querySpec, index);
|
||||
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.indexName(LOCAL_INDEX_MESSAGE_UUID_NAME)
|
||||
.projectionExpression(KEY_SORT)
|
||||
.consistentRead(true)
|
||||
.keyConditionExpression("#part = :part AND #uuid = :uuid")
|
||||
.expressionAttributeNames(Map.of(
|
||||
"#part", KEY_PARTITION,
|
||||
"#uuid", LOCAL_INDEX_MESSAGE_UUID_KEY_SORT))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":part", partitionKey,
|
||||
":uuid", convertLocalIndexMessageUuidSortKey(messageUuid)))
|
||||
.build();
|
||||
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(partitionKey, queryRequest);
|
||||
});
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Optional<OutgoingMessageEntity> deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(Table table, byte[] partitionKey, QuerySpec querySpec, QueryApi queryApi) {
|
||||
private Optional<OutgoingMessageEntity> deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(AttributeValue partitionKey, QueryRequest queryRequest) {
|
||||
Optional<OutgoingMessageEntity> result = Optional.empty();
|
||||
for (Item item : queryApi.query(querySpec)) {
|
||||
final byte[] rangeKeyValue = item.getBinary(KEY_SORT);
|
||||
DeleteItemSpec deleteItemSpec = new DeleteItemSpec().withPrimaryKey(KEY_PARTITION, partitionKey, KEY_SORT, rangeKeyValue);
|
||||
for (Map<String, AttributeValue> item : db().query(queryRequest).items()) {
|
||||
final byte[] rangeKeyValue = item.get(KEY_SORT).b().asByteArray();
|
||||
DeleteItemRequest.Builder deleteItemRequest = DeleteItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.key(Map.of(KEY_PARTITION, partitionKey, KEY_SORT, AttributeValues.fromByteArray(rangeKeyValue)));
|
||||
if (result.isEmpty()) {
|
||||
deleteItemSpec.withReturnValues(ReturnValue.ALL_OLD);
|
||||
deleteItemRequest.returnValues(ReturnValue.ALL_OLD);
|
||||
}
|
||||
final DeleteItemOutcome deleteItemOutcome = table.deleteItem(deleteItemSpec);
|
||||
if (deleteItemOutcome.getItem() != null && deleteItemOutcome.getItem().hasAttribute(KEY_PARTITION)) {
|
||||
result = Optional.of(convertItemToOutgoingMessageEntity(deleteItemOutcome.getItem()));
|
||||
final DeleteItemResponse deleteItemResponse = db().deleteItem(deleteItemRequest.build());
|
||||
if (deleteItemResponse.attributes() != null && deleteItemResponse.attributes().containsKey(KEY_PARTITION)) {
|
||||
result = Optional.of(convertItemToOutgoingMessageEntity(deleteItemResponse.attributes()));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
@@ -190,74 +208,88 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
|
||||
|
||||
public void deleteAllMessagesForAccount(final UUID destinationAccountUuid) {
|
||||
deleteByAccount.record(() -> {
|
||||
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QuerySpec querySpec = new QuerySpec().withHashKey(KEY_PARTITION, partitionKey)
|
||||
.withProjectionExpression(KEY_SORT)
|
||||
.withConsistentRead(true);
|
||||
deleteRowsMatchingQuery(partitionKey, querySpec);
|
||||
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.projectionExpression(KEY_SORT)
|
||||
.consistentRead(true)
|
||||
.keyConditionExpression("#part = :part")
|
||||
.expressionAttributeNames(Map.of("#part", KEY_PARTITION))
|
||||
.expressionAttributeValues(Map.of(":part", partitionKey))
|
||||
.build();
|
||||
deleteRowsMatchingQuery(partitionKey, queryRequest);
|
||||
});
|
||||
}
|
||||
|
||||
public void deleteAllMessagesForDevice(final UUID destinationAccountUuid, final long destinationDeviceId) {
|
||||
deleteByDevice.record(() -> {
|
||||
final byte[] partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QuerySpec querySpec = new QuerySpec().withKeyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
|
||||
.withNameMap(Map.of("#part", KEY_PARTITION,
|
||||
"#sort", KEY_SORT))
|
||||
.withValueMap(Map.of(":part", partitionKey,
|
||||
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
|
||||
.withProjectionExpression(KEY_SORT)
|
||||
.withConsistentRead(true);
|
||||
deleteRowsMatchingQuery(partitionKey, querySpec);
|
||||
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
|
||||
final QueryRequest queryRequest = QueryRequest.builder()
|
||||
.tableName(tableName)
|
||||
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
|
||||
.expressionAttributeNames(Map.of(
|
||||
"#part", KEY_PARTITION,
|
||||
"#sort", KEY_SORT))
|
||||
.expressionAttributeValues(Map.of(
|
||||
":part", partitionKey,
|
||||
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
|
||||
.projectionExpression(KEY_SORT)
|
||||
.consistentRead(true)
|
||||
.build();
|
||||
deleteRowsMatchingQuery(partitionKey, queryRequest);
|
||||
});
|
||||
}
|
||||
|
||||
private OutgoingMessageEntity convertItemToOutgoingMessageEntity(Item message) {
|
||||
final SortKey sortKey = convertSortKey(message.getBinary(KEY_SORT));
|
||||
final UUID messageUuid = convertLocalIndexMessageUuidSortKey(message.getBinary(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT));
|
||||
final int type = message.getInt(KEY_TYPE);
|
||||
final String relay = message.getString(KEY_RELAY);
|
||||
final long timestamp = message.getLong(KEY_TIMESTAMP);
|
||||
final String source = message.getString(KEY_SOURCE);
|
||||
final UUID sourceUuid = message.hasAttribute(KEY_SOURCE_UUID) ? convertUuidFromBytes(message.getBinary(KEY_SOURCE_UUID), "message source uuid") : null;
|
||||
final int sourceDevice = message.hasAttribute(KEY_SOURCE_DEVICE) ? message.getInt(KEY_SOURCE_DEVICE) : 0;
|
||||
final byte[] messageBytes = message.getBinary(KEY_MESSAGE);
|
||||
final byte[] content = message.getBinary(KEY_CONTENT);
|
||||
private OutgoingMessageEntity convertItemToOutgoingMessageEntity(Map<String, AttributeValue> message) {
|
||||
final SortKey sortKey = convertSortKey(message.get(KEY_SORT).b().asByteArray());
|
||||
final UUID messageUuid = convertLocalIndexMessageUuidSortKey(message.get(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT).b().asByteArray());
|
||||
final int type = AttributeValues.getInt(message, KEY_TYPE, 0);
|
||||
final String relay = AttributeValues.getString(message, KEY_RELAY, null);
|
||||
final long timestamp = AttributeValues.getLong(message, KEY_TIMESTAMP, 0L);
|
||||
final String source = AttributeValues.getString(message, KEY_SOURCE, null);
|
||||
final UUID sourceUuid = AttributeValues.getUUID(message, KEY_SOURCE_UUID, null);
|
||||
final int sourceDevice = AttributeValues.getInt(message, KEY_SOURCE_DEVICE, 0);
|
||||
final byte[] messageBytes = AttributeValues.getByteArray(message, KEY_MESSAGE, null);
|
||||
final byte[] content = AttributeValues.getByteArray(message, KEY_CONTENT, null);
|
||||
return new OutgoingMessageEntity(-1L, false, messageUuid, type, relay, timestamp, source, sourceUuid, sourceDevice, messageBytes, content, sortKey.getServerTimestamp());
|
||||
}
|
||||
|
||||
private void deleteRowsMatchingQuery(byte[] partitionKey, QuerySpec querySpec) {
|
||||
final Table table = getDynamoDb().getTable(tableName);
|
||||
writeInBatches(table.query(querySpec), (itemBatch) -> deleteItems(partitionKey, itemBatch));
|
||||
private void deleteRowsMatchingQuery(AttributeValue partitionKey, QueryRequest querySpec) {
|
||||
writeInBatches(db().query(querySpec).items(), (itemBatch) -> deleteItems(partitionKey, itemBatch));
|
||||
}
|
||||
|
||||
private void deleteItems(byte[] partitionKey, List<Item> items) {
|
||||
final TableWriteItems tableWriteItems = new TableWriteItems(tableName);
|
||||
items.stream().map(item -> new PrimaryKey(KEY_PARTITION, partitionKey, KEY_SORT, item.getBinary(KEY_SORT))).forEach(tableWriteItems::addPrimaryKeyToDelete);
|
||||
executeTableWriteItemsUntilComplete(tableWriteItems);
|
||||
private void deleteItems(AttributeValue partitionKey, List<Map<String, AttributeValue>> items) {
|
||||
List<WriteRequest> deletes = items.stream()
|
||||
.map(item -> WriteRequest.builder()
|
||||
.deleteRequest(DeleteRequest.builder().key(Map.of(
|
||||
KEY_PARTITION, partitionKey,
|
||||
KEY_SORT, item.get(KEY_SORT))).build())
|
||||
.build())
|
||||
.collect(Collectors.toList());
|
||||
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
|
||||
}
|
||||
|
||||
private long getTtlForMessage(MessageProtos.Envelope message) {
|
||||
return message.getServerTimestamp() / 1000 + timeToLive.getSeconds();
|
||||
}
|
||||
|
||||
private static byte[] convertPartitionKey(final UUID destinationAccountUuid) {
|
||||
return UUIDUtil.toBytes(destinationAccountUuid);
|
||||
private static AttributeValue convertPartitionKey(final UUID destinationAccountUuid) {
|
||||
return AttributeValues.fromUUID(destinationAccountUuid);
|
||||
}
|
||||
|
||||
private static byte[] convertSortKey(final long destinationDeviceId, final long serverTimestamp, final UUID messageUuid) {
|
||||
private static AttributeValue convertSortKey(final long destinationDeviceId, final long serverTimestamp, final UUID messageUuid) {
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[32]);
|
||||
byteBuffer.putLong(destinationDeviceId);
|
||||
byteBuffer.putLong(serverTimestamp);
|
||||
byteBuffer.putLong(messageUuid.getMostSignificantBits());
|
||||
byteBuffer.putLong(messageUuid.getLeastSignificantBits());
|
||||
return byteBuffer.array();
|
||||
return AttributeValues.fromByteBuffer(byteBuffer.flip());
|
||||
}
|
||||
|
||||
private static byte[] convertDestinationDeviceIdToSortKeyPrefix(final long destinationDeviceId) {
|
||||
private static AttributeValue convertDestinationDeviceIdToSortKeyPrefix(final long destinationDeviceId) {
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
|
||||
byteBuffer.putLong(destinationDeviceId);
|
||||
return byteBuffer.array();
|
||||
return AttributeValues.fromByteBuffer(byteBuffer.flip());
|
||||
}
|
||||
|
||||
private static SortKey convertSortKey(final byte[] bytes) {
|
||||
@@ -273,8 +305,8 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
|
||||
return new SortKey(destinationDeviceId, serverTimestamp, new UUID(mostSigBits, leastSigBits));
|
||||
}
|
||||
|
||||
private static byte[] convertLocalIndexMessageUuidSortKey(final UUID messageUuid) {
|
||||
return UUIDUtil.toBytes(messageUuid);
|
||||
private static AttributeValue convertLocalIndexMessageUuidSortKey(final UUID messageUuid) {
|
||||
return AttributeValues.fromUUID(messageUuid);
|
||||
}
|
||||
|
||||
private static UUID convertLocalIndexMessageUuidSortKey(final byte[] bytes) {
|
||||
|
||||
@@ -1,42 +1,52 @@
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import java.util.stream.Collectors;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
|
||||
|
||||
public class MigrationDeletedAccounts extends AbstractDynamoDbStore {
|
||||
|
||||
private final Table table;
|
||||
private final String tableName;
|
||||
|
||||
static final String KEY_UUID = "U";
|
||||
|
||||
public MigrationDeletedAccounts(DynamoDB dynamoDb, String tableName) {
|
||||
public MigrationDeletedAccounts(DynamoDbClient dynamoDb, String tableName) {
|
||||
super(dynamoDb);
|
||||
|
||||
table = dynamoDb.getTable(tableName);
|
||||
this.tableName = tableName;
|
||||
}
|
||||
|
||||
public void put(UUID uuid) {
|
||||
table.putItem(new Item()
|
||||
.withPrimaryKey(primaryKey(uuid)));
|
||||
db().putItem(PutItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.item(primaryKey(uuid))
|
||||
.build());
|
||||
}
|
||||
|
||||
public List<UUID> getRecentlyDeletedUuids() {
|
||||
|
||||
final List<UUID> uuids = new ArrayList<>();
|
||||
Optional<ScanResponse> firstPage = db().scanPaginator(ScanRequest.builder()
|
||||
.tableName(tableName)
|
||||
.build()).stream().findAny(); // get the first available response
|
||||
|
||||
for (Item item : table.scan(new ScanSpec()).firstPage()) {
|
||||
// only process one page each time. If we have a significant backlog at the end of the migration
|
||||
// we can handle it separately
|
||||
uuids.add(UUIDUtil.fromByteBuffer(item.getByteBuffer(KEY_UUID)));
|
||||
if (firstPage.isPresent()) {
|
||||
for (Map<String, AttributeValue> item : firstPage.get().items()) {
|
||||
// only process one page each time. If we have a significant backlog at the end of the migration
|
||||
// we can handle it separately
|
||||
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
|
||||
}
|
||||
}
|
||||
|
||||
return uuids;
|
||||
@@ -45,20 +55,17 @@ public class MigrationDeletedAccounts extends AbstractDynamoDbStore {
|
||||
public void delete(List<UUID> uuids) {
|
||||
|
||||
writeInBatches(uuids, (batch) -> {
|
||||
List<WriteRequest> deletes = batch.stream().map((uuid) -> WriteRequest.builder().deleteRequest(DeleteRequest.builder()
|
||||
.key(primaryKey(uuid))
|
||||
.build()).build()).collect(Collectors.toList());
|
||||
|
||||
final TableWriteItems deleteItems = new TableWriteItems(table.getTableName());
|
||||
|
||||
for (UUID uuid : batch) {
|
||||
deleteItems.addPrimaryKeyToDelete(primaryKey(uuid));
|
||||
}
|
||||
|
||||
executeTableWriteItemsUntilComplete(deleteItems);
|
||||
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
|
||||
});
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static PrimaryKey primaryKey(UUID uuid) {
|
||||
return new PrimaryKey(KEY_UUID, UUIDUtil.toBytes(uuid));
|
||||
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
|
||||
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,43 +1,44 @@
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.Page;
|
||||
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
|
||||
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
|
||||
|
||||
public class MigrationRetryAccounts extends AbstractDynamoDbStore {
|
||||
|
||||
private final Table table;
|
||||
private final String tableName;
|
||||
|
||||
static final String KEY_UUID = "U";
|
||||
|
||||
public MigrationRetryAccounts(DynamoDB dynamoDb, String tableName) {
|
||||
public MigrationRetryAccounts(DynamoDbClient dynamoDb, String tableName) {
|
||||
super(dynamoDb);
|
||||
|
||||
table = dynamoDb.getTable(tableName);
|
||||
this.tableName = tableName;
|
||||
}
|
||||
|
||||
public void put(UUID uuid) {
|
||||
table.putItem(new Item()
|
||||
.withPrimaryKey(primaryKey(uuid)));
|
||||
db().putItem(PutItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.item(primaryKey(uuid))
|
||||
.build());
|
||||
}
|
||||
|
||||
public List<UUID> getUuids(int max) {
|
||||
|
||||
final List<UUID> uuids = new ArrayList<>();
|
||||
|
||||
for (Page<Item, ScanOutcome> page : table.scan(new ScanSpec()).pages()) {
|
||||
for (ScanResponse response : db().scanPaginator(ScanRequest.builder().tableName(tableName).build())) {
|
||||
|
||||
for (Item item : page) {
|
||||
uuids.add(UUIDUtil.fromByteBuffer(item.getByteBuffer(KEY_UUID)));
|
||||
for (Map<String, AttributeValue> item : response.items()) {
|
||||
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
|
||||
|
||||
if (uuids.size() >= max) {
|
||||
break;
|
||||
@@ -53,8 +54,8 @@ public class MigrationRetryAccounts extends AbstractDynamoDbStore {
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static PrimaryKey primaryKey(UUID uuid) {
|
||||
return new PrimaryKey(KEY_UUID, UUIDUtil.toBytes(uuid));
|
||||
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
|
||||
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,666 +0,0 @@
|
||||
/*
|
||||
* Copyright 2013-2020 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
// Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
// source: PubSubMessage.proto
|
||||
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
public final class PubSubProtos {
|
||||
private PubSubProtos() {}
|
||||
public static void registerAllExtensions(
|
||||
com.google.protobuf.ExtensionRegistry registry) {
|
||||
}
|
||||
public interface PubSubMessageOrBuilder
|
||||
extends com.google.protobuf.MessageOrBuilder {
|
||||
|
||||
// optional .textsecure.PubSubMessage.Type type = 1;
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
boolean hasType();
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type getType();
|
||||
|
||||
// optional bytes content = 2;
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
boolean hasContent();
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
com.google.protobuf.ByteString getContent();
|
||||
}
|
||||
/**
|
||||
* Protobuf type {@code textsecure.PubSubMessage}
|
||||
*/
|
||||
public static final class PubSubMessage extends
|
||||
com.google.protobuf.GeneratedMessage
|
||||
implements PubSubMessageOrBuilder {
|
||||
// Use PubSubMessage.newBuilder() to construct.
|
||||
private PubSubMessage(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
|
||||
super(builder);
|
||||
this.unknownFields = builder.getUnknownFields();
|
||||
}
|
||||
private PubSubMessage(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
|
||||
|
||||
private static final PubSubMessage defaultInstance;
|
||||
public static PubSubMessage getDefaultInstance() {
|
||||
return defaultInstance;
|
||||
}
|
||||
|
||||
public PubSubMessage getDefaultInstanceForType() {
|
||||
return defaultInstance;
|
||||
}
|
||||
|
||||
private final com.google.protobuf.UnknownFieldSet unknownFields;
|
||||
@java.lang.Override
|
||||
public final com.google.protobuf.UnknownFieldSet
|
||||
getUnknownFields() {
|
||||
return this.unknownFields;
|
||||
}
|
||||
private PubSubMessage(
|
||||
com.google.protobuf.CodedInputStream input,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws com.google.protobuf.InvalidProtocolBufferException {
|
||||
initFields();
|
||||
int mutable_bitField0_ = 0;
|
||||
com.google.protobuf.UnknownFieldSet.Builder unknownFields =
|
||||
com.google.protobuf.UnknownFieldSet.newBuilder();
|
||||
try {
|
||||
boolean done = false;
|
||||
while (!done) {
|
||||
int tag = input.readTag();
|
||||
switch (tag) {
|
||||
case 0:
|
||||
done = true;
|
||||
break;
|
||||
default: {
|
||||
if (!parseUnknownField(input, unknownFields,
|
||||
extensionRegistry, tag)) {
|
||||
done = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
int rawValue = input.readEnum();
|
||||
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type value = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.valueOf(rawValue);
|
||||
if (value == null) {
|
||||
unknownFields.mergeVarintField(1, rawValue);
|
||||
} else {
|
||||
bitField0_ |= 0x00000001;
|
||||
type_ = value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 18: {
|
||||
bitField0_ |= 0x00000002;
|
||||
content_ = input.readBytes();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
|
||||
throw e.setUnfinishedMessage(this);
|
||||
} catch (java.io.IOException e) {
|
||||
throw new com.google.protobuf.InvalidProtocolBufferException(
|
||||
e.getMessage()).setUnfinishedMessage(this);
|
||||
} finally {
|
||||
this.unknownFields = unknownFields.build();
|
||||
makeExtensionsImmutable();
|
||||
}
|
||||
}
|
||||
public static final com.google.protobuf.Descriptors.Descriptor
|
||||
getDescriptor() {
|
||||
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_descriptor;
|
||||
}
|
||||
|
||||
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
|
||||
internalGetFieldAccessorTable() {
|
||||
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_fieldAccessorTable
|
||||
.ensureFieldAccessorsInitialized(
|
||||
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.class, org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Builder.class);
|
||||
}
|
||||
|
||||
public static com.google.protobuf.Parser<PubSubMessage> PARSER =
|
||||
new com.google.protobuf.AbstractParser<PubSubMessage>() {
|
||||
public PubSubMessage parsePartialFrom(
|
||||
com.google.protobuf.CodedInputStream input,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws com.google.protobuf.InvalidProtocolBufferException {
|
||||
return new PubSubMessage(input, extensionRegistry);
|
||||
}
|
||||
};
|
||||
|
||||
@java.lang.Override
|
||||
public com.google.protobuf.Parser<PubSubMessage> getParserForType() {
|
||||
return PARSER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Protobuf enum {@code textsecure.PubSubMessage.Type}
|
||||
*/
|
||||
public enum Type
|
||||
implements com.google.protobuf.ProtocolMessageEnum {
|
||||
/**
|
||||
* <code>UNKNOWN = 0;</code>
|
||||
*/
|
||||
UNKNOWN(0, 0),
|
||||
/**
|
||||
* <code>QUERY_DB = 1;</code>
|
||||
*/
|
||||
QUERY_DB(1, 1),
|
||||
/**
|
||||
* <code>DELIVER = 2;</code>
|
||||
*/
|
||||
DELIVER(2, 2),
|
||||
/**
|
||||
* <code>KEEPALIVE = 3;</code>
|
||||
*/
|
||||
KEEPALIVE(3, 3),
|
||||
/**
|
||||
* <code>CLOSE = 4;</code>
|
||||
*/
|
||||
CLOSE(4, 4),
|
||||
/**
|
||||
* <code>CONNECTED = 5;</code>
|
||||
*/
|
||||
CONNECTED(5, 5),
|
||||
;
|
||||
|
||||
/**
|
||||
* <code>UNKNOWN = 0;</code>
|
||||
*/
|
||||
public static final int UNKNOWN_VALUE = 0;
|
||||
/**
|
||||
* <code>QUERY_DB = 1;</code>
|
||||
*/
|
||||
public static final int QUERY_DB_VALUE = 1;
|
||||
/**
|
||||
* <code>DELIVER = 2;</code>
|
||||
*/
|
||||
public static final int DELIVER_VALUE = 2;
|
||||
/**
|
||||
* <code>KEEPALIVE = 3;</code>
|
||||
*/
|
||||
public static final int KEEPALIVE_VALUE = 3;
|
||||
/**
|
||||
* <code>CLOSE = 4;</code>
|
||||
*/
|
||||
public static final int CLOSE_VALUE = 4;
|
||||
/**
|
||||
* <code>CONNECTED = 5;</code>
|
||||
*/
|
||||
public static final int CONNECTED_VALUE = 5;
|
||||
|
||||
|
||||
public final int getNumber() { return value; }
|
||||
|
||||
public static Type valueOf(int value) {
|
||||
switch (value) {
|
||||
case 0: return UNKNOWN;
|
||||
case 1: return QUERY_DB;
|
||||
case 2: return DELIVER;
|
||||
case 3: return KEEPALIVE;
|
||||
case 4: return CLOSE;
|
||||
case 5: return CONNECTED;
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static com.google.protobuf.Internal.EnumLiteMap<Type>
|
||||
internalGetValueMap() {
|
||||
return internalValueMap;
|
||||
}
|
||||
private static com.google.protobuf.Internal.EnumLiteMap<Type>
|
||||
internalValueMap =
|
||||
new com.google.protobuf.Internal.EnumLiteMap<Type>() {
|
||||
public Type findValueByNumber(int number) {
|
||||
return Type.valueOf(number);
|
||||
}
|
||||
};
|
||||
|
||||
public final com.google.protobuf.Descriptors.EnumValueDescriptor
|
||||
getValueDescriptor() {
|
||||
return getDescriptor().getValues().get(index);
|
||||
}
|
||||
public final com.google.protobuf.Descriptors.EnumDescriptor
|
||||
getDescriptorForType() {
|
||||
return getDescriptor();
|
||||
}
|
||||
public static final com.google.protobuf.Descriptors.EnumDescriptor
|
||||
getDescriptor() {
|
||||
return org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.getDescriptor().getEnumTypes().get(0);
|
||||
}
|
||||
|
||||
private static final Type[] VALUES = values();
|
||||
|
||||
public static Type valueOf(
|
||||
com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
|
||||
if (desc.getType() != getDescriptor()) {
|
||||
throw new java.lang.IllegalArgumentException(
|
||||
"EnumValueDescriptor is not for this type.");
|
||||
}
|
||||
return VALUES[desc.getIndex()];
|
||||
}
|
||||
|
||||
private final int index;
|
||||
private final int value;
|
||||
|
||||
private Type(int index, int value) {
|
||||
this.index = index;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(enum_scope:textsecure.PubSubMessage.Type)
|
||||
}
|
||||
|
||||
private int bitField0_;
|
||||
// optional .textsecure.PubSubMessage.Type type = 1;
|
||||
public static final int TYPE_FIELD_NUMBER = 1;
|
||||
private org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type type_;
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
public boolean hasType() {
|
||||
return ((bitField0_ & 0x00000001) == 0x00000001);
|
||||
}
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type getType() {
|
||||
return type_;
|
||||
}
|
||||
|
||||
// optional bytes content = 2;
|
||||
public static final int CONTENT_FIELD_NUMBER = 2;
|
||||
private com.google.protobuf.ByteString content_;
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
public boolean hasContent() {
|
||||
return ((bitField0_ & 0x00000002) == 0x00000002);
|
||||
}
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
public com.google.protobuf.ByteString getContent() {
|
||||
return content_;
|
||||
}
|
||||
|
||||
private void initFields() {
|
||||
type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
|
||||
content_ = com.google.protobuf.ByteString.EMPTY;
|
||||
}
|
||||
private byte memoizedIsInitialized = -1;
|
||||
public final boolean isInitialized() {
|
||||
byte isInitialized = memoizedIsInitialized;
|
||||
if (isInitialized != -1) return isInitialized == 1;
|
||||
|
||||
memoizedIsInitialized = 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
public void writeTo(com.google.protobuf.CodedOutputStream output)
|
||||
throws java.io.IOException {
|
||||
getSerializedSize();
|
||||
if (((bitField0_ & 0x00000001) == 0x00000001)) {
|
||||
output.writeEnum(1, type_.getNumber());
|
||||
}
|
||||
if (((bitField0_ & 0x00000002) == 0x00000002)) {
|
||||
output.writeBytes(2, content_);
|
||||
}
|
||||
getUnknownFields().writeTo(output);
|
||||
}
|
||||
|
||||
private int memoizedSerializedSize = -1;
|
||||
public int getSerializedSize() {
|
||||
int size = memoizedSerializedSize;
|
||||
if (size != -1) return size;
|
||||
|
||||
size = 0;
|
||||
if (((bitField0_ & 0x00000001) == 0x00000001)) {
|
||||
size += com.google.protobuf.CodedOutputStream
|
||||
.computeEnumSize(1, type_.getNumber());
|
||||
}
|
||||
if (((bitField0_ & 0x00000002) == 0x00000002)) {
|
||||
size += com.google.protobuf.CodedOutputStream
|
||||
.computeBytesSize(2, content_);
|
||||
}
|
||||
size += getUnknownFields().getSerializedSize();
|
||||
memoizedSerializedSize = size;
|
||||
return size;
|
||||
}
|
||||
|
||||
private static final long serialVersionUID = 0L;
|
||||
@java.lang.Override
|
||||
protected java.lang.Object writeReplace()
|
||||
throws java.io.ObjectStreamException {
|
||||
return super.writeReplace();
|
||||
}
|
||||
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
|
||||
com.google.protobuf.ByteString data)
|
||||
throws com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
|
||||
com.google.protobuf.ByteString data,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data, extensionRegistry);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(byte[] data)
|
||||
throws com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
|
||||
byte[] data,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws com.google.protobuf.InvalidProtocolBufferException {
|
||||
return PARSER.parseFrom(data, extensionRegistry);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(java.io.InputStream input)
|
||||
throws java.io.IOException {
|
||||
return PARSER.parseFrom(input);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
|
||||
java.io.InputStream input,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
return PARSER.parseFrom(input, extensionRegistry);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseDelimitedFrom(java.io.InputStream input)
|
||||
throws java.io.IOException {
|
||||
return PARSER.parseDelimitedFrom(input);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseDelimitedFrom(
|
||||
java.io.InputStream input,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
return PARSER.parseDelimitedFrom(input, extensionRegistry);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
|
||||
com.google.protobuf.CodedInputStream input)
|
||||
throws java.io.IOException {
|
||||
return PARSER.parseFrom(input);
|
||||
}
|
||||
public static org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parseFrom(
|
||||
com.google.protobuf.CodedInputStream input,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
return PARSER.parseFrom(input, extensionRegistry);
|
||||
}
|
||||
|
||||
public static Builder newBuilder() { return Builder.create(); }
|
||||
public Builder newBuilderForType() { return newBuilder(); }
|
||||
public static Builder newBuilder(org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage prototype) {
|
||||
return newBuilder().mergeFrom(prototype);
|
||||
}
|
||||
public Builder toBuilder() { return newBuilder(this); }
|
||||
|
||||
@java.lang.Override
|
||||
protected Builder newBuilderForType(
|
||||
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
|
||||
Builder builder = new Builder(parent);
|
||||
return builder;
|
||||
}
|
||||
/**
|
||||
* Protobuf type {@code textsecure.PubSubMessage}
|
||||
*/
|
||||
public static final class Builder extends
|
||||
com.google.protobuf.GeneratedMessage.Builder<Builder>
|
||||
implements org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessageOrBuilder {
|
||||
public static final com.google.protobuf.Descriptors.Descriptor
|
||||
getDescriptor() {
|
||||
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_descriptor;
|
||||
}
|
||||
|
||||
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
|
||||
internalGetFieldAccessorTable() {
|
||||
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_fieldAccessorTable
|
||||
.ensureFieldAccessorsInitialized(
|
||||
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.class, org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Builder.class);
|
||||
}
|
||||
|
||||
// Construct using org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.newBuilder()
|
||||
private Builder() {
|
||||
maybeForceBuilderInitialization();
|
||||
}
|
||||
|
||||
private Builder(
|
||||
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
|
||||
super(parent);
|
||||
maybeForceBuilderInitialization();
|
||||
}
|
||||
private void maybeForceBuilderInitialization() {
|
||||
if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
|
||||
}
|
||||
}
|
||||
private static Builder create() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public Builder clear() {
|
||||
super.clear();
|
||||
type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
|
||||
bitField0_ = (bitField0_ & ~0x00000001);
|
||||
content_ = com.google.protobuf.ByteString.EMPTY;
|
||||
bitField0_ = (bitField0_ & ~0x00000002);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder clone() {
|
||||
return create().mergeFrom(buildPartial());
|
||||
}
|
||||
|
||||
public com.google.protobuf.Descriptors.Descriptor
|
||||
getDescriptorForType() {
|
||||
return org.whispersystems.textsecuregcm.storage.PubSubProtos.internal_static_textsecure_PubSubMessage_descriptor;
|
||||
}
|
||||
|
||||
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage getDefaultInstanceForType() {
|
||||
return org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.getDefaultInstance();
|
||||
}
|
||||
|
||||
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage build() {
|
||||
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage result = buildPartial();
|
||||
if (!result.isInitialized()) {
|
||||
throw newUninitializedMessageException(result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage buildPartial() {
|
||||
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage result = new org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage(this);
|
||||
int from_bitField0_ = bitField0_;
|
||||
int to_bitField0_ = 0;
|
||||
if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
|
||||
to_bitField0_ |= 0x00000001;
|
||||
}
|
||||
result.type_ = type_;
|
||||
if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
|
||||
to_bitField0_ |= 0x00000002;
|
||||
}
|
||||
result.content_ = content_;
|
||||
result.bitField0_ = to_bitField0_;
|
||||
onBuilt();
|
||||
return result;
|
||||
}
|
||||
|
||||
public Builder mergeFrom(com.google.protobuf.Message other) {
|
||||
if (other instanceof org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage) {
|
||||
return mergeFrom((org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage)other);
|
||||
} else {
|
||||
super.mergeFrom(other);
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
public Builder mergeFrom(org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage other) {
|
||||
if (other == org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.getDefaultInstance()) return this;
|
||||
if (other.hasType()) {
|
||||
setType(other.getType());
|
||||
}
|
||||
if (other.hasContent()) {
|
||||
setContent(other.getContent());
|
||||
}
|
||||
this.mergeUnknownFields(other.getUnknownFields());
|
||||
return this;
|
||||
}
|
||||
|
||||
public final boolean isInitialized() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public Builder mergeFrom(
|
||||
com.google.protobuf.CodedInputStream input,
|
||||
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
|
||||
throws java.io.IOException {
|
||||
org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage parsedMessage = null;
|
||||
try {
|
||||
parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
|
||||
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
|
||||
parsedMessage = (org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage) e.getUnfinishedMessage();
|
||||
throw e;
|
||||
} finally {
|
||||
if (parsedMessage != null) {
|
||||
mergeFrom(parsedMessage);
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
private int bitField0_;
|
||||
|
||||
// optional .textsecure.PubSubMessage.Type type = 1;
|
||||
private org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
public boolean hasType() {
|
||||
return ((bitField0_ & 0x00000001) == 0x00000001);
|
||||
}
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
public org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type getType() {
|
||||
return type_;
|
||||
}
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
public Builder setType(org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type value) {
|
||||
if (value == null) {
|
||||
throw new NullPointerException();
|
||||
}
|
||||
bitField0_ |= 0x00000001;
|
||||
type_ = value;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* <code>optional .textsecure.PubSubMessage.Type type = 1;</code>
|
||||
*/
|
||||
public Builder clearType() {
|
||||
bitField0_ = (bitField0_ & ~0x00000001);
|
||||
type_ = org.whispersystems.textsecuregcm.storage.PubSubProtos.PubSubMessage.Type.UNKNOWN;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
|
||||
// optional bytes content = 2;
|
||||
private com.google.protobuf.ByteString content_ = com.google.protobuf.ByteString.EMPTY;
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
public boolean hasContent() {
|
||||
return ((bitField0_ & 0x00000002) == 0x00000002);
|
||||
}
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
public com.google.protobuf.ByteString getContent() {
|
||||
return content_;
|
||||
}
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
public Builder setContent(com.google.protobuf.ByteString value) {
|
||||
if (value == null) {
|
||||
throw new NullPointerException();
|
||||
}
|
||||
bitField0_ |= 0x00000002;
|
||||
content_ = value;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* <code>optional bytes content = 2;</code>
|
||||
*/
|
||||
public Builder clearContent() {
|
||||
bitField0_ = (bitField0_ & ~0x00000002);
|
||||
content_ = getDefaultInstance().getContent();
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(builder_scope:textsecure.PubSubMessage)
|
||||
}
|
||||
|
||||
static {
|
||||
defaultInstance = new PubSubMessage(true);
|
||||
defaultInstance.initFields();
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(class_scope:textsecure.PubSubMessage)
|
||||
}
|
||||
|
||||
private static com.google.protobuf.Descriptors.Descriptor
|
||||
internal_static_textsecure_PubSubMessage_descriptor;
|
||||
private static
|
||||
com.google.protobuf.GeneratedMessage.FieldAccessorTable
|
||||
internal_static_textsecure_PubSubMessage_fieldAccessorTable;
|
||||
|
||||
public static com.google.protobuf.Descriptors.FileDescriptor
|
||||
getDescriptor() {
|
||||
return descriptor;
|
||||
}
|
||||
private static com.google.protobuf.Descriptors.FileDescriptor
|
||||
descriptor;
|
||||
static {
|
||||
java.lang.String[] descriptorData = {
|
||||
"\n\023PubSubMessage.proto\022\ntextsecure\"\247\001\n\rPu" +
|
||||
"bSubMessage\022,\n\004type\030\001 \001(\0162\036.textsecure.P" +
|
||||
"ubSubMessage.Type\022\017\n\007content\030\002 \001(\014\"W\n\004Ty" +
|
||||
"pe\022\013\n\007UNKNOWN\020\000\022\014\n\010QUERY_DB\020\001\022\013\n\007DELIVER" +
|
||||
"\020\002\022\r\n\tKEEPALIVE\020\003\022\t\n\005CLOSE\020\004\022\r\n\tCONNECTE" +
|
||||
"D\020\005B8\n(org.whispersystems.textsecuregcm." +
|
||||
"storageB\014PubSubProtos"
|
||||
};
|
||||
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
|
||||
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
|
||||
public com.google.protobuf.ExtensionRegistry assignDescriptors(
|
||||
com.google.protobuf.Descriptors.FileDescriptor root) {
|
||||
descriptor = root;
|
||||
internal_static_textsecure_PubSubMessage_descriptor =
|
||||
getDescriptor().getMessageTypes().get(0);
|
||||
internal_static_textsecure_PubSubMessage_fieldAccessorTable = new
|
||||
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
|
||||
internal_static_textsecure_PubSubMessage_descriptor,
|
||||
new java.lang.String[] { "Type", "Content", });
|
||||
return null;
|
||||
}
|
||||
};
|
||||
com.google.protobuf.Descriptors.FileDescriptor
|
||||
.internalBuildGeneratedFileFrom(descriptorData,
|
||||
new com.google.protobuf.Descriptors.FileDescriptor[] {
|
||||
}, assigner);
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(outer_class_scope)
|
||||
}
|
||||
@@ -5,25 +5,23 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.PutItemSpec;
|
||||
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
|
||||
|
||||
/**
|
||||
* Stores push challenge tokens. Users may have at most one outstanding push challenge token at a time.
|
||||
*/
|
||||
public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
|
||||
|
||||
private final Table table;
|
||||
private final String tableName;
|
||||
private final Clock clock;
|
||||
|
||||
static final String KEY_ACCOUNT_UUID = "U";
|
||||
@@ -33,15 +31,15 @@ public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
|
||||
private static final Map<String, String> UUID_NAME_MAP = Map.of("#uuid", KEY_ACCOUNT_UUID);
|
||||
private static final Map<String, String> CHALLENGE_TOKEN_NAME_MAP = Map.of("#challenge", ATTR_CHALLENGE_TOKEN);
|
||||
|
||||
public PushChallengeDynamoDb(final DynamoDB dynamoDB, final String tableName) {
|
||||
public PushChallengeDynamoDb(final DynamoDbClient dynamoDB, final String tableName) {
|
||||
this(dynamoDB, tableName, Clock.systemUTC());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
PushChallengeDynamoDb(final DynamoDB dynamoDB, final String tableName, final Clock clock) {
|
||||
PushChallengeDynamoDb(final DynamoDbClient dynamoDB, final String tableName, final Clock clock) {
|
||||
super(dynamoDB);
|
||||
|
||||
this.table = dynamoDB.getTable(tableName);
|
||||
this.tableName = tableName;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
@@ -57,13 +55,15 @@ public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
|
||||
*/
|
||||
public boolean add(final UUID accountUuid, final byte[] challengeToken, final Duration ttl) {
|
||||
try {
|
||||
table.putItem( new PutItemSpec()
|
||||
.withItem(new Item()
|
||||
.withBinary(KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(accountUuid))
|
||||
.withBinary(ATTR_CHALLENGE_TOKEN, challengeToken)
|
||||
.withNumber(ATTR_TTL, getExpirationTimestamp(ttl)))
|
||||
.withConditionExpression("attribute_not_exists(#uuid)")
|
||||
.withNameMap(UUID_NAME_MAP));
|
||||
db().putItem(PutItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.item(Map.of(
|
||||
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(accountUuid),
|
||||
ATTR_CHALLENGE_TOKEN, AttributeValues.fromByteArray(challengeToken),
|
||||
ATTR_TTL, AttributeValues.fromLong(getExpirationTimestamp(ttl))))
|
||||
.conditionExpression("attribute_not_exists(#uuid)")
|
||||
.expressionAttributeNames(UUID_NAME_MAP)
|
||||
.build());
|
||||
return true;
|
||||
} catch (final ConditionalCheckFailedException e) {
|
||||
return false;
|
||||
@@ -84,11 +84,13 @@ public class PushChallengeDynamoDb extends AbstractDynamoDbStore {
|
||||
*/
|
||||
public boolean remove(final UUID accountUuid, final byte[] challengeToken) {
|
||||
try {
|
||||
table.deleteItem(new DeleteItemSpec()
|
||||
.withPrimaryKey(KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(accountUuid))
|
||||
.withConditionExpression("#challenge = :challenge")
|
||||
.withNameMap(CHALLENGE_TOKEN_NAME_MAP)
|
||||
.withValueMap(Map.of(":challenge", challengeToken)));
|
||||
db().deleteItem(DeleteItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(accountUuid)))
|
||||
.conditionExpression("#challenge = :challenge")
|
||||
.expressionAttributeNames(CHALLENGE_TOKEN_NAME_MAP)
|
||||
.expressionAttributeValues(Map.of(":challenge", AttributeValues.fromByteArray(challengeToken)))
|
||||
.build());
|
||||
return true;
|
||||
} catch (final ConditionalCheckFailedException e) {
|
||||
return false;
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DeleteItemOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
|
||||
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
public class ReportMessageDynamoDb {
|
||||
|
||||
@@ -16,33 +17,30 @@ public class ReportMessageDynamoDb {
|
||||
|
||||
static final Duration TIME_TO_LIVE = Duration.ofDays(7);
|
||||
|
||||
private final Table table;
|
||||
private final DynamoDbClient db;
|
||||
private final String tableName;
|
||||
|
||||
public ReportMessageDynamoDb(final DynamoDB dynamoDB, final String tableName) {
|
||||
|
||||
this.table = dynamoDB.getTable(tableName);
|
||||
public ReportMessageDynamoDb(final DynamoDbClient dynamoDB, final String tableName) {
|
||||
this.db = dynamoDB;
|
||||
this.tableName = tableName;
|
||||
}
|
||||
|
||||
public void store(byte[] hash) {
|
||||
|
||||
table.putItem(buildItemForHash(hash));
|
||||
}
|
||||
|
||||
private Item buildItemForHash(byte[] hash) {
|
||||
return new Item()
|
||||
.withBinary(KEY_HASH, hash)
|
||||
.withLong(ATTR_TTL, Instant.now().plus(TIME_TO_LIVE).getEpochSecond());
|
||||
db.putItem(PutItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.item(Map.of(
|
||||
KEY_HASH, AttributeValues.fromByteArray(hash),
|
||||
ATTR_TTL, AttributeValues.fromLong(Instant.now().plus(TIME_TO_LIVE).getEpochSecond())
|
||||
))
|
||||
.build());
|
||||
}
|
||||
|
||||
public boolean remove(byte[] hash) {
|
||||
|
||||
final DeleteItemSpec deleteItemSpec = new DeleteItemSpec()
|
||||
.withPrimaryKey(KEY_HASH, hash)
|
||||
.withReturnValues(ReturnValue.ALL_OLD);
|
||||
|
||||
final DeleteItemOutcome outcome = table.deleteItem(deleteItemSpec);
|
||||
|
||||
return outcome.getItem() != null;
|
||||
final DeleteItemResponse deleteItemResponse = db.deleteItem(DeleteItemRequest.builder()
|
||||
.tableName(tableName)
|
||||
.key(Map.of(KEY_HASH, AttributeValues.fromByteArray(hash)))
|
||||
.returnValues(ReturnValue.ALL_OLD)
|
||||
.build());
|
||||
return !deleteItemResponse.attributes().isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.dropwizard.lifecycle.Managed;
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
@@ -53,12 +52,6 @@ public class AsnManager implements Managed {
|
||||
|
||||
@Override
|
||||
public void start() throws Exception {
|
||||
try {
|
||||
handleAsnTableChanged(asnTableMonitor.getObject());
|
||||
} catch (final Exception e) {
|
||||
log.warn("Failed to load initial IP-to-ASN map", e);
|
||||
}
|
||||
|
||||
asnTableMonitor.start();
|
||||
}
|
||||
|
||||
@@ -76,10 +69,10 @@ public class AsnManager implements Managed {
|
||||
}
|
||||
}
|
||||
|
||||
private void handleAsnTableChanged(final S3Object asnTableObject) {
|
||||
private void handleAsnTableChanged(final InputStream asnTableObject) {
|
||||
REFRESH_TIMER.record(() -> {
|
||||
try {
|
||||
handleAsnTableChanged(new GZIPInputStream(asnTableObject.getObjectContent()));
|
||||
handleAsnTableChangedStream(new GZIPInputStream(asnTableObject));
|
||||
} catch (final IOException e) {
|
||||
log.error("Retrieved object was not a gzip archive", e);
|
||||
}
|
||||
@@ -87,7 +80,7 @@ public class AsnManager implements Managed {
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void handleAsnTableChanged(final InputStream inputStream) {
|
||||
void handleAsnTableChangedStream(final InputStream inputStream) {
|
||||
try (final InputStreamReader reader = new InputStreamReader(inputStream)) {
|
||||
asnTable.set(new AsnTable(reader));
|
||||
} catch (final Exception e) {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Copyright 2013-2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import software.amazon.awssdk.core.SdkBytes;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
/** AwsAV provides static helper methods for working with AWS AttributeValues. */
|
||||
public class AttributeValues {
|
||||
|
||||
public static AttributeValue fromString(String value) {
|
||||
return AttributeValue.builder().s(value).build();
|
||||
}
|
||||
|
||||
public static AttributeValue fromLong(long value) {
|
||||
return AttributeValue.builder().n(Long.toString(value)).build();
|
||||
}
|
||||
|
||||
public static AttributeValue fromInt(int value) {
|
||||
return AttributeValue.builder().n(Integer.toString(value)).build();
|
||||
}
|
||||
|
||||
public static AttributeValue fromByteArray(byte[] value) {
|
||||
return AttributeValues.fromSdkBytes(SdkBytes.fromByteArray(value));
|
||||
}
|
||||
|
||||
public static AttributeValue fromByteBuffer(ByteBuffer value) {
|
||||
return AttributeValues.fromSdkBytes(SdkBytes.fromByteBuffer(value));
|
||||
}
|
||||
|
||||
public static AttributeValue fromUUID(UUID uuid) {
|
||||
return AttributeValues.fromSdkBytes(SdkBytes.fromByteArrayUnsafe(UUIDUtil.toBytes(uuid)));
|
||||
}
|
||||
|
||||
public static AttributeValue fromSdkBytes(SdkBytes value) {
|
||||
return AttributeValue.builder().b(value).build();
|
||||
}
|
||||
|
||||
private static int toInt(AttributeValue av) {
|
||||
return Integer.parseInt(av.n());
|
||||
}
|
||||
|
||||
private static long toLong(AttributeValue av) {
|
||||
return Long.parseLong(av.n());
|
||||
}
|
||||
|
||||
private static UUID toUUID(AttributeValue av) {
|
||||
return UUIDUtil.fromBytes(av.b().asByteArrayUnsafe()); // We're guaranteed not to modify the byte array
|
||||
}
|
||||
|
||||
private static byte[] toByteArray(AttributeValue av) {
|
||||
return av.b().asByteArray();
|
||||
}
|
||||
|
||||
private static String toString(AttributeValue av) {
|
||||
return av.s();
|
||||
}
|
||||
|
||||
public static Optional<AttributeValue> get(Map<String, AttributeValue> item, String key) {
|
||||
return Optional.ofNullable(item.get(key));
|
||||
}
|
||||
|
||||
public static int getInt(Map<String, AttributeValue> item, String key, int defaultValue) {
|
||||
return AttributeValues.get(item, key).map(AttributeValues::toInt).orElse(defaultValue);
|
||||
}
|
||||
|
||||
public static String getString(Map<String, AttributeValue> item, String key, String defaultValue) {
|
||||
return AttributeValues.get(item, key).map(AttributeValues::toString).orElse(defaultValue);
|
||||
}
|
||||
|
||||
public static long getLong(Map<String, AttributeValue> item, String key, long defaultValue) {
|
||||
return AttributeValues.get(item, key).map(AttributeValues::toLong).orElse(defaultValue);
|
||||
}
|
||||
|
||||
public static byte[] getByteArray(Map<String, AttributeValue> item, String key, byte[] defaultValue) {
|
||||
return AttributeValues.get(item, key).map(AttributeValues::toByteArray).orElse(defaultValue);
|
||||
}
|
||||
|
||||
public static UUID getUUID(Map<String, AttributeValue> item, String key, UUID defaultValue) {
|
||||
return AttributeValues.get(item, key).map(AttributeValues::toUUID).orElse(defaultValue);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import org.whispersystems.textsecuregcm.configuration.DynamoDbConfiguration;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration;
|
||||
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
|
||||
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClientBuilder;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
public class DynamoDbFromConfig {
|
||||
private static ClientOverrideConfiguration clientOverrideConfiguration(DynamoDbConfiguration config) {
|
||||
return ClientOverrideConfiguration.builder()
|
||||
.apiCallTimeout(config.getClientExecutionTimeout())
|
||||
.apiCallAttemptTimeout(config.getClientRequestTimeout())
|
||||
.build();
|
||||
}
|
||||
public static DynamoDbClient client(DynamoDbConfiguration config, AwsCredentialsProvider credentialsProvider) {
|
||||
return DynamoDbClient.builder()
|
||||
.region(Region.of(config.getRegion()))
|
||||
.credentialsProvider(credentialsProvider)
|
||||
.overrideConfiguration(clientOverrideConfiguration(config))
|
||||
.build();
|
||||
}
|
||||
public static DynamoDbAsyncClient asyncClient(DynamoDbConfiguration config, AwsCredentialsProvider credentialsProvider, Executor executor) {
|
||||
DynamoDbAsyncClientBuilder builder = DynamoDbAsyncClient.builder()
|
||||
.region(Region.of(config.getRegion()))
|
||||
.credentialsProvider(credentialsProvider)
|
||||
.overrideConfiguration(clientOverrideConfiguration(config));
|
||||
if (executor != null) {
|
||||
builder.asyncConfiguration(ClientAsyncConfiguration.builder()
|
||||
.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR,
|
||||
executor)
|
||||
.build());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
}
|
||||
@@ -5,14 +5,10 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.dropwizard.lifecycle.Managed;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
@@ -22,6 +18,14 @@ import java.util.function.Consumer;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
|
||||
import software.amazon.awssdk.core.ResponseInputStream;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
|
||||
|
||||
/**
|
||||
* An S3 object monitor watches a specific object in an S3 bucket and notifies a listener if that object changes.
|
||||
@@ -36,11 +40,11 @@ public class S3ObjectMonitor implements Managed {
|
||||
private final Duration refreshInterval;
|
||||
private ScheduledFuture<?> refreshFuture;
|
||||
|
||||
private final Consumer<S3Object> changeListener;
|
||||
private final Consumer<InputStream> changeListener;
|
||||
|
||||
private final AtomicReference<String> lastETag = new AtomicReference<>();
|
||||
|
||||
private final AmazonS3 s3Client;
|
||||
private final S3Client s3Client;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(S3ObjectMonitor.class);
|
||||
|
||||
@@ -51,11 +55,11 @@ public class S3ObjectMonitor implements Managed {
|
||||
final long maxObjectSize,
|
||||
final ScheduledExecutorService refreshExecutorService,
|
||||
final Duration refreshInterval,
|
||||
final Consumer<S3Object> changeListener) {
|
||||
final Consumer<InputStream> changeListener) {
|
||||
|
||||
this(AmazonS3ClientBuilder.standard()
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance())
|
||||
.withRegion(s3Region)
|
||||
this(S3Client.builder()
|
||||
.region(Region.of(s3Region))
|
||||
.credentialsProvider(InstanceProfileCredentialsProvider.create())
|
||||
.build(),
|
||||
s3Bucket,
|
||||
objectKey,
|
||||
@@ -67,13 +71,13 @@ public class S3ObjectMonitor implements Managed {
|
||||
|
||||
@VisibleForTesting
|
||||
S3ObjectMonitor(
|
||||
final AmazonS3 s3Client,
|
||||
final S3Client s3Client,
|
||||
final String s3Bucket,
|
||||
final String objectKey,
|
||||
final long maxObjectSize,
|
||||
final ScheduledExecutorService refreshExecutorService,
|
||||
final Duration refreshInterval,
|
||||
final Consumer<S3Object> changeListener) {
|
||||
final Consumer<InputStream> changeListener) {
|
||||
|
||||
this.s3Client = s3Client;
|
||||
this.s3Bucket = s3Bucket;
|
||||
@@ -92,8 +96,13 @@ public class S3ObjectMonitor implements Managed {
|
||||
throw new RuntimeException("S3 object manager already started");
|
||||
}
|
||||
|
||||
// Run the first request immediately/blocking, then start subsequent calls.
|
||||
log.info("Initial request for s3://{}/{}", s3Bucket, objectKey);
|
||||
refresh();
|
||||
|
||||
refreshFuture = refreshExecutorService
|
||||
.scheduleAtFixedRate(this::refresh, refreshInterval.toMillis(), refreshInterval.toMillis(), TimeUnit.MILLISECONDS);
|
||||
.scheduleAtFixedRate(this::refresh, refreshInterval.toMillis(), refreshInterval.toMillis(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -106,21 +115,24 @@ public class S3ObjectMonitor implements Managed {
|
||||
/**
|
||||
* Immediately returns the monitored S3 object regardless of whether it has changed since it was last retrieved.
|
||||
*
|
||||
* @return the current version of the monitored S3 object
|
||||
*
|
||||
* @return the current version of the monitored S3 object. Caller should close() this upon completion.
|
||||
* @throws IOException if the retrieved S3 object is larger than the configured maximum size
|
||||
*/
|
||||
public S3Object getObject() throws IOException {
|
||||
final S3Object s3Object = s3Client.getObject(s3Bucket, objectKey);
|
||||
@VisibleForTesting
|
||||
ResponseInputStream<GetObjectResponse> getObject() throws IOException {
|
||||
ResponseInputStream<GetObjectResponse> response = s3Client.getObject(GetObjectRequest.builder()
|
||||
.key(objectKey)
|
||||
.bucket(s3Bucket)
|
||||
.build());
|
||||
|
||||
lastETag.set(s3Object.getObjectMetadata().getETag());
|
||||
lastETag.set(response.response().eTag());
|
||||
|
||||
if (s3Object.getObjectMetadata().getContentLength() <= maxObjectSize) {
|
||||
return s3Object;
|
||||
if (response.response().contentLength() <= maxObjectSize) {
|
||||
return response;
|
||||
} else {
|
||||
log.warn("Object at s3://{}/{} has a size of {} bytes, which exceeds the maximum allowed size of {} bytes",
|
||||
s3Bucket, objectKey, s3Object.getObjectMetadata().getContentLength(), maxObjectSize);
|
||||
|
||||
s3Bucket, objectKey, response.response().contentLength(), maxObjectSize);
|
||||
response.abort();
|
||||
throw new IOException("S3 object too large");
|
||||
}
|
||||
}
|
||||
@@ -132,18 +144,20 @@ public class S3ObjectMonitor implements Managed {
|
||||
@VisibleForTesting
|
||||
void refresh() {
|
||||
try {
|
||||
final ObjectMetadata objectMetadata = s3Client.getObjectMetadata(s3Bucket, objectKey);
|
||||
final HeadObjectResponse objectMetadata = s3Client.headObject(HeadObjectRequest.builder()
|
||||
.bucket(s3Bucket)
|
||||
.key(objectKey)
|
||||
.build());
|
||||
|
||||
final String initialETag = lastETag.get();
|
||||
final String refreshedETag = objectMetadata.getETag();
|
||||
final String refreshedETag = objectMetadata.eTag();
|
||||
|
||||
if (!StringUtils.equals(initialETag, refreshedETag) && lastETag.compareAndSet(initialETag, refreshedETag)) {
|
||||
final S3Object s3Object = getObject();
|
||||
|
||||
log.info("Object at s3://{}/{} has changed; new eTag is {} and object size is {} bytes",
|
||||
s3Bucket, objectKey, s3Object.getObjectMetadata().getETag(), s3Object.getObjectMetadata().getContentLength());
|
||||
|
||||
changeListener.accept(s3Object);
|
||||
try (final ResponseInputStream<GetObjectResponse> response = getObject()) {
|
||||
log.info("Object at s3://{}/{} has changed; new eTag is {} and object size is {} bytes",
|
||||
s3Bucket, objectKey, response.response().eTag(), response.response().contentLength());
|
||||
changeListener.accept(response);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
log.warn("Failed to refresh monitored object", e);
|
||||
|
||||
@@ -7,14 +7,12 @@ package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.dropwizard.lifecycle.Managed;
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
import io.micrometer.core.instrument.Metrics;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.Collections;
|
||||
@@ -58,12 +56,6 @@ public class TorExitNodeManager implements Managed {
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
try {
|
||||
handleExitListChanged(exitListMonitor.getObject());
|
||||
} catch (final Exception e) {
|
||||
log.warn("Failed to load initial Tor exit node list", e);
|
||||
}
|
||||
|
||||
exitListMonitor.start();
|
||||
}
|
||||
|
||||
@@ -76,12 +68,12 @@ public class TorExitNodeManager implements Managed {
|
||||
return exitNodeAddresses.get().contains(address);
|
||||
}
|
||||
|
||||
private void handleExitListChanged(final S3Object exitList) {
|
||||
REFRESH_TIMER.record(() -> handleExitListChanged(exitList.getObjectContent()));
|
||||
private void handleExitListChanged(final InputStream exitList) {
|
||||
REFRESH_TIMER.record(() -> handleExitListChangedStream(exitList));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void handleExitListChanged(final InputStream inputStream) {
|
||||
void handleExitListChangedStream(final InputStream inputStream) {
|
||||
try (final BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
|
||||
exitNodeAddresses.set(reader.lines().collect(Collectors.toSet()));
|
||||
} catch (final Exception e) {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import java.nio.BufferUnderflowException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.UUID;
|
||||
|
||||
@@ -26,12 +27,15 @@ public class UUIDUtil {
|
||||
}
|
||||
|
||||
public static UUID fromByteBuffer(final ByteBuffer byteBuffer) {
|
||||
if (byteBuffer.array().length != 16) {
|
||||
throw new IllegalArgumentException("unexpected byte array length; was " + byteBuffer.array().length + " but expected 16");
|
||||
try {
|
||||
final long mostSigBits = byteBuffer.getLong();
|
||||
final long leastSigBits = byteBuffer.getLong();
|
||||
if (byteBuffer.hasRemaining()) {
|
||||
throw new IllegalArgumentException("unexpected byte array length; was greater than 16");
|
||||
}
|
||||
return new UUID(mostSigBits, leastSigBits);
|
||||
} catch (BufferUnderflowException e) {
|
||||
throw new IllegalArgumentException("unexpected byte array length; was less than 16");
|
||||
}
|
||||
|
||||
final long mostSigBits = byteBuffer.getLong();
|
||||
final long leastSigBits = byteBuffer.getLong();
|
||||
return new UUID(mostSigBits, leastSigBits);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,7 +171,7 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
|
||||
messagesManager.delete(account.getUuid(), device.getId(), storedMessageInfo.get().getGuid());
|
||||
}
|
||||
|
||||
if (message.getType() != Envelope.Type.RECEIPT) {
|
||||
if (message.getType() != Envelope.Type.SERVER_DELIVERY_RECEIPT) {
|
||||
recordMessageDeliveryDuration(message.getTimestamp(), device);
|
||||
sendDeliveryReceiptFor(message);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright 2013-2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.workers;
|
||||
|
||||
import io.dropwizard.cli.Command;
|
||||
import io.dropwizard.setup.Bootstrap;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import net.sourceforge.argparse4j.inf.Namespace;
|
||||
import net.sourceforge.argparse4j.inf.Subparser;
|
||||
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
|
||||
|
||||
public class CheckDynamicConfigurationCommand extends Command {
|
||||
|
||||
public CheckDynamicConfigurationCommand() {
|
||||
super("check-dynamic-config", "Check validity of a dynamic configuration file");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(final Subparser subparser) {
|
||||
subparser.addArgument("file")
|
||||
.type(String.class)
|
||||
.required(true)
|
||||
.help("Dynamic configuration file to check");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(final Bootstrap<?> bootstrap, final Namespace namespace) throws Exception {
|
||||
final Path path = Path.of(namespace.getString("file"));
|
||||
|
||||
if (DynamicConfigurationManager.parseConfiguration(Files.readString(path)).isPresent()) {
|
||||
System.out.println("Dynamic configuration file at " + path + " is valid");
|
||||
} else {
|
||||
System.err.println("Dynamic configuration file at " + path + " is not valid");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,11 +9,6 @@ import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import io.dropwizard.Application;
|
||||
import io.dropwizard.cli.EnvironmentCommand;
|
||||
@@ -59,6 +54,9 @@ import org.whispersystems.textsecuregcm.storage.ReportMessageManager;
|
||||
import org.whispersystems.textsecuregcm.storage.ReservedUsernames;
|
||||
import org.whispersystems.textsecuregcm.storage.Usernames;
|
||||
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
||||
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
|
||||
public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfiguration> {
|
||||
|
||||
@@ -100,64 +98,20 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
|
||||
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi, configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
|
||||
ClientResources redisClusterClientResources = ClientResources.builder().build();
|
||||
|
||||
AmazonDynamoDBClientBuilder clientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(configuration.getMessageDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) configuration.getMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
|
||||
AmazonDynamoDBClientBuilder keysDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(configuration.getKeysDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getKeysDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) configuration.getKeysDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
|
||||
AmazonDynamoDBClientBuilder accountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(configuration.getAccountsDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) configuration.getAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
|
||||
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
|
||||
new LinkedBlockingDeque<>());
|
||||
|
||||
AmazonDynamoDBAsyncClientBuilder accountsDynamoDbAsyncClientBuilder = AmazonDynamoDBAsyncClientBuilder
|
||||
.standard()
|
||||
.withRegion(accountsDynamoDbClientBuilder.getRegion())
|
||||
.withClientConfiguration(accountsDynamoDbClientBuilder.getClientConfiguration())
|
||||
.withCredentials(accountsDynamoDbClientBuilder.getCredentials())
|
||||
.withExecutorFactory(() -> accountsDynamoDbMigrationThreadPool);
|
||||
|
||||
AmazonDynamoDBClientBuilder migrationDeletedAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
|
||||
AmazonDynamoDBClientBuilder migrationRetryAccountsDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(configuration.getMigrationRetryAccountsDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getMigrationRetryAccountsDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) configuration.getMigrationRetryAccountsDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
|
||||
AmazonDynamoDBClientBuilder reportMessageDynamoDbClientBuilder = AmazonDynamoDBClientBuilder
|
||||
.standard()
|
||||
.withRegion(configuration.getReportMessageDynamoDbConfiguration().getRegion())
|
||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getReportMessageDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
||||
.withRequestTimeout((int) configuration.getReportMessageDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance());
|
||||
|
||||
DynamoDB messageDynamoDb = new DynamoDB(clientBuilder.build());
|
||||
DynamoDB preKeysDynamoDb = new DynamoDB(keysDynamoDbClientBuilder.build());
|
||||
DynamoDB reportMessagesDynamoDb = new DynamoDB(reportMessageDynamoDbClientBuilder.build());
|
||||
|
||||
AmazonDynamoDB accountsDynamoDbClient = accountsDynamoDbClientBuilder.build();
|
||||
AmazonDynamoDBAsync accountsDynamoDbAsyncClient = accountsDynamoDbAsyncClientBuilder.build();
|
||||
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig.client(configuration.getReportMessageDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
DynamoDbClient messageDynamoDb = DynamoDbFromConfig.client(configuration.getMessageDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
DynamoDbClient preKeysDynamoDb = DynamoDbFromConfig.client(configuration.getKeysDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(configuration.getAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(configuration.getAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
|
||||
accountsDynamoDbMigrationThreadPool);
|
||||
|
||||
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", configuration.getCacheClusterConfiguration(), redisClusterClientResources);
|
||||
|
||||
@@ -173,14 +127,16 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
|
||||
|
||||
ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(dynamicConfigurationManager);
|
||||
|
||||
DynamoDB migrationDeletedAccountsDynamoDb = new DynamoDB(migrationDeletedAccountsDynamoDbClientBuilder.build());
|
||||
DynamoDB migrationRetryAccountsDynamoDb = new DynamoDB(migrationRetryAccountsDynamoDbClientBuilder.build());
|
||||
DynamoDbClient migrationDeletedAccountsDynamoDb = DynamoDbFromConfig.client(configuration.getMigrationDeletedAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(configuration.getMigrationRetryAccountsDynamoDbConfiguration(),
|
||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||
|
||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(migrationDeletedAccountsDynamoDb, configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
|
||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, configuration.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
|
||||
|
||||
Accounts accounts = new Accounts(accountDatabase);
|
||||
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, new DynamoDB(accountsDynamoDbClient), configuration.getAccountsDynamoDbConfiguration().getTableName(), configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
|
||||
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, configuration.getAccountsDynamoDbConfiguration().getTableName(), configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
|
||||
Usernames usernames = new Usernames(accountDatabase);
|
||||
Profiles profiles = new Profiles(accountDatabase);
|
||||
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
/*
|
||||
* Copyright 2013-2020 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.workers;
|
||||
|
||||
import io.dropwizard.cli.ConfiguredCommand;
|
||||
import io.dropwizard.setup.Bootstrap;
|
||||
import io.lettuce.core.resource.ClientResources;
|
||||
import net.sourceforge.argparse4j.inf.Namespace;
|
||||
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
|
||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class GetRedisCommandStatsCommand extends ConfiguredCommand<WhisperServerConfiguration> {
|
||||
|
||||
public GetRedisCommandStatsCommand() {
|
||||
super("rediscommandstats", "Dump Redis command stats");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void run(final Bootstrap<WhisperServerConfiguration> bootstrap, final Namespace namespace, final WhisperServerConfiguration config) {
|
||||
final ClientResources redisClusterClientResources = ClientResources.builder().build();
|
||||
|
||||
final FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", config.getCacheClusterConfiguration(), redisClusterClientResources);
|
||||
final FaultTolerantRedisCluster messagesCacheCluster = new FaultTolerantRedisCluster("messages_cluster", config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
|
||||
final FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", config.getMetricsClusterConfiguration(), redisClusterClientResources);
|
||||
|
||||
for (final FaultTolerantRedisCluster cluster : List.of(cacheCluster, messagesCacheCluster, metricsCluster)) {
|
||||
cluster.useCluster(connection -> connection.sync()
|
||||
.upstream()
|
||||
.commands()
|
||||
.info("commandstats")
|
||||
.asMap()
|
||||
.forEach((node, commandStats) -> {
|
||||
System.out.format("# %s - %s\n\n", cluster.getName(), node.getUri());
|
||||
System.out.println(commandStats);
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright 2013-2020 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.workers;
|
||||
|
||||
import io.dropwizard.cli.ConfiguredCommand;
|
||||
import io.dropwizard.setup.Bootstrap;
|
||||
import io.lettuce.core.resource.ClientResources;
|
||||
import net.sourceforge.argparse4j.inf.Namespace;
|
||||
import net.sourceforge.argparse4j.inf.Subparser;
|
||||
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
|
||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
||||
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class GetRedisSlowlogCommand extends ConfiguredCommand<WhisperServerConfiguration> {
|
||||
|
||||
public GetRedisSlowlogCommand() {
|
||||
super("redisslowlog", "Dump a JSON blob describing slow Redis operations");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(final Subparser subparser) {
|
||||
super.configure(subparser);
|
||||
|
||||
subparser.addArgument("-n", "--entries")
|
||||
.dest("entries")
|
||||
.type(Integer.class)
|
||||
.required(false)
|
||||
.setDefault(128)
|
||||
.help("The maximum number of SLOWLOG entries to retrieve per cluster node");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void run(final Bootstrap<WhisperServerConfiguration> bootstrap, final Namespace namespace, final WhisperServerConfiguration config) throws Exception {
|
||||
final int entries = namespace.getInt("entries");
|
||||
|
||||
final ClientResources redisClusterClientResources = ClientResources.builder().build();
|
||||
|
||||
final FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", config.getCacheClusterConfiguration(), redisClusterClientResources);
|
||||
final FaultTolerantRedisCluster messagesCacheCluster = new FaultTolerantRedisCluster("messages_cluster", config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
|
||||
final FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", config.getMetricsClusterConfiguration(), redisClusterClientResources);
|
||||
|
||||
final Map<String, List<Object>> slowlogsByUri = new HashMap<>();
|
||||
|
||||
for (final FaultTolerantRedisCluster cluster : List.of(cacheCluster, messagesCacheCluster, metricsCluster)) {
|
||||
cluster.useCluster(connection -> connection.sync()
|
||||
.upstream()
|
||||
.commands()
|
||||
.slowlogGet(entries)
|
||||
.asMap()
|
||||
.forEach((node, slowlogs) -> slowlogsByUri.put(node.getUri().toString(), slowlogs)));
|
||||
}
|
||||
|
||||
SystemMapper.getMapper().writeValue(System.out, slowlogsByUri);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright 2013-2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.workers;
|
||||
|
||||
import io.dropwizard.cli.Command;
|
||||
import io.dropwizard.setup.Bootstrap;
|
||||
import net.sourceforge.argparse4j.inf.Namespace;
|
||||
import net.sourceforge.argparse4j.inf.Subparser;
|
||||
import org.whispersystems.textsecuregcm.WhisperServerVersion;
|
||||
|
||||
public class ServerVersionCommand extends Command {
|
||||
|
||||
public ServerVersionCommand() {
|
||||
super("version", "Print the version of the service");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(final Subparser subparser) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(final Bootstrap<?> bootstrap, final Namespace namespace) throws Exception {
|
||||
System.out.println(WhisperServerVersion.getServerVersion());
|
||||
}
|
||||
}
|
||||
@@ -25,8 +25,10 @@ message Envelope {
|
||||
CIPHERTEXT = 1;
|
||||
KEY_EXCHANGE = 2;
|
||||
PREKEY_BUNDLE = 3;
|
||||
RECEIPT = 5;
|
||||
SERVER_DELIVERY_RECEIPT = 5;
|
||||
UNIDENTIFIED_SENDER = 6;
|
||||
reserved 7;
|
||||
PLAINTEXT_CONTENT = 8; // for decryption error receipts
|
||||
}
|
||||
|
||||
optional Type type = 1;
|
||||
@@ -31,8 +31,8 @@ class DynamicConfigurationTest {
|
||||
void testParseExperimentConfig() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertFalse(emptyConfig.getExperimentEnrollmentConfiguration("test").isPresent());
|
||||
}
|
||||
@@ -51,8 +51,8 @@ class DynamicConfigurationTest {
|
||||
" enrolledUuids:\n" +
|
||||
" - 71618739-114c-4b1f-bb0d-6478a44eb600";
|
||||
|
||||
final DynamicConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(experimentConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration config =
|
||||
DynamicConfigurationManager.parseConfiguration(experimentConfigYaml).orElseThrow();
|
||||
|
||||
assertFalse(config.getExperimentEnrollmentConfiguration("unconfigured").isPresent());
|
||||
|
||||
@@ -79,8 +79,8 @@ class DynamicConfigurationTest {
|
||||
void testParsePreRegistrationExperiments() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertFalse(emptyConfig.getPreRegistrationEnrollmentConfiguration("test").isPresent());
|
||||
}
|
||||
@@ -105,8 +105,8 @@ class DynamicConfigurationTest {
|
||||
" excludedCountryCodes:\n" +
|
||||
" - 47";
|
||||
|
||||
final DynamicConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(experimentConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration config =
|
||||
DynamicConfigurationManager.parseConfiguration(experimentConfigYaml).orElseThrow();
|
||||
|
||||
assertFalse(config.getPreRegistrationEnrollmentConfiguration("unconfigured").isPresent());
|
||||
|
||||
@@ -152,14 +152,14 @@ class DynamicConfigurationTest {
|
||||
void testParseRemoteDeprecationConfig() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertNotNull(emptyConfig.getRemoteDeprecationConfiguration());
|
||||
}
|
||||
|
||||
{
|
||||
final String experimentConfigYaml =
|
||||
final String remoteDeprecationConfig =
|
||||
"remoteDeprecation:\n" +
|
||||
" minimumVersions:\n" +
|
||||
" IOS: 1.2.3\n" +
|
||||
@@ -172,8 +172,9 @@ class DynamicConfigurationTest {
|
||||
" DESKTOP:\n" +
|
||||
" - 1.4.0-beta.2";
|
||||
|
||||
final DynamicConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(experimentConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration config =
|
||||
DynamicConfigurationManager.parseConfiguration(remoteDeprecationConfig).orElseThrow();
|
||||
|
||||
final DynamicRemoteDeprecationConfiguration remoteDeprecationConfiguration = config
|
||||
.getRemoteDeprecationConfiguration();
|
||||
|
||||
@@ -191,8 +192,8 @@ class DynamicConfigurationTest {
|
||||
void testParseMessageRateConfiguration() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertFalse(emptyConfig.getMessageRateConfiguration().isEnforceUnsealedSenderRateLimit());
|
||||
}
|
||||
@@ -202,8 +203,8 @@ class DynamicConfigurationTest {
|
||||
"messageRate:\n" +
|
||||
" enforceUnsealedSenderRateLimit: true";
|
||||
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(messageRateConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(messageRateConfigYaml).orElseThrow();
|
||||
|
||||
assertTrue(emptyConfig.getMessageRateConfiguration().isEnforceUnsealedSenderRateLimit());
|
||||
}
|
||||
@@ -213,19 +214,19 @@ class DynamicConfigurationTest {
|
||||
void testParseFeatureFlags() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertTrue(emptyConfig.getActiveFeatureFlags().isEmpty());
|
||||
}
|
||||
|
||||
{
|
||||
final String emptyConfigYaml =
|
||||
final String featureFlagYaml =
|
||||
"featureFlags:\n"
|
||||
+ " - testFlag";
|
||||
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(featureFlagYaml).orElseThrow();
|
||||
|
||||
assertTrue(emptyConfig.getActiveFeatureFlags().contains("testFlag"));
|
||||
}
|
||||
@@ -235,22 +236,22 @@ class DynamicConfigurationTest {
|
||||
void testParseTwilioConfiguration() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertTrue(emptyConfig.getTwilioConfiguration().getNumbers().isEmpty());
|
||||
}
|
||||
|
||||
{
|
||||
final String emptyConfigYaml =
|
||||
final String twilioConfigYaml =
|
||||
"twilio:\n"
|
||||
+ " numbers:\n"
|
||||
+ " - 2135551212\n"
|
||||
+ " - 2135551313";
|
||||
|
||||
final DynamicTwilioConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class)
|
||||
.getTwilioConfiguration();
|
||||
final DynamicTwilioConfiguration config =
|
||||
DynamicConfigurationManager.parseConfiguration(twilioConfigYaml).orElseThrow()
|
||||
.getTwilioConfiguration();
|
||||
|
||||
assertEquals(List.of("2135551212", "2135551313"), config.getNumbers());
|
||||
}
|
||||
@@ -260,8 +261,8 @@ class DynamicConfigurationTest {
|
||||
void testParsePaymentsConfiguration() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertTrue(emptyConfig.getPaymentsConfiguration().getAllowedCountryCodes().isEmpty());
|
||||
}
|
||||
@@ -272,9 +273,9 @@ class DynamicConfigurationTest {
|
||||
+ " allowedCountryCodes:\n"
|
||||
+ " - 44";
|
||||
|
||||
final DynamicPaymentsConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(paymentsConfigYaml, DynamicConfiguration.class)
|
||||
.getPaymentsConfiguration();
|
||||
final DynamicPaymentsConfiguration config =
|
||||
DynamicConfigurationManager.parseConfiguration(paymentsConfigYaml).orElseThrow()
|
||||
.getPaymentsConfiguration();
|
||||
|
||||
assertEquals(Set.of("44"), config.getAllowedCountryCodes());
|
||||
}
|
||||
@@ -284,8 +285,8 @@ class DynamicConfigurationTest {
|
||||
void testParseSignupCaptchaConfiguration() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertTrue(emptyConfig.getSignupCaptchaConfiguration().getCountryCodes().isEmpty());
|
||||
}
|
||||
@@ -296,9 +297,9 @@ class DynamicConfigurationTest {
|
||||
+ " countryCodes:\n"
|
||||
+ " - 1";
|
||||
|
||||
final DynamicSignupCaptchaConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(signupCaptchaConfig, DynamicConfiguration.class)
|
||||
.getSignupCaptchaConfiguration();
|
||||
final DynamicSignupCaptchaConfiguration config =
|
||||
DynamicConfigurationManager.parseConfiguration(signupCaptchaConfig).orElseThrow()
|
||||
.getSignupCaptchaConfiguration();
|
||||
|
||||
assertEquals(Set.of("1"), config.getCountryCodes());
|
||||
}
|
||||
@@ -308,8 +309,8 @@ class DynamicConfigurationTest {
|
||||
void testParseAccountsDynamoDbMigrationConfiguration() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isBackgroundMigrationEnabled());
|
||||
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled());
|
||||
@@ -326,9 +327,9 @@ class DynamicConfigurationTest {
|
||||
+ " readEnabled: true\n"
|
||||
+ " writeEnabled: true";
|
||||
|
||||
final DynamicAccountsDynamoDbMigrationConfiguration config = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(accountsDynamoDbMigrationConfig, DynamicConfiguration.class)
|
||||
.getAccountsDynamoDbMigrationConfiguration();
|
||||
final DynamicAccountsDynamoDbMigrationConfiguration config =
|
||||
DynamicConfigurationManager.parseConfiguration(accountsDynamoDbMigrationConfig).orElseThrow()
|
||||
.getAccountsDynamoDbMigrationConfiguration();
|
||||
|
||||
assertTrue(config.isBackgroundMigrationEnabled());
|
||||
assertEquals(100, config.getBackgroundMigrationExecutorThreads());
|
||||
@@ -342,8 +343,8 @@ class DynamicConfigurationTest {
|
||||
void testParseLimits() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER.readValue(
|
||||
emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertThat(emptyConfig.getLimits().getUnsealedSenderNumber().getMaxCardinality()).isEqualTo(100);
|
||||
assertThat(emptyConfig.getLimits().getUnsealedSenderNumber().getTtl()).isEqualTo(Duration.ofDays(1));
|
||||
@@ -355,9 +356,10 @@ class DynamicConfigurationTest {
|
||||
+ " unsealedSenderNumber:\n"
|
||||
+ " maxCardinality: 99\n"
|
||||
+ " ttl: PT23H";
|
||||
final CardinalityRateLimitConfiguration unsealedSenderNumber = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(limitsConfig, DynamicConfiguration.class)
|
||||
.getLimits().getUnsealedSenderNumber();
|
||||
|
||||
final CardinalityRateLimitConfiguration unsealedSenderNumber =
|
||||
DynamicConfigurationManager.parseConfiguration(limitsConfig).orElseThrow()
|
||||
.getLimits().getUnsealedSenderNumber();
|
||||
|
||||
assertThat(unsealedSenderNumber.getMaxCardinality()).isEqualTo(99);
|
||||
assertThat(unsealedSenderNumber.getTtl()).isEqualTo(Duration.ofHours(23));
|
||||
@@ -368,8 +370,8 @@ class DynamicConfigurationTest {
|
||||
void testParseRateLimitReset() throws JsonProcessingException {
|
||||
{
|
||||
final String emptyConfigYaml = "test: true";
|
||||
final DynamicConfiguration emptyConfig = DynamicConfigurationManager.OBJECT_MAPPER.readValue(
|
||||
emptyConfigYaml, DynamicConfiguration.class);
|
||||
final DynamicConfiguration emptyConfig =
|
||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||
|
||||
assertThat(emptyConfig.getRateLimitChallengeConfiguration().getClientSupportedVersions()).isEmpty();
|
||||
assertThat(emptyConfig.getRateLimitChallengeConfiguration().isPreKeyLimitEnforced()).isFalse();
|
||||
@@ -384,9 +386,11 @@ class DynamicConfigurationTest {
|
||||
+ " IOS: 5.1.0\n"
|
||||
+ " ANDROID: 5.2.0\n"
|
||||
+ " DESKTOP: 5.0.0";
|
||||
DynamicRateLimitChallengeConfiguration rateLimitChallengeConfiguration = DynamicConfigurationManager.OBJECT_MAPPER
|
||||
.readValue(rateLimitChallengeConfig, DynamicConfiguration.class)
|
||||
.getRateLimitChallengeConfiguration();
|
||||
|
||||
DynamicRateLimitChallengeConfiguration rateLimitChallengeConfiguration =
|
||||
DynamicConfigurationManager.parseConfiguration(rateLimitChallengeConfig).orElseThrow()
|
||||
.getRateLimitChallengeConfiguration();
|
||||
|
||||
final Map<ClientPlatform, Semver> clientSupportedVersions = rateLimitChallengeConfiguration.getClientSupportedVersions();
|
||||
|
||||
assertThat(clientSupportedVersions.get(ClientPlatform.IOS)).isEqualTo(new Semver("5.1.0"));
|
||||
|
||||
@@ -11,26 +11,12 @@ import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.Page;
|
||||
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
|
||||
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
|
||||
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
|
||||
import com.amazonaws.services.dynamodbv2.model.KeyType;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
@@ -46,7 +32,22 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
||||
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
|
||||
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeyType;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
|
||||
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
|
||||
|
||||
class AccountsDynamoDbTest {
|
||||
|
||||
@@ -59,49 +60,75 @@ class AccountsDynamoDbTest {
|
||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
||||
.tableName(ACCOUNTS_TABLE_NAME)
|
||||
.hashKey(AccountsDynamoDb.KEY_ACCOUNT_UUID)
|
||||
.attributeDefinition(new AttributeDefinition(AccountsDynamoDb.KEY_ACCOUNT_UUID, ScalarAttributeType.B))
|
||||
.attributeDefinition(AttributeDefinition.builder()
|
||||
.attributeName(AccountsDynamoDb.KEY_ACCOUNT_UUID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
private AccountsDynamoDb accountsDynamoDb;
|
||||
|
||||
private Table migrationDeletedAccountsTable;
|
||||
private Table migrationRetryAccountsTable;
|
||||
|
||||
@BeforeEach
|
||||
void setupAccountsDao() {
|
||||
CreateTableRequest createNumbersTableRequest = CreateTableRequest.builder()
|
||||
.tableName(NUMBERS_TABLE_NAME)
|
||||
.keySchema(KeySchemaElement.builder()
|
||||
.attributeName(AccountsDynamoDb.ATTR_ACCOUNT_E164)
|
||||
.keyType(KeyType.HASH)
|
||||
.build())
|
||||
.attributeDefinitions(AttributeDefinition.builder()
|
||||
.attributeName(AccountsDynamoDb.ATTR_ACCOUNT_E164)
|
||||
.attributeType(ScalarAttributeType.S)
|
||||
.build())
|
||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
||||
.build();
|
||||
|
||||
CreateTableRequest createNumbersTableRequest = new CreateTableRequest()
|
||||
.withTableName(NUMBERS_TABLE_NAME)
|
||||
.withKeySchema(new KeySchemaElement(AccountsDynamoDb.ATTR_ACCOUNT_E164, KeyType.HASH))
|
||||
.withAttributeDefinitions(new AttributeDefinition(AccountsDynamoDb.ATTR_ACCOUNT_E164, ScalarAttributeType.S))
|
||||
.withProvisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT);
|
||||
dynamoDbExtension.getDynamoDbClient().createTable(createNumbersTableRequest);
|
||||
|
||||
final Table numbersTable = dynamoDbExtension.getDynamoDB().createTable(createNumbersTableRequest);
|
||||
final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder()
|
||||
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
||||
.keySchema(KeySchemaElement.builder()
|
||||
.attributeName(MigrationDeletedAccounts.KEY_UUID)
|
||||
.keyType(KeyType.HASH)
|
||||
.build())
|
||||
.attributeDefinitions(AttributeDefinition.builder()
|
||||
.attributeName(MigrationDeletedAccounts.KEY_UUID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
||||
.build();
|
||||
|
||||
final CreateTableRequest createMigrationDeletedAccountsTableRequest = new CreateTableRequest()
|
||||
.withTableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
||||
.withKeySchema(new KeySchemaElement(MigrationDeletedAccounts.KEY_UUID, KeyType.HASH))
|
||||
.withAttributeDefinitions(new AttributeDefinition(MigrationDeletedAccounts.KEY_UUID, ScalarAttributeType.B))
|
||||
.withProvisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT);
|
||||
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest);
|
||||
|
||||
migrationDeletedAccountsTable = dynamoDbExtension.getDynamoDB().createTable(createMigrationDeletedAccountsTableRequest);
|
||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(
|
||||
dynamoDbExtension.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME);
|
||||
|
||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDB(),
|
||||
migrationDeletedAccountsTable.getTableName());
|
||||
final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder()
|
||||
.tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
|
||||
.keySchema(KeySchemaElement.builder()
|
||||
.attributeName(MigrationRetryAccounts.KEY_UUID)
|
||||
.keyType(KeyType.HASH)
|
||||
.build())
|
||||
.attributeDefinitions(AttributeDefinition.builder()
|
||||
.attributeName(MigrationRetryAccounts.KEY_UUID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
||||
.build();
|
||||
|
||||
final CreateTableRequest createMigrationRetryAccountsTableRequest = new CreateTableRequest()
|
||||
.withTableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
|
||||
.withKeySchema(new KeySchemaElement(MigrationRetryAccounts.KEY_UUID, KeyType.HASH))
|
||||
.withAttributeDefinitions(new AttributeDefinition(MigrationRetryAccounts.KEY_UUID, ScalarAttributeType.B))
|
||||
.withProvisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT);
|
||||
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest);
|
||||
|
||||
migrationRetryAccountsTable = dynamoDbExtension.getDynamoDB().createTable(createMigrationRetryAccountsTableRequest);
|
||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts((dynamoDbExtension.getDynamoDbClient()),
|
||||
MIGRATION_RETRY_ACCOUNTS_TABLE_NAME);
|
||||
|
||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts((dynamoDbExtension.getDynamoDB()),
|
||||
migrationRetryAccountsTable.getTableName());
|
||||
|
||||
this.accountsDynamoDb = new AccountsDynamoDb(dynamoDbExtension.getClient(), dynamoDbExtension.getAsyncClient(), new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), dynamoDbExtension.getDynamoDB(), dynamoDbExtension.getTableName(), numbersTable.getTableName(),
|
||||
migrationDeletedAccounts, migrationRetryAccounts);
|
||||
this.accountsDynamoDb = new AccountsDynamoDb(
|
||||
dynamoDbExtension.getDynamoDbClient(),
|
||||
dynamoDbExtension.getDynamoDbAsyncClient(),
|
||||
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
|
||||
dynamoDbExtension.getTableName(),
|
||||
NUMBERS_TABLE_NAME,
|
||||
migrationDeletedAccounts,
|
||||
migrationRetryAccounts);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -262,8 +289,12 @@ class AccountsDynamoDbTest {
|
||||
|
||||
verifyRecentlyDeletedAccountsTableItemCount(1);
|
||||
|
||||
assertThat(migrationDeletedAccountsTable
|
||||
.getItem(MigrationDeletedAccounts.primaryKey(deletedAccount.getUuid()))).isNotNull();
|
||||
Map<String, AttributeValue> primaryKey = MigrationDeletedAccounts.primaryKey(deletedAccount.getUuid());
|
||||
assertThat(dynamoDbExtension.getDynamoDbClient().getItem(GetItemRequest.builder()
|
||||
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
||||
.key(Map.of(MigrationDeletedAccounts.KEY_UUID, primaryKey.get(MigrationDeletedAccounts.KEY_UUID)))
|
||||
.build()))
|
||||
.isNotNull();
|
||||
|
||||
accountsDynamoDb.deleteRecentlyDeletedUuids();
|
||||
|
||||
@@ -273,8 +304,10 @@ class AccountsDynamoDbTest {
|
||||
private void verifyRecentlyDeletedAccountsTableItemCount(int expectedItemCount) {
|
||||
int totalItems = 0;
|
||||
|
||||
for (Page<Item, ScanOutcome> page : migrationDeletedAccountsTable.scan(new ScanSpec()).pages()) {
|
||||
for (Item ignored : page) {
|
||||
for (ScanResponse page : dynamoDbExtension.getDynamoDbClient().scanPaginator(ScanRequest.builder()
|
||||
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
||||
.build())) {
|
||||
for (Map<String, AttributeValue> item : page.items()) {
|
||||
totalItems++;
|
||||
}
|
||||
}
|
||||
@@ -306,16 +339,15 @@ class AccountsDynamoDbTest {
|
||||
configuration.setRingBufferSizeInClosedState(2);
|
||||
configuration.setFailureRateThreshold(50);
|
||||
|
||||
final AmazonDynamoDB client = mock(AmazonDynamoDB.class);
|
||||
final DynamoDB dynamoDB = new DynamoDB(client);
|
||||
final DynamoDbClient client = mock(DynamoDbClient.class);
|
||||
|
||||
when(client.transactWriteItems(any()))
|
||||
when(client.transactWriteItems(any(TransactWriteItemsRequest.class)))
|
||||
.thenThrow(RuntimeException.class);
|
||||
|
||||
when(client.updateItem(any()))
|
||||
when(client.updateItem(any(UpdateItemRequest.class)))
|
||||
.thenThrow(RuntimeException.class);
|
||||
|
||||
AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(AmazonDynamoDBAsync.class), mock(ThreadPoolExecutor.class), dynamoDB, ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME, mock(
|
||||
AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(DynamoDbAsyncClient.class), mock(ThreadPoolExecutor.class), ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME, mock(
|
||||
MigrationDeletedAccounts.class), mock(MigrationRetryAccounts.class));
|
||||
Account account = generateAccount("+14151112222", UUID.randomUUID());
|
||||
|
||||
@@ -408,20 +440,22 @@ class AccountsDynamoDbTest {
|
||||
}
|
||||
|
||||
private void verifyStoredState(String number, UUID uuid, Account expecting) {
|
||||
final Table accounts = dynamoDbExtension.getDynamoDB().getTable(dynamoDbExtension.getTableName());
|
||||
final DynamoDbClient db = dynamoDbExtension.getDynamoDbClient();
|
||||
|
||||
Item item = accounts.getItem(new GetItemSpec()
|
||||
.withPrimaryKey(AccountsDynamoDb.KEY_ACCOUNT_UUID, UUIDUtil.toByteBuffer(uuid))
|
||||
.withConsistentRead(true));
|
||||
final GetItemResponse get = db.getItem(GetItemRequest.builder()
|
||||
.tableName(dynamoDbExtension.getTableName())
|
||||
.key(Map.of(AccountsDynamoDb.KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
|
||||
.consistentRead(true)
|
||||
.build());
|
||||
|
||||
if (item != null) {
|
||||
String data = new String(item.getBinary(AccountsDynamoDb.ATTR_ACCOUNT_DATA), StandardCharsets.UTF_8);
|
||||
if (get.hasItem()) {
|
||||
String data = new String(get.item().get(AccountsDynamoDb.ATTR_ACCOUNT_DATA).b().asByteArray(), StandardCharsets.UTF_8);
|
||||
assertThat(data).isNotEmpty();
|
||||
|
||||
assertThat(item.getNumber(AccountsDynamoDb.ATTR_MIGRATION_VERSION).intValue())
|
||||
assertThat(AttributeValues.getInt(get.item(), AccountsDynamoDb.ATTR_MIGRATION_VERSION, -1))
|
||||
.isEqualTo(expecting.getDynamoDbMigrationVersion());
|
||||
|
||||
Account result = AccountsDynamoDb.fromItem(item);
|
||||
Account result = AccountsDynamoDb.fromItem(get.item());
|
||||
verifyStoredState(number, uuid, result, expecting);
|
||||
} else {
|
||||
throw new AssertionError("No data");
|
||||
|
||||
@@ -1,33 +1,35 @@
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import com.almworks.sqlite4java.SQLite;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
|
||||
import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
|
||||
import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndex;
|
||||
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
|
||||
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
|
||||
import java.net.ServerSocket;
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.extension.AfterEachCallback;
|
||||
import org.junit.jupiter.api.extension.BeforeEachCallback;
|
||||
import org.junit.jupiter.api.extension.ExtensionContext;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeyType;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
|
||||
|
||||
public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback {
|
||||
|
||||
static final String DEFAULT_TABLE_NAME = "test_table";
|
||||
|
||||
static final ProvisionedThroughput DEFAULT_PROVISIONED_THROUGHPUT = new ProvisionedThroughput(20L, 20L);
|
||||
static final ProvisionedThroughput DEFAULT_PROVISIONED_THROUGHPUT = ProvisionedThroughput.builder()
|
||||
.readCapacityUnits(20L)
|
||||
.writeCapacityUnits(20L)
|
||||
.build();
|
||||
|
||||
private DynamoDBProxyServer server;
|
||||
private int port;
|
||||
@@ -42,9 +44,8 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
|
||||
private final long readCapacityUnits;
|
||||
private final long writeCapacityUnits;
|
||||
|
||||
private AmazonDynamoDB client;
|
||||
private AmazonDynamoDBAsync asyncClient;
|
||||
private DynamoDB dynamoDB;
|
||||
private DynamoDbClient dynamoDB2;
|
||||
private DynamoDbAsyncClient dynamoAsyncDB2;
|
||||
|
||||
private DynamoDbExtension(String tableName, String hashKey, String rangeKey, List<AttributeDefinition> attributeDefinitions, List<GlobalSecondaryIndex> globalSecondaryIndexes, long readCapacityUnits,
|
||||
long writeCapacityUnits) {
|
||||
@@ -87,26 +88,33 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
|
||||
KeySchemaElement[] keySchemaElements;
|
||||
if (rangeKeyName == null) {
|
||||
keySchemaElements = new KeySchemaElement[] {
|
||||
new KeySchemaElement(hashKeyName, "HASH"),
|
||||
KeySchemaElement.builder().attributeName(hashKeyName).keyType(KeyType.HASH).build(),
|
||||
};
|
||||
} else {
|
||||
keySchemaElements = new KeySchemaElement[] {
|
||||
new KeySchemaElement(hashKeyName, "HASH"),
|
||||
new KeySchemaElement(rangeKeyName, "RANGE")
|
||||
KeySchemaElement.builder().attributeName(hashKeyName).keyType(KeyType.HASH).build(),
|
||||
KeySchemaElement.builder().attributeName(rangeKeyName).keyType(KeyType.RANGE).build(),
|
||||
};
|
||||
}
|
||||
|
||||
final CreateTableRequest createTableRequest = new CreateTableRequest()
|
||||
.withTableName(tableName)
|
||||
.withKeySchema(keySchemaElements)
|
||||
.withAttributeDefinitions(attributeDefinitions.isEmpty() ? null : attributeDefinitions)
|
||||
.withGlobalSecondaryIndexes(globalSecondaryIndexes.isEmpty() ? null : globalSecondaryIndexes)
|
||||
.withProvisionedThroughput(new ProvisionedThroughput(readCapacityUnits, writeCapacityUnits));
|
||||
final CreateTableRequest createTableRequest = CreateTableRequest.builder()
|
||||
.tableName(tableName)
|
||||
.keySchema(keySchemaElements)
|
||||
.attributeDefinitions(attributeDefinitions.isEmpty() ? null : attributeDefinitions)
|
||||
.globalSecondaryIndexes(globalSecondaryIndexes.isEmpty() ? null : globalSecondaryIndexes)
|
||||
.provisionedThroughput(ProvisionedThroughput.builder()
|
||||
.readCapacityUnits(readCapacityUnits)
|
||||
.writeCapacityUnits(writeCapacityUnits)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
getDynamoDB().createTable(createTableRequest);
|
||||
getDynamoDbClient().createTable(createTableRequest);
|
||||
}
|
||||
|
||||
private void startServer() throws Exception {
|
||||
// Even though we're using AWS SDK v2, Dynamo's local implementation's canonical location
|
||||
// is within v1 (https://github.com/aws/aws-sdk-java-v2/issues/982). This does support
|
||||
// v2 clients, though.
|
||||
SQLite.setLibraryPath("target/lib"); // if you see a library failed to load error, you need to run mvn test-compile at least once first
|
||||
ServerSocket serverSocket = new ServerSocket(0);
|
||||
serverSocket.setReuseAddress(false);
|
||||
@@ -117,18 +125,18 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
|
||||
}
|
||||
|
||||
private void initializeClient() {
|
||||
AmazonDynamoDBClientBuilder clientBuilder = AmazonDynamoDBClientBuilder.standard()
|
||||
.withEndpointConfiguration(
|
||||
new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
|
||||
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")));
|
||||
client = clientBuilder.build();
|
||||
|
||||
asyncClient = AmazonDynamoDBAsyncClientBuilder.standard()
|
||||
.withEndpointConfiguration(clientBuilder.getEndpoint())
|
||||
.withCredentials(clientBuilder.getCredentials())
|
||||
.build();
|
||||
|
||||
dynamoDB = new DynamoDB(client);
|
||||
dynamoDB2 = DynamoDbClient.builder()
|
||||
.endpointOverride(URI.create("http://localhost:" + port))
|
||||
.region(Region.of("local-test-region"))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create("accessKey", "secretKey")))
|
||||
.build();
|
||||
dynamoAsyncDB2 = DynamoDbAsyncClient.builder()
|
||||
.endpointOverride(URI.create("http://localhost:" + port))
|
||||
.region(Region.of("local-test-region"))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create("accessKey", "secretKey")))
|
||||
.build();
|
||||
}
|
||||
|
||||
static class DynamoDbExtensionBuilder {
|
||||
@@ -140,8 +148,8 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
|
||||
private List<AttributeDefinition> attributeDefinitions = new ArrayList<>();
|
||||
private List<GlobalSecondaryIndex> globalSecondaryIndexes = new ArrayList<>();
|
||||
|
||||
private long readCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.getReadCapacityUnits();
|
||||
private long writeCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.getWriteCapacityUnits();
|
||||
private long readCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.readCapacityUnits();
|
||||
private long writeCapacityUnits = DEFAULT_PROVISIONED_THROUGHPUT.writeCapacityUnits();
|
||||
|
||||
private DynamoDbExtensionBuilder() {
|
||||
|
||||
@@ -178,16 +186,12 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
|
||||
}
|
||||
}
|
||||
|
||||
public AmazonDynamoDB getClient() {
|
||||
return client;
|
||||
public DynamoDbClient getDynamoDbClient() {
|
||||
return dynamoDB2;
|
||||
}
|
||||
|
||||
public AmazonDynamoDBAsync getAsyncClient() {
|
||||
return asyncClient;
|
||||
}
|
||||
|
||||
public DynamoDB getDynamoDB() {
|
||||
return dynamoDB;
|
||||
public DynamoDbAsyncClient getDynamoDbAsyncClient() {
|
||||
return dynamoAsyncDB2;
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
|
||||
@@ -5,13 +5,13 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
|
||||
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
|
||||
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
import org.whispersystems.textsecuregcm.tests.util.LocalDynamoDbRule;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeyType;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||
|
||||
public class KeysDynamoDbRule extends LocalDynamoDbRule {
|
||||
public static final String TABLE_NAME = "Signal_Keys_Test";
|
||||
@@ -19,18 +19,22 @@ public class KeysDynamoDbRule extends LocalDynamoDbRule {
|
||||
@Override
|
||||
protected void before() throws Throwable {
|
||||
super.before();
|
||||
|
||||
final DynamoDB dynamoDB = getDynamoDB();
|
||||
|
||||
final CreateTableRequest createTableRequest = new CreateTableRequest()
|
||||
.withTableName(TABLE_NAME)
|
||||
.withKeySchema(new KeySchemaElement(KeysDynamoDb.KEY_ACCOUNT_UUID, "HASH"),
|
||||
new KeySchemaElement(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID, "RANGE"))
|
||||
.withAttributeDefinitions(new AttributeDefinition(KeysDynamoDb.KEY_ACCOUNT_UUID, ScalarAttributeType.B),
|
||||
new AttributeDefinition(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID, ScalarAttributeType.B))
|
||||
.withProvisionedThroughput(new ProvisionedThroughput(20L, 20L));
|
||||
|
||||
dynamoDB.createTable(createTableRequest);
|
||||
getDynamoDbClient().createTable(CreateTableRequest.builder()
|
||||
.tableName(TABLE_NAME)
|
||||
.keySchema(
|
||||
KeySchemaElement.builder().attributeName(KeysDynamoDb.KEY_ACCOUNT_UUID).keyType(KeyType.HASH).build(),
|
||||
KeySchemaElement.builder().attributeName(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID).keyType(KeyType.RANGE)
|
||||
.build())
|
||||
.attributeDefinitions(AttributeDefinition.builder()
|
||||
.attributeName(KeysDynamoDb.KEY_ACCOUNT_UUID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build(),
|
||||
AttributeDefinition.builder()
|
||||
.attributeName(KeysDynamoDb.KEY_DEVICE_ID_KEY_ID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(20L).writeCapacityUnits(20L).build())
|
||||
.build());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -9,6 +9,7 @@ import org.junit.Before;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.whispersystems.textsecuregcm.entities.PreKey;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
@@ -17,6 +18,7 @@ import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -34,7 +36,7 @@ public class KeysDynamoDbTest {
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
keysDynamoDb = new KeysDynamoDb(dynamoDbRule.getDynamoDB(), KeysDynamoDbRule.TABLE_NAME);
|
||||
keysDynamoDb = new KeysDynamoDb(dynamoDbRule.getDynamoDbClient(), KeysDynamoDbRule.TABLE_NAME);
|
||||
|
||||
account = mock(Account.class);
|
||||
when(account.getNumber()).thenReturn(ACCOUNT_NUMBER);
|
||||
@@ -133,4 +135,10 @@ public class KeysDynamoDbTest {
|
||||
assertEquals(0, keysDynamoDb.getCount(account, DEVICE_ID));
|
||||
assertEquals(1, keysDynamoDb.getCount(account, DEVICE_ID + 1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSortKeyPrefix() {
|
||||
AttributeValue got = KeysDynamoDb.getSortKeyPrefix(123);
|
||||
assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 123}, got.b().asByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,12 +9,6 @@ import static org.junit.Assert.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||
import com.amazonaws.services.dynamodbv2.document.ItemCollection;
|
||||
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
|
||||
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
|
||||
import com.google.protobuf.ByteString;
|
||||
import io.lettuce.core.cluster.SlotHash;
|
||||
import java.nio.ByteBuffer;
|
||||
@@ -22,6 +16,7 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -38,6 +33,10 @@ import org.whispersystems.textsecuregcm.entities.MessageProtos;
|
||||
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
|
||||
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
|
||||
import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule;
|
||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
||||
|
||||
public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
|
||||
|
||||
@@ -62,7 +61,7 @@ public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
|
||||
connection.sync().upstream().commands().configSet("notify-keyspace-events", "K$glz");
|
||||
});
|
||||
|
||||
final MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
|
||||
final MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDbClient(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
|
||||
final AccountsManager accountsManager = mock(AccountsManager.class);
|
||||
final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
|
||||
|
||||
@@ -142,17 +141,16 @@ public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
|
||||
|
||||
final List<MessageProtos.Envelope> persistedMessages = new ArrayList<>(messageCount);
|
||||
|
||||
DynamoDB dynamoDB = messagesDynamoDbRule.getDynamoDB();
|
||||
Table table = dynamoDB.getTable(MessagesDynamoDbRule.TABLE_NAME);
|
||||
final ItemCollection<ScanOutcome> scan = table.scan(new ScanSpec());
|
||||
for (Item item : scan) {
|
||||
persistedMessages.add(MessageProtos.Envelope.newBuilder()
|
||||
.setServerGuid(convertBinaryToUuid(item.getBinary("U")).toString())
|
||||
.setType(MessageProtos.Envelope.Type.valueOf(item.getInt("T")))
|
||||
.setTimestamp(item.getLong("TS"))
|
||||
.setServerTimestamp(extractServerTimestamp(item.getBinary("S")))
|
||||
.setContent(ByteString.copyFrom(item.getBinary("C")))
|
||||
.build());
|
||||
DynamoDbClient dynamoDB = messagesDynamoDbRule.getDynamoDbClient();
|
||||
for (Map<String, AttributeValue> item : dynamoDB
|
||||
.scan(ScanRequest.builder().tableName(MessagesDynamoDbRule.TABLE_NAME).build()).items()) {
|
||||
persistedMessages.add(MessageProtos.Envelope.newBuilder()
|
||||
.setServerGuid(AttributeValues.getUUID(item, "U", null).toString())
|
||||
.setType(MessageProtos.Envelope.Type.valueOf(AttributeValues.getInt(item, "T", -1)))
|
||||
.setTimestamp(AttributeValues.getLong(item, "TS", -1))
|
||||
.setServerTimestamp(extractServerTimestamp(AttributeValues.getByteArray(item, "S", null)))
|
||||
.setContent(ByteString.copyFrom(AttributeValues.getByteArray(item, "C", null)))
|
||||
.build());
|
||||
}
|
||||
|
||||
assertEquals(expectedMessages, persistedMessages);
|
||||
|
||||
@@ -4,10 +4,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||
|
||||
class MigrationDeletedAccountsTest {
|
||||
|
||||
@@ -15,13 +15,16 @@ class MigrationDeletedAccountsTest {
|
||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
||||
.tableName("deleted_accounts_test")
|
||||
.hashKey(MigrationDeletedAccounts.KEY_UUID)
|
||||
.attributeDefinition(new AttributeDefinition(MigrationDeletedAccounts.KEY_UUID, ScalarAttributeType.B))
|
||||
.attributeDefinition(AttributeDefinition.builder()
|
||||
.attributeName(MigrationDeletedAccounts.KEY_UUID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
@Test
|
||||
void test() {
|
||||
|
||||
final MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDB(),
|
||||
final MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDbClient(),
|
||||
dynamoDbExtension.getTableName());
|
||||
|
||||
UUID firstUuid = UUID.randomUUID();
|
||||
|
||||
@@ -2,12 +2,12 @@ package org.whispersystems.textsecuregcm.storage;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||
|
||||
class MigrationRetryAccountsTest {
|
||||
|
||||
@@ -15,13 +15,16 @@ class MigrationRetryAccountsTest {
|
||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
||||
.tableName("account_migration_errors_test")
|
||||
.hashKey(MigrationRetryAccounts.KEY_UUID)
|
||||
.attributeDefinition(new AttributeDefinition(MigrationRetryAccounts.KEY_UUID, ScalarAttributeType.B))
|
||||
.attributeDefinition(AttributeDefinition.builder()
|
||||
.attributeName(MigrationRetryAccounts.KEY_UUID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
@Test
|
||||
void test() {
|
||||
|
||||
final MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(dynamoDbExtension.getDynamoDB(),
|
||||
final MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(dynamoDbExtension.getDynamoDbClient(),
|
||||
dynamoDbExtension.getTableName());
|
||||
|
||||
UUID firstUuid = UUID.randomUUID();
|
||||
|
||||
@@ -9,8 +9,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
@@ -20,6 +18,8 @@ import java.util.UUID;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||
|
||||
class PushChallengeDynamoDbTest {
|
||||
|
||||
@@ -34,12 +34,15 @@ class PushChallengeDynamoDbTest {
|
||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
||||
.tableName(TABLE_NAME)
|
||||
.hashKey(PushChallengeDynamoDb.KEY_ACCOUNT_UUID)
|
||||
.attributeDefinition(new AttributeDefinition(PushChallengeDynamoDb.KEY_ACCOUNT_UUID, ScalarAttributeType.B))
|
||||
.attributeDefinition(AttributeDefinition.builder()
|
||||
.attributeName(PushChallengeDynamoDb.KEY_ACCOUNT_UUID)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
this.pushChallengeDynamoDb = new PushChallengeDynamoDb(dynamoDbExtension.getDynamoDB(), TABLE_NAME, Clock.fixed(
|
||||
this.pushChallengeDynamoDb = new PushChallengeDynamoDb(dynamoDbExtension.getDynamoDbClient(), TABLE_NAME, Clock.fixed(
|
||||
Instant.ofEpochMilli(CURRENT_TIME_MILLIS), ZoneId.systemDefault()));
|
||||
}
|
||||
|
||||
|
||||
@@ -4,13 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
import java.util.UUID;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||
|
||||
class ReportMessageDynamoDbTest {
|
||||
|
||||
@@ -22,13 +22,16 @@ class ReportMessageDynamoDbTest {
|
||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
||||
.tableName(TABLE_NAME)
|
||||
.hashKey(ReportMessageDynamoDb.KEY_HASH)
|
||||
.attributeDefinition(new AttributeDefinition(ReportMessageDynamoDb.KEY_HASH, ScalarAttributeType.B))
|
||||
.attributeDefinition(AttributeDefinition.builder()
|
||||
.attributeName(ReportMessageDynamoDb.KEY_HASH)
|
||||
.attributeType(ScalarAttributeType.B)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
this.reportMessageDynamoDb = new ReportMessageDynamoDb(dynamoDbExtension.getDynamoDB(), TABLE_NAME);
|
||||
this.reportMessageDynamoDb = new ReportMessageDynamoDb(dynamoDbExtension.getDynamoDbClient(), TABLE_NAME);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -266,6 +266,8 @@ class AccountControllerTest {
|
||||
apnSender,
|
||||
usernamesManager,
|
||||
verifyExperimentEnrollmentManager);
|
||||
|
||||
clearInvocations(AuthHelper.DISABLED_DEVICE);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -322,6 +324,51 @@ class AccountControllerTest {
|
||||
assertThat(captor.getValue().getChallengeData().isPresent()).isTrue();
|
||||
assertThat(captor.getValue().getChallengeData().get().length()).isEqualTo(32);
|
||||
assertThat(captor.getValue().getMessage()).contains("\"challenge\" : \"" + captor.getValue().getChallengeData().get() + "\"");
|
||||
assertThat(captor.getValue().isVoip()).isTrue();
|
||||
|
||||
verifyNoMoreInteractions(gcmSender);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetApnPreauthExplicitVoip() throws Exception {
|
||||
Response response = resources.getJerseyTest()
|
||||
.target("/v1/accounts/apn/preauth/mytoken/+14152222222")
|
||||
.queryParam("voip", "true")
|
||||
.request()
|
||||
.get();
|
||||
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
|
||||
ArgumentCaptor<ApnMessage> captor = ArgumentCaptor.forClass(ApnMessage.class);
|
||||
|
||||
verify(apnSender, times(1)).sendMessage(captor.capture());
|
||||
assertThat(captor.getValue().getApnId()).isEqualTo("mytoken");
|
||||
assertThat(captor.getValue().getChallengeData().isPresent()).isTrue();
|
||||
assertThat(captor.getValue().getChallengeData().get().length()).isEqualTo(32);
|
||||
assertThat(captor.getValue().getMessage()).contains("\"challenge\" : \"" + captor.getValue().getChallengeData().get() + "\"");
|
||||
assertThat(captor.getValue().isVoip()).isTrue();
|
||||
|
||||
verifyNoMoreInteractions(gcmSender);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetApnPreauthExplicitNoVoip() throws Exception {
|
||||
Response response = resources.getJerseyTest()
|
||||
.target("/v1/accounts/apn/preauth/mytoken/+14152222222")
|
||||
.queryParam("voip", "false")
|
||||
.request()
|
||||
.get();
|
||||
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
|
||||
ArgumentCaptor<ApnMessage> captor = ArgumentCaptor.forClass(ApnMessage.class);
|
||||
|
||||
verify(apnSender, times(1)).sendMessage(captor.capture());
|
||||
assertThat(captor.getValue().getApnId()).isEqualTo("mytoken");
|
||||
assertThat(captor.getValue().getChallengeData().isPresent()).isTrue();
|
||||
assertThat(captor.getValue().getChallengeData().get().length()).isEqualTo(32);
|
||||
assertThat(captor.getValue().getMessage()).contains("\"challenge\" : \"" + captor.getValue().getChallengeData().get() + "\"");
|
||||
assertThat(captor.getValue().isVoip()).isFalse();
|
||||
|
||||
verifyNoMoreInteractions(gcmSender);
|
||||
}
|
||||
@@ -1342,6 +1389,23 @@ class AccountControllerTest {
|
||||
verify(directoryQueue, never()).refreshRegisteredUser(any(Account.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetApnIdNoVoip() throws Exception {
|
||||
Response response =
|
||||
resources.getJerseyTest()
|
||||
.target("/v1/accounts/apn/")
|
||||
.request()
|
||||
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.DISABLED_NUMBER, AuthHelper.DISABLED_PASSWORD))
|
||||
.put(Entity.json(new ApnRegistrationId("first", null)));
|
||||
|
||||
assertThat(response.getStatus()).isEqualTo(204);
|
||||
|
||||
verify(AuthHelper.DISABLED_DEVICE, times(1)).setApnId(eq("first"));
|
||||
verify(AuthHelper.DISABLED_DEVICE, times(1)).setVoipApnId(null);
|
||||
verify(accountsManager, times(1)).update(eq(AuthHelper.DISABLED_ACCOUNT));
|
||||
verify(directoryQueue, never()).refreshRegisteredUser(any(Account.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetApnIdByUuid() throws Exception {
|
||||
Response response =
|
||||
|
||||
@@ -42,6 +42,7 @@ import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
||||
import org.whispersystems.textsecuregcm.storage.Account;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||
import org.whispersystems.textsecuregcm.storage.Device;
|
||||
import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
|
||||
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
||||
import org.whispersystems.textsecuregcm.storage.PendingDevicesManager;
|
||||
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
|
||||
@@ -114,6 +115,7 @@ public class DeviceControllerTest {
|
||||
when(account.isEnabled()).thenReturn(false);
|
||||
when(account.isGroupsV2Supported()).thenReturn(true);
|
||||
when(account.isGv1MigrationSupported()).thenReturn(true);
|
||||
when(account.isSenderKeySupported()).thenReturn(true);
|
||||
|
||||
when(pendingDevicesManager.getCodeForNumber(AuthHelper.VALID_NUMBER)).thenReturn(Optional.of(new StoredVerificationCode("5678901", System.currentTimeMillis(), null)));
|
||||
when(pendingDevicesManager.getCodeForNumber(AuthHelper.VALID_NUMBER_TWO)).thenReturn(Optional.of(new StoredVerificationCode("1112223", System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(31), null)));
|
||||
@@ -221,8 +223,7 @@ public class DeviceControllerTest {
|
||||
@Test
|
||||
@Parameters(method = "argumentsForDeviceDowngradeCapabilitiesTest")
|
||||
public void deviceDowngradeCapabilitiesTest(final String userAgent, final boolean gv2, final boolean gv2_2, final boolean gv2_3, final int expectedStatus) throws Exception {
|
||||
Device.DeviceCapabilities deviceCapabilities = new Device.DeviceCapabilities(gv2, gv2_2, gv2_3, true, false, true,
|
||||
false);
|
||||
DeviceCapabilities deviceCapabilities = new DeviceCapabilities(gv2, gv2_2, gv2_3, true, false, true, true);
|
||||
AccountAttributes accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
|
||||
Response response = resources.getJerseyTest()
|
||||
.target("/v1/devices/5678901")
|
||||
@@ -262,8 +263,7 @@ public class DeviceControllerTest {
|
||||
|
||||
@Test
|
||||
public void deviceDowngradeGv1MigrationTest() {
|
||||
Device.DeviceCapabilities deviceCapabilities = new Device.DeviceCapabilities(true, true, true, true, false, false,
|
||||
false);
|
||||
DeviceCapabilities deviceCapabilities = new DeviceCapabilities(true, true, true, true, false, false, true);
|
||||
AccountAttributes accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
|
||||
Response response = resources.getJerseyTest()
|
||||
.target("/v1/devices/5678901")
|
||||
@@ -274,7 +274,7 @@ public class DeviceControllerTest {
|
||||
|
||||
assertThat(response.getStatus()).isEqualTo(409);
|
||||
|
||||
deviceCapabilities = new Device.DeviceCapabilities(true, true, true, true, false, true, false);
|
||||
deviceCapabilities = new DeviceCapabilities(true, true, true, true, false, true, true);
|
||||
accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
|
||||
response = resources.getJerseyTest()
|
||||
.target("/v1/devices/5678901")
|
||||
@@ -284,6 +284,31 @@ public class DeviceControllerTest {
|
||||
.put(Entity.entity(accountAttributes, MediaType.APPLICATION_JSON_TYPE));
|
||||
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deviceDowngradeSenderKeyTest() {
|
||||
DeviceCapabilities deviceCapabilities = new DeviceCapabilities(true, true, true, true, true, true, false);
|
||||
AccountAttributes accountAttributes =
|
||||
new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
|
||||
Response response = resources
|
||||
.getJerseyTest()
|
||||
.target("/v1/devices/5678901")
|
||||
.request()
|
||||
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
|
||||
.header("User-Agent", "Signal-Android/5.42.8675309 Android/30")
|
||||
.put(Entity.entity(accountAttributes, MediaType.APPLICATION_JSON_TYPE));
|
||||
assertThat(response.getStatus()).isEqualTo(409);
|
||||
|
||||
deviceCapabilities = new DeviceCapabilities(true, true, true, true, true, true, true);
|
||||
accountAttributes = new AccountAttributes(false, 1234, null, null, null, true, deviceCapabilities);
|
||||
response = resources
|
||||
.getJerseyTest()
|
||||
.target("/v1/devices/5678901")
|
||||
.request()
|
||||
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
|
||||
.header("User-Agent", "Signal-Android/5.42.8675309 Android/30")
|
||||
.put(Entity.entity(accountAttributes, MediaType.APPLICATION_JSON_TYPE));
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -478,7 +478,7 @@ class MessageControllerTest {
|
||||
|
||||
List<OutgoingMessageEntity> messages = new LinkedList<>() {{
|
||||
add(new OutgoingMessageEntity(1L, false, messageGuidOne, Envelope.Type.CIPHERTEXT_VALUE, null, timestampOne, "+14152222222", sourceUuid, 2, "hi there".getBytes(), null, 0));
|
||||
add(new OutgoingMessageEntity(2L, false, null, Envelope.Type.RECEIPT_VALUE, null, timestampTwo, "+14152222222", sourceUuid, 2, null, null, 0));
|
||||
add(new OutgoingMessageEntity(2L, false, null, Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE, null, timestampTwo, "+14152222222", sourceUuid, 2, null, null, 0));
|
||||
}};
|
||||
|
||||
OutgoingMessageEntityList messagesList = new OutgoingMessageEntityList(messages, false);
|
||||
@@ -515,7 +515,7 @@ class MessageControllerTest {
|
||||
|
||||
List<OutgoingMessageEntity> messages = new LinkedList<OutgoingMessageEntity>() {{
|
||||
add(new OutgoingMessageEntity(1L, false, UUID.randomUUID(), Envelope.Type.CIPHERTEXT_VALUE, null, timestampOne, "+14152222222", UUID.randomUUID(), 2, "hi there".getBytes(), null, 0));
|
||||
add(new OutgoingMessageEntity(2L, false, UUID.randomUUID(), Envelope.Type.RECEIPT_VALUE, null, timestampTwo, "+14152222222", UUID.randomUUID(), 2, null, null, 0));
|
||||
add(new OutgoingMessageEntity(2L, false, UUID.randomUUID(), Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE, null, timestampTwo, "+14152222222", UUID.randomUUID(), 2, null, null, 0));
|
||||
}};
|
||||
|
||||
OutgoingMessageEntityList messagesList = new OutgoingMessageEntityList(messages, false);
|
||||
@@ -546,7 +546,7 @@ class MessageControllerTest {
|
||||
|
||||
when(messagesManager.delete(AuthHelper.VALID_UUID, 1, "+14152222222", 31338))
|
||||
.thenReturn(Optional.of(new OutgoingMessageEntity(31337L, true, null,
|
||||
Envelope.Type.RECEIPT_VALUE,
|
||||
Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE,
|
||||
null, System.currentTimeMillis(),
|
||||
"+14152222222", sourceUuid, 1, null, null, 0)));
|
||||
|
||||
@@ -583,37 +583,6 @@ class MessageControllerTest {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource
|
||||
void testOnlineMessage(final String fixture, final boolean expectedOnline) throws Exception {
|
||||
|
||||
final Response response =
|
||||
resources.getJerseyTest()
|
||||
.target(String.format("/v1/messages/%s", SINGLE_DEVICE_RECIPIENT))
|
||||
.request()
|
||||
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
|
||||
.put(Entity.entity(mapper.readValue(jsonFixture(fixture), IncomingMessageList.class),
|
||||
MediaType.APPLICATION_JSON_TYPE));
|
||||
|
||||
assertThat("Good Response", response.getStatus(), is(equalTo(200)));
|
||||
|
||||
verify(messageSender, times(1)).sendMessage(any(Account.class), any(Device.class), any(Envelope.class), eq(expectedOnline));
|
||||
}
|
||||
|
||||
private static Stream<Arguments> testOnlineMessage() {
|
||||
return Stream.of(
|
||||
Arguments.of("fixtures/current_message_single_device.json", false), // default to `false` when absent
|
||||
Arguments.of("fixtures/online_message_true.json", true),
|
||||
Arguments.of("fixtures/online_message_false.json", false),
|
||||
// iOS versions prior to 5.5.0.7 send `online` on IncomingMessageList.message, rather on the top-level entity.
|
||||
// This causes some odd client behaviors, such as persisted typing indicators, so we have a temporary
|
||||
// server-side adaptation.
|
||||
Arguments.of("fixtures/online_message_true_nested_property.json", true),
|
||||
Arguments.of("fixtures/online_message_false_nested_property.json", false)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReportMessage() {
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ public class APNSenderTest {
|
||||
|
||||
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
|
||||
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
|
||||
assertThat(notification.getValue().getTopic()).isEqualTo("foo.voip");
|
||||
|
||||
@@ -112,7 +112,7 @@ public class APNSenderTest {
|
||||
|
||||
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
|
||||
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NSE_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
|
||||
assertThat(notification.getValue().getTopic()).isEqualTo("foo");
|
||||
|
||||
@@ -153,7 +153,7 @@ public class APNSenderTest {
|
||||
|
||||
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
|
||||
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
|
||||
|
||||
assertThat(apnResult.getStatus()).isEqualTo(ApnResult.Status.NO_SUCH_USER);
|
||||
@@ -256,7 +256,7 @@ public class APNSenderTest {
|
||||
|
||||
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
|
||||
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
|
||||
|
||||
assertThat(apnResult.getStatus()).isEqualTo(ApnResult.Status.NO_SUCH_USER);
|
||||
@@ -346,7 +346,7 @@ public class APNSenderTest {
|
||||
|
||||
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
|
||||
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
|
||||
|
||||
assertThat(apnResult.getStatus()).isEqualTo(ApnResult.Status.GENERIC_FAILURE);
|
||||
@@ -387,7 +387,7 @@ public class APNSenderTest {
|
||||
|
||||
assertThat(notification.getValue().getToken()).isEqualTo(DESTINATION_APN_ID);
|
||||
assertThat(notification.getValue().getExpiration()).isEqualTo(Instant.ofEpochMilli(ApnMessage.MAX_EXPIRATION));
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPayload()).isEqualTo(ApnMessage.APN_VOIP_NOTIFICATION_PAYLOAD);
|
||||
assertThat(notification.getValue().getPriority()).isEqualTo(DeliveryPriority.IMMEDIATE);
|
||||
|
||||
verifyNoMoreInteractions(apnsClient);
|
||||
|
||||
@@ -5,10 +5,11 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.tests.storage;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.whispersystems.textsecuregcm.storage.Account;
|
||||
import org.whispersystems.textsecuregcm.storage.Device;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
@@ -16,11 +17,11 @@ import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.whispersystems.textsecuregcm.storage.Account;
|
||||
import org.whispersystems.textsecuregcm.storage.Device;
|
||||
import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
|
||||
|
||||
public class AccountTest {
|
||||
|
||||
@@ -38,6 +39,10 @@ public class AccountTest {
|
||||
private final Device gv1MigrationIncapableDevice = mock(Device.class);
|
||||
private final Device gv1MigrationIncapableExpiredDevice = mock(Device.class);
|
||||
|
||||
private final Device senderKeyCapableDevice = mock(Device.class);
|
||||
private final Device senderKeyIncapableDevice = mock(Device.class);
|
||||
private final Device senderKeyIncapableExpiredDevice = mock(Device.class);
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
when(oldMasterDevice.getLastSeen()).thenReturn(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(366));
|
||||
@@ -72,17 +77,29 @@ public class AccountTest {
|
||||
when(gv2IncapableExpiredDevice.getLastSeen()).thenReturn(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(31));
|
||||
when(gv2IncapableExpiredDevice.isEnabled()).thenReturn(false);
|
||||
|
||||
when(gv1MigrationCapableDevice.getCapabilities()).thenReturn(new Device.DeviceCapabilities(true, true, true, true, true, true,
|
||||
false));
|
||||
when(gv1MigrationCapableDevice.getCapabilities()).thenReturn(
|
||||
new DeviceCapabilities(true, true, true, true, true, true, false));
|
||||
when(gv1MigrationCapableDevice.isEnabled()).thenReturn(true);
|
||||
|
||||
when(gv1MigrationIncapableDevice.getCapabilities()).thenReturn(new Device.DeviceCapabilities(true, true, true, true, true, false,
|
||||
false));
|
||||
when(gv1MigrationIncapableDevice.getCapabilities()).thenReturn(
|
||||
new DeviceCapabilities(true, true, true, true, true, false, false));
|
||||
when(gv1MigrationIncapableDevice.isEnabled()).thenReturn(true);
|
||||
|
||||
when(gv1MigrationIncapableExpiredDevice.getCapabilities()).thenReturn(new Device.DeviceCapabilities(true, true, true, true, true, false,
|
||||
false));
|
||||
when(gv1MigrationIncapableExpiredDevice.getCapabilities()).thenReturn(
|
||||
new DeviceCapabilities(true, true, true, true, true, false, false));
|
||||
when(gv1MigrationIncapableExpiredDevice.isEnabled()).thenReturn(false);
|
||||
|
||||
when(senderKeyCapableDevice.getCapabilities()).thenReturn(
|
||||
new DeviceCapabilities(true, true, true, true, true, true, true));
|
||||
when(senderKeyCapableDevice.isEnabled()).thenReturn(true);
|
||||
|
||||
when(senderKeyIncapableDevice.getCapabilities()).thenReturn(
|
||||
new DeviceCapabilities(true, true, true, true, true, true, false));
|
||||
when(senderKeyIncapableDevice.isEnabled()).thenReturn(true);
|
||||
|
||||
when(senderKeyIncapableExpiredDevice.getCapabilities()).thenReturn(
|
||||
new DeviceCapabilities(true, true, true, true, true, true, false));
|
||||
when(senderKeyIncapableExpiredDevice.isEnabled()).thenReturn(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -137,8 +154,8 @@ public class AccountTest {
|
||||
final Device nonTransferCapableMasterDevice = mock(Device.class);
|
||||
final Device transferCapableLinkedDevice = mock(Device.class);
|
||||
|
||||
final Device.DeviceCapabilities transferCapabilities = mock(Device.DeviceCapabilities.class);
|
||||
final Device.DeviceCapabilities nonTransferCapabilities = mock(Device.DeviceCapabilities.class);
|
||||
final DeviceCapabilities transferCapabilities = mock(DeviceCapabilities.class);
|
||||
final DeviceCapabilities nonTransferCapabilities = mock(DeviceCapabilities.class);
|
||||
|
||||
when(transferCapableMasterDevice.getId()).thenReturn(1L);
|
||||
when(transferCapableMasterDevice.isMaster()).thenReturn(true);
|
||||
@@ -205,4 +222,15 @@ public class AccountTest {
|
||||
assertFalse(new Account("+18005551234", UUID.randomUUID(), Set.of(gv1MigrationCapableDevice, gv1MigrationIncapableDevice), "1234".getBytes(StandardCharsets.UTF_8)).isGv1MigrationSupported());
|
||||
assertTrue(new Account("+18005551234", UUID.randomUUID(), Set.of(gv1MigrationCapableDevice, gv1MigrationIncapableExpiredDevice), "1234".getBytes(StandardCharsets.UTF_8)).isGv1MigrationSupported());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isSenderKeySupported() {
|
||||
assertThat(new Account("+18005551234", UUID.randomUUID(), Set.of(senderKeyCapableDevice),
|
||||
"1234".getBytes(StandardCharsets.UTF_8)).isSenderKeySupported()).isTrue();
|
||||
assertThat(new Account("+18005551234", UUID.randomUUID(), Set.of(senderKeyCapableDevice, senderKeyIncapableDevice),
|
||||
"1234".getBytes(StandardCharsets.UTF_8)).isSenderKeySupported()).isFalse();
|
||||
assertThat(new Account("+18005551234", UUID.randomUUID(),
|
||||
Set.of(senderKeyCapableDevice, senderKeyIncapableExpiredDevice),
|
||||
"1234".getBytes(StandardCharsets.UTF_8)).isSenderKeySupported()).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
|
||||
import io.lettuce.core.RedisException;
|
||||
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
|
||||
import java.util.HashSet;
|
||||
@@ -32,6 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
||||
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
||||
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
|
||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
||||
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
||||
@@ -41,12 +41,14 @@ import org.whispersystems.textsecuregcm.storage.Account;
|
||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||
import org.whispersystems.textsecuregcm.storage.Device;
|
||||
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
|
||||
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
|
||||
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
||||
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
||||
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
||||
import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
|
||||
|
||||
class AccountsManagerTest {
|
||||
|
||||
@@ -383,7 +385,7 @@ class AccountsManagerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCompareAccounts() {
|
||||
void testCompareAccounts() throws Exception {
|
||||
RedisAdvancedClusterCommands<String, String> commands = mock(RedisAdvancedClusterCommands.class);
|
||||
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
|
||||
Accounts accounts = mock(Accounts.class);
|
||||
@@ -409,9 +411,58 @@ class AccountsManagerTest {
|
||||
|
||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
{
|
||||
Device device1 = new Device();
|
||||
device1.setId(1L);
|
||||
|
||||
a1.addDevice(device1);
|
||||
|
||||
assertEquals(Optional.of("devices"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
Device device2 = new Device();
|
||||
device2.setId(1L);
|
||||
|
||||
a2.addDevice(device2);
|
||||
|
||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
device1.setLastSeen(1L);
|
||||
|
||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
device1.setName("name");
|
||||
|
||||
assertEquals(Optional.of("devices"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
device1.setName(null);
|
||||
|
||||
device1.setSignedPreKey(new SignedPreKey(1L, "123", "456"));
|
||||
device2.setSignedPreKey(new SignedPreKey(2L, "123", "456"));
|
||||
|
||||
assertEquals(Optional.of("masterDeviceSignedPreKey"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
device1.setSignedPreKey(null);
|
||||
device2.setSignedPreKey(null);
|
||||
|
||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
device1.setApnId("123");
|
||||
Thread.sleep(5);
|
||||
device2.setApnId("123");
|
||||
|
||||
assertEquals(Optional.of("masterDevicePushTimestamp"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
a1.removeDevice(1L);
|
||||
a2.removeDevice(1L);
|
||||
|
||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
}
|
||||
|
||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
a1.setDynamoDbMigrationVersion(1);
|
||||
|
||||
assertEquals(Optional.of("migrationVersion"), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(a1), Optional.of(a2)));
|
||||
|
||||
a2.setProfileName("name");
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ public class MessagesDynamoDbTest {
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
messagesDynamoDb = new MessagesDynamoDb(dynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
|
||||
messagesDynamoDb = new MessagesDynamoDb(dynamoDbRule.getDynamoDbClient(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -6,16 +6,16 @@
|
||||
package org.whispersystems.textsecuregcm.tests.util;
|
||||
|
||||
import com.almworks.sqlite4java.SQLite;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
|
||||
import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
|
||||
import org.junit.rules.ExternalResource;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||
|
||||
import java.net.ServerSocket;
|
||||
import java.net.URI;
|
||||
|
||||
public class LocalDynamoDbRule extends ExternalResource {
|
||||
private DynamoDBProxyServer server;
|
||||
@@ -43,11 +43,12 @@ public class LocalDynamoDbRule extends ExternalResource {
|
||||
super.after();
|
||||
}
|
||||
|
||||
public DynamoDB getDynamoDB() {
|
||||
AmazonDynamoDBClientBuilder clientBuilder =
|
||||
AmazonDynamoDBClientBuilder.standard()
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
|
||||
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")));
|
||||
return new DynamoDB(clientBuilder.build());
|
||||
public DynamoDbClient getDynamoDbClient() {
|
||||
return DynamoDbClient.builder()
|
||||
.endpointOverride(URI.create("http://localhost:" + port))
|
||||
.region(Region.of("local-test-region"))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create("accessKey", "secretKey")))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,15 +5,15 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.tests.util;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
|
||||
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
|
||||
import com.amazonaws.services.dynamodbv2.model.LocalSecondaryIndex;
|
||||
import com.amazonaws.services.dynamodbv2.model.Projection;
|
||||
import com.amazonaws.services.dynamodbv2.model.ProjectionType;
|
||||
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
|
||||
import software.amazon.awssdk.services.dynamodb.model.KeyType;
|
||||
import software.amazon.awssdk.services.dynamodb.model.LocalSecondaryIndex;
|
||||
import software.amazon.awssdk.services.dynamodb.model.Projection;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
|
||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||
|
||||
public class MessagesDynamoDbRule extends LocalDynamoDbRule {
|
||||
|
||||
@@ -22,20 +22,21 @@ public class MessagesDynamoDbRule extends LocalDynamoDbRule {
|
||||
@Override
|
||||
protected void before() throws Throwable {
|
||||
super.before();
|
||||
DynamoDB dynamoDB = getDynamoDB();
|
||||
CreateTableRequest createTableRequest = new CreateTableRequest()
|
||||
.withTableName(TABLE_NAME)
|
||||
.withKeySchema(new KeySchemaElement("H", "HASH"),
|
||||
new KeySchemaElement("S", "RANGE"))
|
||||
.withAttributeDefinitions(new AttributeDefinition("H", ScalarAttributeType.B),
|
||||
new AttributeDefinition("S", ScalarAttributeType.B),
|
||||
new AttributeDefinition("U", ScalarAttributeType.B))
|
||||
.withProvisionedThroughput(new ProvisionedThroughput(20L, 20L))
|
||||
.withLocalSecondaryIndexes(new LocalSecondaryIndex().withIndexName("Message_UUID_Index")
|
||||
.withKeySchema(new KeySchemaElement("H", "HASH"),
|
||||
new KeySchemaElement("U", "RANGE"))
|
||||
.withProjection(new Projection().withProjectionType(ProjectionType.KEYS_ONLY)));
|
||||
dynamoDB.createTable(createTableRequest);
|
||||
getDynamoDbClient().createTable(CreateTableRequest.builder()
|
||||
.tableName(TABLE_NAME)
|
||||
.keySchema(KeySchemaElement.builder().attributeName("H").keyType(KeyType.HASH).build(),
|
||||
KeySchemaElement.builder().attributeName("S").keyType(KeyType.RANGE).build())
|
||||
.attributeDefinitions(
|
||||
AttributeDefinition.builder().attributeName("H").attributeType(ScalarAttributeType.B).build(),
|
||||
AttributeDefinition.builder().attributeName("S").attributeType(ScalarAttributeType.B).build(),
|
||||
AttributeDefinition.builder().attributeName("U").attributeType(ScalarAttributeType.B).build())
|
||||
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(20L).writeCapacityUnits(20L).build())
|
||||
.localSecondaryIndexes(LocalSecondaryIndex.builder().indexName("Message_UUID_Index")
|
||||
.keySchema(KeySchemaElement.builder().attributeName("H").keyType(KeyType.HASH).build(),
|
||||
KeySchemaElement.builder().attributeName("U").keyType(KeyType.RANGE).build())
|
||||
.projection(Projection.builder().projectionType(ProjectionType.KEYS_ONLY).build())
|
||||
.build())
|
||||
.build());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -8,17 +8,11 @@ package org.whispersystems.textsecuregcm.util;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.whispersystems.textsecuregcm.configuration.MonitoredS3ObjectConfiguration;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.Inet4Address;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
@@ -34,7 +28,7 @@ class AsnManagerTest {
|
||||
assertEquals(Optional.empty(), asnManager.getAsn("10.0.0.1"));
|
||||
|
||||
try (final InputStream tableInputStream = getClass().getResourceAsStream("ip2asn-test.tsv")) {
|
||||
asnManager.handleAsnTableChanged(tableInputStream);
|
||||
asnManager.handleAsnTableChangedStream(tableInputStream);
|
||||
}
|
||||
|
||||
assertEquals(Optional.of(7922L), asnManager.getAsn("50.79.54.1"));
|
||||
|
||||
@@ -12,7 +12,6 @@ import java.io.InputStreamReader;
|
||||
import java.net.Inet4Address;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.Optional;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2013-2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
public class AttributeValuesTest {
|
||||
@Test
|
||||
void testUUIDRoundTrip() {
|
||||
UUID orig = UUID.randomUUID();
|
||||
AttributeValue av = AttributeValues.fromUUID(orig);
|
||||
UUID returned = AttributeValues.getUUID(Map.of("foo", av), "foo", null);
|
||||
assertEquals(orig, returned);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLongRoundTrip() {
|
||||
long orig = 12345;
|
||||
AttributeValue av = AttributeValues.fromLong(orig);
|
||||
long returned = AttributeValues.getLong(Map.of("foo", av), "foo", -1);
|
||||
assertEquals(orig, returned);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIntRoundTrip() {
|
||||
int orig = 12345;
|
||||
AttributeValue av = AttributeValues.fromInt(orig);
|
||||
int returned = AttributeValues.getInt(Map.of("foo", av), "foo", -1);
|
||||
assertEquals(orig, returned);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testByteBuffer() {
|
||||
byte[] bytes = {1, 2, 3};
|
||||
ByteBuffer bb = ByteBuffer.wrap(bytes);
|
||||
AttributeValue av = AttributeValues.fromByteBuffer(bb);
|
||||
byte[] returned = av.b().asByteArray();
|
||||
assertArrayEquals(bytes, returned);
|
||||
returned = AttributeValues.getByteArray(Map.of("foo", av), "foo", null);
|
||||
assertArrayEquals(bytes, returned);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testByteBuffer2() {
|
||||
final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[8]);
|
||||
byteBuffer.putLong(123);
|
||||
assertEquals(byteBuffer.remaining(), 0);
|
||||
AttributeValue av = AttributeValues.fromByteBuffer(byteBuffer.flip());
|
||||
assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 123}, AttributeValues.getByteArray(Map.of("foo", av), "foo", null));
|
||||
}
|
||||
}
|
||||
@@ -1,35 +1,39 @@
|
||||
package org.whispersystems.textsecuregcm.util;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.function.Consumer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import software.amazon.awssdk.core.ResponseInputStream;
|
||||
import software.amazon.awssdk.http.AbortableInputStream;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
|
||||
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
|
||||
|
||||
class S3ObjectMonitorTest {
|
||||
|
||||
@Test
|
||||
void refresh() {
|
||||
final AmazonS3 s3Client = mock(AmazonS3.class);
|
||||
final ObjectMetadata metadata = mock(ObjectMetadata.class);
|
||||
final S3Object s3Object = mock(S3Object.class);
|
||||
final S3Client s3Client = mock(S3Client.class);
|
||||
|
||||
final String bucket = "s3bucket";
|
||||
final String objectKey = "greatest-smooth-jazz-hits-of-all-time.zip";
|
||||
|
||||
//noinspection unchecked
|
||||
final Consumer<S3Object> listener = mock(Consumer.class);
|
||||
final Consumer<InputStream> listener = mock(Consumer.class);
|
||||
|
||||
final S3ObjectMonitor objectMonitor = new S3ObjectMonitor(
|
||||
s3Client,
|
||||
@@ -40,28 +44,27 @@ class S3ObjectMonitorTest {
|
||||
Duration.ofMinutes(1),
|
||||
listener);
|
||||
|
||||
when(metadata.getETag()).thenReturn(UUID.randomUUID().toString());
|
||||
when(s3Object.getObjectMetadata()).thenReturn(metadata);
|
||||
when(s3Client.getObjectMetadata(bucket, objectKey)).thenReturn(metadata);
|
||||
when(s3Client.getObject(bucket, objectKey)).thenReturn(s3Object);
|
||||
String uuid = UUID.randomUUID().toString();
|
||||
when(s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(
|
||||
HeadObjectResponse.builder().eTag(uuid).build());
|
||||
ResponseInputStream<GetObjectResponse> ris = responseInputStreamFromString("abc", uuid);
|
||||
when(s3Client.getObject(GetObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(ris);
|
||||
|
||||
objectMonitor.refresh();
|
||||
objectMonitor.refresh();
|
||||
|
||||
verify(listener).accept(s3Object);
|
||||
verify(listener).accept(ris);
|
||||
}
|
||||
|
||||
@Test
|
||||
void refreshAfterGet() throws IOException {
|
||||
final AmazonS3 s3Client = mock(AmazonS3.class);
|
||||
final ObjectMetadata metadata = mock(ObjectMetadata.class);
|
||||
final S3Object s3Object = mock(S3Object.class);
|
||||
final S3Client s3Client = mock(S3Client.class);
|
||||
|
||||
final String bucket = "s3bucket";
|
||||
final String objectKey = "greatest-smooth-jazz-hits-of-all-time.zip";
|
||||
|
||||
//noinspection unchecked
|
||||
final Consumer<S3Object> listener = mock(Consumer.class);
|
||||
final Consumer<InputStream> listener = mock(Consumer.class);
|
||||
|
||||
final S3ObjectMonitor objectMonitor = new S3ObjectMonitor(
|
||||
s3Client,
|
||||
@@ -72,29 +75,34 @@ class S3ObjectMonitorTest {
|
||||
Duration.ofMinutes(1),
|
||||
listener);
|
||||
|
||||
when(metadata.getETag()).thenReturn(UUID.randomUUID().toString());
|
||||
when(s3Object.getObjectMetadata()).thenReturn(metadata);
|
||||
when(s3Client.getObjectMetadata(bucket, objectKey)).thenReturn(metadata);
|
||||
when(s3Client.getObject(bucket, objectKey)).thenReturn(s3Object);
|
||||
String uuid = UUID.randomUUID().toString();
|
||||
when(s3Client.headObject(HeadObjectRequest.builder().key(objectKey).bucket(bucket).build()))
|
||||
.thenReturn(HeadObjectResponse.builder().eTag(uuid).build());
|
||||
ResponseInputStream<GetObjectResponse> responseInputStream = responseInputStreamFromString("abc", uuid);
|
||||
when(s3Client.getObject(GetObjectRequest.builder().key(objectKey).bucket(bucket).build())).thenReturn(responseInputStream);
|
||||
|
||||
objectMonitor.getObject();
|
||||
objectMonitor.refresh();
|
||||
|
||||
verify(listener, never()).accept(s3Object);
|
||||
verify(listener, never()).accept(responseInputStream);
|
||||
}
|
||||
|
||||
private ResponseInputStream<GetObjectResponse> responseInputStreamFromString(String s, String etag) {
|
||||
byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
|
||||
AbortableInputStream ais = AbortableInputStream.create(new ByteArrayInputStream(bytes));
|
||||
return new ResponseInputStream<>(GetObjectResponse.builder().contentLength((long) bytes.length).eTag(etag).build(), ais);
|
||||
}
|
||||
|
||||
@Test
|
||||
void refreshOversizedObject() {
|
||||
final AmazonS3 s3Client = mock(AmazonS3.class);
|
||||
final ObjectMetadata metadata = mock(ObjectMetadata.class);
|
||||
final S3Object s3Object = mock(S3Object.class);
|
||||
final S3Client s3Client = mock(S3Client.class);
|
||||
|
||||
final String bucket = "s3bucket";
|
||||
final String objectKey = "greatest-smooth-jazz-hits-of-all-time.zip";
|
||||
final long maxObjectSize = 16 * 1024 * 1024;
|
||||
|
||||
//noinspection unchecked
|
||||
final Consumer<S3Object> listener = mock(Consumer.class);
|
||||
final Consumer<InputStream> listener = mock(Consumer.class);
|
||||
|
||||
final S3ObjectMonitor objectMonitor = new S3ObjectMonitor(
|
||||
s3Client,
|
||||
@@ -105,11 +113,11 @@ class S3ObjectMonitorTest {
|
||||
Duration.ofMinutes(1),
|
||||
listener);
|
||||
|
||||
when(metadata.getETag()).thenReturn(UUID.randomUUID().toString());
|
||||
when(metadata.getContentLength()).thenReturn(maxObjectSize + 1);
|
||||
when(s3Object.getObjectMetadata()).thenReturn(metadata);
|
||||
when(s3Client.getObjectMetadata(bucket, objectKey)).thenReturn(metadata);
|
||||
when(s3Client.getObject(bucket, objectKey)).thenReturn(s3Object);
|
||||
String uuid = UUID.randomUUID().toString();
|
||||
when(s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(
|
||||
HeadObjectResponse.builder().eTag(uuid).contentLength(maxObjectSize+1).build());
|
||||
ResponseInputStream<GetObjectResponse> ris = responseInputStreamFromString("a".repeat((int) maxObjectSize+1), uuid);
|
||||
when(s3Client.getObject(GetObjectRequest.builder().bucket(bucket).key(objectKey).build())).thenReturn(ris);
|
||||
|
||||
objectMonitor.refresh();
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ public class TorExitNodeManagerTest extends AbstractRedisClusterTest {
|
||||
assertFalse(torExitNodeManager.isTorExitNode("10.0.0.1"));
|
||||
assertFalse(torExitNodeManager.isTorExitNode("10.0.0.2"));
|
||||
|
||||
torExitNodeManager.handleExitListChanged(
|
||||
torExitNodeManager.handleExitListChangedStream(
|
||||
new ByteArrayInputStream("10.0.0.1\n10.0.0.2".getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
assertTrue(torExitNodeManager.isTorExitNode("10.0.0.1"));
|
||||
|
||||
@@ -79,7 +79,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest
|
||||
|
||||
executorService = Executors.newSingleThreadExecutor();
|
||||
messagesCache = new MessagesCache(getRedisCluster(), getRedisCluster(), executorService);
|
||||
messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
|
||||
messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDbClient(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
|
||||
reportMessageManager = mock(ReportMessageManager.class);
|
||||
account = mock(Account.class);
|
||||
device = mock(Device.class);
|
||||
|
||||
@@ -823,7 +823,7 @@ public class WebSocketConnectionTest {
|
||||
}
|
||||
|
||||
private OutgoingMessageEntity createMessage(long id, boolean cached, String sender, UUID senderUuid, long timestamp, boolean receipt, String content) {
|
||||
return new OutgoingMessageEntity(id, cached, UUID.randomUUID(), receipt ? Envelope.Type.RECEIPT_VALUE : Envelope.Type.CIPHERTEXT_VALUE,
|
||||
return new OutgoingMessageEntity(id, cached, UUID.randomUUID(), receipt ? Envelope.Type.SERVER_DELIVERY_RECEIPT_VALUE : Envelope.Type.CIPHERTEXT_VALUE,
|
||||
null, timestamp, sender, senderUuid, 1, content.getBytes(), null, 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"online": false,
|
||||
"messages": [
|
||||
{
|
||||
"type": 1,
|
||||
"destinationDeviceId": 1,
|
||||
"body": "Zm9vYmFyego",
|
||||
"timestamp": 1234
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"type": 1,
|
||||
"destinationDeviceId": 1,
|
||||
"body": "Zm9vYmFyego",
|
||||
"timestamp": 1234,
|
||||
"online": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"online": true,
|
||||
"messages": [
|
||||
{
|
||||
"type": 1,
|
||||
"destinationDeviceId": 1,
|
||||
"body": "Zm9vYmFyego",
|
||||
"timestamp": 1234
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"type": 1,
|
||||
"destinationDeviceId": 1,
|
||||
"body": "Zm9vYmFyego",
|
||||
"timestamp": 1234,
|
||||
"online": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -5,12 +5,10 @@
|
||||
<parent>
|
||||
<artifactId>TextSecureServer</artifactId>
|
||||
<groupId>org.whispersystems.textsecure</groupId>
|
||||
<version>1.0</version>
|
||||
<version>JGITVER</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>websocket-resources</artifactId>
|
||||
<version>${TextSecureServer.version}</version>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -85,7 +83,6 @@
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>2.6.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
@@ -103,7 +100,6 @@
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-inline</artifactId>
|
||||
<version>${mockito.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
Reference in New Issue
Block a user