Compare commits

...

97 Commits

Author SHA1 Message Date
Chris Eager
b02b00818b Remove Subscriptions.PCI attribute 2023-01-04 11:31:46 -06:00
Chris Eager
010f88a2ad Remove Subscriptions.C attribute 2023-01-04 11:31:46 -06:00
Jon Chambers
60edf4835f Add a pni capability to UserCapabilities 2022-12-21 16:26:07 -05:00
Jon Chambers
a60450d931 Convert UserCapabilities to a record 2022-12-21 16:26:07 -05:00
erik-signal
d138fa45df Handle edge cases of Math.abs on integers. 2022-12-20 12:25:04 -05:00
Katherine Yen
2c2c497c12 Define reregistrationIdleDays DistributionSummary with custom expiry 2022-12-20 09:21:24 -08:00
Katherine Yen
cb5d3840d9 Add paymentActivation capability 2022-12-20 09:20:42 -08:00
Fedor Indutny
9aceaa7a4d Introduce ArtController 2022-12-19 11:58:16 -08:00
Katherine Yen
636c8ba384 Add metric for distribution of account idle time at reregistration 2022-12-16 13:50:29 -08:00
Ravi Khadiwala
ac78eb1425 Update to the latest version of the abusive message filter 2022-12-16 11:28:30 -06:00
Ravi Khadiwala
65ad3fe623 Add hCaptcha support 2022-12-16 11:28:30 -06:00
Sergey Skrobotov
dcec90fc52 Update to the latest version of the abusive message filter 2022-12-13 13:30:47 -08:00
Chris Eager
24ac32e6e6 Add PayPalExperienceProfileInput.userAction 2022-12-13 10:03:58 -06:00
Katherine Yen
26f5ffdde3 Enable case-sensitive usernames 2022-12-13 07:59:37 -08:00
Jon Chambers
a883426402 Simplify account cleaner 2022-12-06 16:21:25 -06:00
Chris Eager
2f21e930e2 Add minimum one-time donation amont to validation error map 2022-12-06 16:21:15 -06:00
Chris Eager
5fb158635c Use existing WebApplicationException entity, if available 2022-12-06 16:21:15 -06:00
Chris Eager
6f844f9ebb Update to the latest version of the abusive message filter 2022-12-06 16:20:17 -06:00
Sergey Skrobotov
d88e358016 Update to the latest version of the abusive message filter 2022-12-05 10:07:40 -08:00
Sergey Skrobotov
9cf2635528 some accounts classes refactorings 2022-12-05 09:30:40 -08:00
Chris Eager
d0e7579f13 Revert transaction descriptor 2022-12-01 18:52:45 -06:00
Chris Eager
cda82b0ea0 Update kotlin + Apollo 2022-12-01 18:11:35 -06:00
Chris Eager
2ecbb18fe5 Add support for one-time PayPal donations 2022-12-01 18:11:35 -06:00
Chris Eager
d40d2389a9 Update to Maven 3.8.6 2022-12-01 18:09:38 -06:00
Chris Eager
df8fb5cab7 Move messages cache stale discard to a separate scheduler 2022-12-01 18:09:28 -06:00
katherine-signal
99ad211c01 Enforce minimum amount by currency for one time donations 2022-11-28 11:44:59 -08:00
katherine-signal
fb4ed20ff5 Remove groups v2 capability
* wip removing groups v2 capabilities

* comments

* finish removing groups v2 references

* hardcode gv1migration flag on user capability, remove other references
2022-11-21 09:31:47 -08:00
Jon Chambers
cb50b44d8f Allow the account cleaner to operate on multiple accounts in parallel 2022-11-18 11:15:00 -05:00
Jon Chambers
ae57853ec4 Simplify deletion reason reporting 2022-11-18 11:15:00 -05:00
Jon Chambers
2881c0fd7e Allow the account cleaner to act on all accounts in a crawled chunk 2022-11-18 11:15:00 -05:00
Chris Eager
483fb0968b Use badge name in level configuration for one-time donations 2022-11-18 11:05:23 -05:00
Jon Chambers
4d37418c15 Update to the latest version of the abusive message filter 2022-11-18 10:55:15 -05:00
Jon Chambers
e8ee4b50ff Retire the legacy "abusive hosts" system in favor of newer tools 2022-11-18 10:54:25 -05:00
Chris Eager
4f8aa2eee2 Mark flaky test @Disabled 2022-11-17 13:23:42 -06:00
Chris Eager
397d3cb45a Add consolidated subscription configuration API 2022-11-16 12:27:00 -06:00
Chris Eager
e883d727fb Note deprecation of localized string 2022-11-16 12:09:00 -06:00
Chris Eager
986545a140 Set error_if_incomplete for subscription payment behavior 2022-11-16 12:08:21 -06:00
Sergey Skrobotov
836307b0c7 adding a metric for ipv4/ipv6 requests count 2022-11-15 11:17:01 -08:00
Sergey Skrobotov
b5a75d3079 Update to the latest version of the abusive message filter 2022-11-15 11:16:55 -08:00
Sergey Skrobotov
c32067759c refactoring: use constants for header names 2022-11-15 11:16:49 -08:00
Chris Eager
7fb7abb593 Update to micrometer 1.10.0 2022-11-15 11:16:41 -08:00
Erik Osheim
0d50b58c60 Update to the latest version of the abusive message filter 2022-11-11 17:09:24 -05:00
Chris Eager
bdf4e24266 Update to the latest version of the abusive message filter 2022-11-11 13:54:19 -06:00
Chris Eager
f41bdf1acb Make MessagesController#getPendingMessages fully async 2022-11-11 13:19:57 -06:00
Chris Eager
77d691df59 Always use reactived message processing in WebSocketConnection 2022-11-11 13:14:39 -06:00
Chris Eager
12300761ab Update reactor-bom to 2020.0.24 2022-11-11 13:14:26 -06:00
Chris Eager
25efcbda81 Update lettuce to 6.2.1.RELEASE 2022-11-11 13:14:26 -06:00
Jon Chambers
a01f96e0e4 Temporarily disable account freezing on contention 2022-11-10 18:53:58 -05:00
erik-signal
1d1e3ba79d Add metric to track newly-locked accounts. 2022-11-10 12:55:08 -05:00
Jon Chambers
2c9c50711f Avoid reading from a stale Account after a contested reglock event 2022-11-10 12:41:50 -05:00
Jon Chambers
d3f0ab8c6d Introduce an alternative exchange rate data provider 2022-11-10 10:25:06 -05:00
erik-signal
80a3a8a43c Lock account when number owner lacks registration lock. 2022-11-09 14:03:09 -05:00
Chris Eager
e6e6eb323d Update metric name 2022-11-08 11:15:42 -06:00
Chris Eager
681a5bafb4 Update MessagesManager#getMessagesForDevice
- add `subscribeOn()`
- use `CompletableFuture` for consistency
2022-11-08 09:38:52 -06:00
Chris Eager
5bec89ecc8 Measure individual message timeouts 2022-11-08 09:37:37 -06:00
Chris Eager
69ed0edb74 Revert "Add more detailed queue processing rate metrics"
This reverts commit bbbab4b8a4.
2022-11-08 09:35:39 -06:00
Chris Eager
ad5925908e Change dispatch queues to LinkedBlockingQueues 2022-11-04 11:08:17 -05:00
Chris Eager
d186245c5c Move all receipt sending work to executor 2022-11-04 11:08:06 -05:00
Chris Eager
bbbab4b8a4 Add more detailed queue processing rate metrics 2022-11-04 11:06:38 -05:00
Chris Eager
f83080eb8d Update metric name 2022-11-03 14:50:20 -05:00
Chris Eager
e0178fa0ea Move additional handling of MessagesManager#delete to executor 2022-11-03 13:02:25 -05:00
Chris Eager
c6a79ca176 Enable metrics on messages fluxes 2022-11-03 13:02:25 -05:00
Chris Eager
6426e6cc49 Enable reactor Schedulers metrics 2022-11-03 13:02:25 -05:00
Chris Eager
b13cb098ce lettuce: set publishOnScheduler to true 2022-11-03 13:02:25 -05:00
Jon Chambers
afda5ca98f Add a test for checking push challenge tokens 2022-11-03 11:14:59 -05:00
Chris Eager
eb57d87513 Remove message listener key only after successfully unsubscribing 2022-11-03 11:09:11 -05:00
Chris Eager
fbf6b9826e tests: only call SQLite.setLibraryPath once 2022-11-03 11:08:43 -05:00
Chris Eager
a01b29a6bd set off_session=true for subscription updates 2022-11-02 14:34:26 -05:00
Chris Eager
102992b095 Set off_session=true when creating subscriptions 2022-11-02 11:30:29 -05:00
Chris Eager
bd69905f2e Remove obsolete donation endpoint 2022-11-02 11:29:03 -05:00
Chris Eager
ce5a4bd94a Update wiremock to 2.34.0 2022-11-02 11:24:54 -05:00
Chris Eager
f65a613815 Update jackson to 2.13.4 2022-11-02 11:24:54 -05:00
sergey-signal
d87c8468bd Update to the latest version of the abusive message filter (#1138) 2022-11-02 09:23:38 -07:00
Chris Eager
aa829af43b Handle expected case of empty flux in message deletion 2022-10-31 12:29:25 -05:00
Chris Eager
c10fda8363 Use reactive streams for WebSocket message queue
Initially, uses `ExperimentEnrollmentManager` to do a safe rollout.
2022-10-31 10:35:37 -05:00
Jon Chambers
4252284405 Update to the latest version of the abusive message filter 2022-10-28 10:50:49 -04:00
Jon Chambers
74d65b37a8 Discard old Twilio machinery and rely entirely on the stand-alone registration service 2022-10-28 10:40:37 -04:00
sergey-signal
78f95e4859 Update to the latest version of the abusive message filter (#1132) 2022-10-27 14:01:16 -07:00
Jon Chambers
91626dea45 Count accounts rather than devices that are stories-capable 2022-10-25 16:36:05 -04:00
sergey-signal
5868d9969a minor changes to utility classes (#1127) 2022-10-25 08:48:56 -07:00
erik-signal
90490c9c84 Clean up the TestClock code a bit more. 2022-10-21 15:27:15 -04:00
Chris Eager
8ea794baef Add additional handling for nullable field in recurring donation record 2022-10-21 12:56:39 -05:00
Chris Eager
70a6c3e8e5 Update to libsignal-server 0.21.1 2022-10-21 12:54:18 -05:00
Jon Chambers
4813803c49 Add .java-version to .gitignore 2022-10-21 12:40:11 -04:00
erik-signal
fe60cf003f Clean up testing with clocks. 2022-10-21 12:39:47 -04:00
erik-signal
0c357bc340 Add metrics tracking story capability adoption. 2022-10-20 12:25:03 -04:00
Chris Eager
b711288faa Run GitHub Action in a container 2022-10-18 16:59:35 -05:00
Jon Chambers
44a5d86641 Revert "Update to libsignal-server 0.21.0"
This reverts commit cccccb4dd6.
2022-10-18 11:44:50 -04:00
Jon Chambers
e7048aa9cf Allow the reconciliation client to trust multiple CA certificates to facilitate certificate rotation 2022-10-18 11:17:47 -04:00
Jon Chambers
0120a85c39 Allow HTTP clients to trust multiple certificates to support certificate rollover 2022-10-18 11:17:47 -04:00
Jon Chambers
a41d047f58 Retire CertificateExpirationGauge in favor of other expiration monitoring tools 2022-10-18 11:17:47 -04:00
Chris Eager
cccccb4dd6 Update to libsignal-server 0.21.0 2022-10-18 11:17:29 -04:00
Jon Chambers
0a64e31625 Check verification codes for changing phone numbers against the stand-alone registration service when possible 2022-10-18 11:17:15 -04:00
Jon Chambers
3c6c6c3706 Use the gRPC BOM instead of calling out dependencies individually 2022-10-18 11:16:56 -04:00
Jon Chambers
8088b58b3b Clarify default value for includeE164 2022-10-18 11:16:06 -04:00
erik-signal
a7d5d51fb4 Improve testing of MultiRecipientMessageProvider 2022-10-17 16:50:39 -04:00
Chris Eager
378d7987a8 device capabilities: prevent stories downgrade 2022-10-17 15:25:13 -04:00
194 changed files with 42468 additions and 6918 deletions

View File

@@ -5,14 +5,19 @@ on: [push]
jobs:
build:
runs-on: ubuntu-latest
container: ubuntu:22.04
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
- name: Set up JDK 17
uses: actions/setup-java@3bc31aaf88e8fc94dc1e632d48af61be5ca8721c
uses: actions/setup-java@de1bb2b0c5634f0fc4438d7aa9944e68f9bf86cc # v3.6.0
with:
distribution: 'temurin'
java-version: 17
cache: 'maven'
env:
# work around an issue with actions/runner setting an incorrect HOME in containers, which breaks maven caching
# https://github.com/actions/setup-java/issues/356
HOME: /root
- name: Build with Maven
run: mvn -e -B verify
run: ./mvnw -e -B verify

1
.gitignore vendored
View File

@@ -16,6 +16,7 @@ config/deploy.properties
/service/config/testing.yml
/service/config/deploy.properties
/service/dependency-reduced-pom.xml
.java-version
.opsmanage
put.sh
deployer-staging.properties

View File

@@ -5,14 +5,14 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
#
# https://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.5/apache-maven-3.8.5-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

View File

@@ -24,7 +24,16 @@
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin.version}</version>
<exclusions>
<exclusion>
<groupId>org.jetbrains</groupId>
<!--
depends on an outdated version (13.0) for JDK 6 compatibility, but its safe to override
https://youtrack.jetbrains.com/issue/KT-25047
-->
<artifactId>annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlinx</groupId>

55
pom.xml
View File

@@ -44,23 +44,24 @@
<properties>
<aws.sdk.version>1.12.287</aws.sdk.version>
<aws.sdk2.version>2.17.258</aws.sdk2.version>
<braintree.version>3.19.0</braintree.version>
<commons-codec.version>1.15</commons-codec.version>
<commons-csv.version>1.8</commons-csv.version>
<commons-io.version>2.9.0</commons-io.version>
<dropwizard.version>2.0.32</dropwizard.version>
<dropwizard-metrics-datadog.version>1.1.13</dropwizard-metrics-datadog.version>
<grpc.version>1.49.0</grpc.version>
<grpc.version>1.49.2</grpc.version>
<gson.version>2.9.0</gson.version>
<guava.version>30.1.1-jre</guava.version>
<jackson.version>2.13.3</jackson.version>
<jackson.version>2.13.4</jackson.version>
<jaxb.version>2.3.1</jaxb.version>
<jedis.version>2.9.0</jedis.version>
<kotlin.version>1.7.10</kotlin.version>
<kotlinx-serialization.version>1.4.0</kotlinx-serialization.version>
<lettuce.version>6.1.9.RELEASE</lettuce.version>
<kotlin.version>1.7.21</kotlin.version>
<kotlinx-serialization.version>1.4.1</kotlinx-serialization.version>
<lettuce.version>6.2.1.RELEASE</lettuce.version>
<libphonenumber.version>8.12.54</libphonenumber.version>
<logstash.logback.version>7.0.1</logstash.logback.version>
<micrometer.version>1.9.3</micrometer.version>
<micrometer.version>1.10.0</micrometer.version>
<mockito.version>4.7.0</mockito.version>
<netty.version>4.1.82.Final</netty.version>
<opentest4j.version>1.2.0</opentest4j.version>
@@ -84,7 +85,7 @@
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
<version>2.13.3</version>
<version>${jackson.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@@ -97,19 +98,10 @@
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<artifactId>grpc-bom</artifactId>
<version>${grpc.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Needed for gRPC with Java 9+ -->
<dependency>
@@ -160,6 +152,20 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-bom</artifactId>
<version>2020.0.24</version> <!-- 3.4.x, see https://github.com/reactor/reactor#bom-versioning-scheme -->
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-bom</artifactId>
<version>${kotlin.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.eatthepath</groupId>
<artifactId>pushy</artifactId>
@@ -286,6 +292,11 @@
<artifactId>stripe-java</artifactId>
<version>${stripe.version}</version>
</dependency>
<dependency>
<groupId>com.braintreepayments.gateway</groupId>
<artifactId>braintree-java</artifactId>
<version>${braintree.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
@@ -300,7 +311,7 @@
<dependency>
<groupId>org.signal</groupId>
<artifactId>libsignal-server</artifactId>
<version>0.18.0</version>
<version>0.21.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@@ -322,7 +333,7 @@
<dependency>
<groupId>com.github.tomakehurst</groupId>
<artifactId>wiremock-jre8</artifactId>
<version>2.33.2</version>
<version>2.34.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -477,7 +488,7 @@
<rules>
<dependencyConvergence/>
<requireMavenVersion>
<version>3.8.3</version>
<version>3.8.6</version>
</requireMavenVersion>
</rules>
</configuration>

View File

@@ -15,6 +15,25 @@ stripe:
idempotencyKeyGenerator: abcdefg12345678= # base64 for creating request idempotency hash
boostDescription: >
Example
supportedCurrencies:
- xts
# - ...
# - Nth supported currency
braintree:
merchantId: unset
publicKey: unset
privateKey: unset
environment: unset
graphqlUrl: unset
merchantAccounts:
# ISO 4217 currency code and its corresponding sub-merchant account
'xts': unset
supportedCurrencies:
- xts
# - ...
# - Nth supported currency
dynamoDbClientConfiguration:
region: us-west-2 # AWS Region
@@ -62,29 +81,6 @@ dynamoDbTables:
subscriptions:
tableName: Example_Subscriptions
twilio: # Twilio gateway configuration
accountId: unset
accountToken: unset
nanpaMessagingServiceSid: unset # Twilio SID for the messaging service to use for NANPA.
messagingServiceSid: unset # Twilio SID for the message service to use for non-NANPA.
verifyServiceSid: unset # Twilio SID for a Verify service
localDomain: example.com # Domain Twilio can connect back to for calls. Should be domain of your service.
defaultClientVerificationTexts:
ios: example %1$s # Text to use for the verification message on iOS. Will be passed to String.format with the verification code as argument 1.
androidNg: example %1$s # Text to use for the verification message on android-ng client types. Will be passed to String.format with the verification code as argument 1.
android202001: example %1$s # Text to use for the verification message on android-2020-01 client types. Will be passed to String.format with the verification code as argument 1.
android202103: example %1$s # Text to use for the verification message on android-2021-03 client types. Will be passed to String.format with the verification code as argument 1.
generic: example %1$s # Text to use when the client type is unrecognized. Will be passed to String.format with the verification code as argument 1.
regionalClientVerificationTexts: # Map of country codes to custom texts
999: # example country code
ios: example %1$s # all keys from defaultClientVerificationTexts are required
androidNg: example %1$s
android202001: example %1$s
android202103: example %1$s
generic: example %1$s
androidAppHash: example # Hash appended to Android
verifyServiceFriendlyName: example # Service name used in template. Requires Twilio account rep to enable
cacheCluster: # Redis server configuration for cache cluster
configurationUri: redis://redis.example.com:6379/
@@ -115,28 +111,29 @@ directory:
- replicationName: example # CDS replication name
replicationUrl: cds.example.com # CDS replication endpoint base url
replicationPassword: example # CDS replication endpoint password
replicationCaCertificate: | # CDS replication endpoint TLS certificate trust root
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
replicationCaCertificates: # CDS replication endpoint TLS certificate trust root
- |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
directoryV2:
client: # Configuration for interfacing with Contact Discovery Service v2 cluster
@@ -237,57 +234,62 @@ recaptcha:
projectPath: projects/example
credentialConfigurationJson: "{ }" # service account configuration for backend authentication
hCaptcha:
apiKey: unset
storageService:
uri: storage.example.com
userAuthenticationTokenSharedSecret: 00000f
storageCaCertificate: |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
storageCaCertificates:
- |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
backupService:
uri: backup.example.com
userAuthenticationTokenSharedSecret: 00000f
backupCaCertificate: |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
backupCaCertificates:
- |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
zkConfig:
serverPublic: ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
@@ -310,17 +312,16 @@ remoteConfig:
paymentsService:
userAuthenticationTokenSharedSecret: 0000000f0000000f0000000f0000000f0000000f0000000f0000000f0000000f # hex-encoded 32-byte secret shared with MobileCoin services used to generate auth tokens for Signal users
fixerApiKey: unset
coinMarketCapApiKey: unset
coinMarketCapCurrencyIds:
MOB: 7878
paymentCurrencies:
# list of symbols for supported currencies
- MOB
donation:
uri: donation.example.com # value
supportedCurrencies:
- # 1st supported currency
- # 2nd supported currency
- # ...
- # Nth supported currency
artService:
userAuthenticationTokenSharedSecret: 0000000f0000000f0000000f0000000f0000000f0000000f0000000f0000000f # hex-encoded 32-byte secret not shared with any external service, but used in ArtController
userAuthenticationTokenUserIdSecret: 00000f # hex-encoded secret to obscure user phone numbers from Sticker Creator
badges:
badges:
@@ -353,27 +354,27 @@ subscription: # configuration for Stripe subscriptions
amount: '10'
id: price_example # stripe ID
boost:
level: 1
expiration: P90D
badge: EXAMPLE
oneTimeDonations:
boost:
level: 1
expiration: P90D
badge: EXAMPLE
gift:
level: 10
expiration: P90D
badge: EXAMPLE
currencies:
# ISO 4217 currency codes and amounts in those currencies
xts:
- '1'
- '2'
- '4'
- '8'
- '20'
- '40'
gift:
level: 10
expiration: P90D
badge: EXAMPLE
currencies:
# ISO 4217 currency codes and amounts in those currencies
xts: '2'
minimum: '0.5'
gift: '2'
boosts:
- '1'
- '2'
- '4'
- '8'
- '20'
- '40'
registrationService:
host: registration.example.com

View File

@@ -228,6 +228,10 @@
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-retry</artifactId>
</dependency>
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-reactor</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
@@ -407,7 +411,6 @@
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
<version>3.3.22.RELEASE</version>
</dependency>
<dependency>
<groupId>io.vavr</groupId>
@@ -420,6 +423,11 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
</dependency>
<dependency>
<groupId>org.signal</groupId>
<artifactId>embedded-redis</artifactId>
@@ -449,6 +457,18 @@
<groupId>com.stripe</groupId>
<artifactId>stripe-java</artifactId>
</dependency>
<dependency>
<groupId>com.braintreepayments.gateway</groupId>
<artifactId>braintree-java</artifactId>
</dependency>
<dependency>
<groupId>com.apollographql.apollo3</groupId>
<artifactId>apollo-api-jvm</artifactId>
<version>3.7.1</version>
</dependency>
</dependencies>
<profiles>
@@ -604,6 +624,31 @@
</arguments>
</configuration>
</plugin>
<plugin>
<groupId>com.github.aoudiamoncef</groupId>
<artifactId>apollo-client-maven-plugin</artifactId>
<version>5.0.0</version>
<executions>
<execution>
<goals>
<goal>generate</goal>
</goals>
<configuration>
<services>
<braintree>
<compilationUnit>
<name>braintree</name>
<compilerParams>
<schemaPackageName>com.braintree.graphql.client</schemaPackageName>
</compilerParams>
</compilationUnit>
</braintree>
</services>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@@ -0,0 +1,9 @@
# https://graphql.braintreepayments.com/reference/#Mutation--chargePaymentMethod
mutation ChargePayPalOneTimePayment($input: ChargePaymentMethodInput!) {
chargePaymentMethod(input: $input) {
transaction {
id,
status
}
}
}

View File

@@ -0,0 +1,7 @@
# https://graphql.braintreepayments.com/reference/#Mutation--createPayPalOneTimePayment
mutation CreatePayPalOneTimePayment($input: CreatePayPalOneTimePaymentInput!) {
createPayPalOneTimePayment(input: $input) {
approvalUrl,
paymentId
}
}

View File

@@ -0,0 +1,8 @@
# https://graphql.braintreepayments.com/reference/#Mutation--tokenizePayPalOneTimePayment
mutation TokenizePayPalOneTimePayment($input: TokenizePayPalOneTimePaymentInput!) {
tokenizePayPalOneTimePayment(input: $input) {
paymentMethod {
id
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm;
@@ -19,20 +19,21 @@ import org.whispersystems.textsecuregcm.configuration.ApnConfiguration;
import org.whispersystems.textsecuregcm.configuration.AppConfigConfiguration;
import org.whispersystems.textsecuregcm.configuration.AwsAttachmentsConfiguration;
import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration;
import org.whispersystems.textsecuregcm.configuration.BoostConfiguration;
import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration;
import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
import org.whispersystems.textsecuregcm.configuration.DirectoryConfiguration;
import org.whispersystems.textsecuregcm.configuration.DirectoryV2Configuration;
import org.whispersystems.textsecuregcm.configuration.DonationConfiguration;
import org.whispersystems.textsecuregcm.configuration.DynamoDbClientConfiguration;
import org.whispersystems.textsecuregcm.configuration.DynamoDbTables;
import org.whispersystems.textsecuregcm.configuration.FcmConfiguration;
import org.whispersystems.textsecuregcm.configuration.GcpAttachmentsConfiguration;
import org.whispersystems.textsecuregcm.configuration.GiftConfiguration;
import org.whispersystems.textsecuregcm.configuration.HCaptchaConfiguration;
import org.whispersystems.textsecuregcm.configuration.MaxDeviceConfiguration;
import org.whispersystems.textsecuregcm.configuration.MessageCacheConfiguration;
import org.whispersystems.textsecuregcm.configuration.OneTimeDonationConfiguration;
import org.whispersystems.textsecuregcm.configuration.PaymentsServiceConfiguration;
import org.whispersystems.textsecuregcm.configuration.ArtServiceConfiguration;
import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration;
import org.whispersystems.textsecuregcm.configuration.RecaptchaConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
@@ -45,7 +46,6 @@ import org.whispersystems.textsecuregcm.configuration.SecureStorageServiceConfig
import org.whispersystems.textsecuregcm.configuration.StripeConfiguration;
import org.whispersystems.textsecuregcm.configuration.SubscriptionConfiguration;
import org.whispersystems.textsecuregcm.configuration.TestDeviceConfiguration;
import org.whispersystems.textsecuregcm.configuration.TwilioConfiguration;
import org.whispersystems.textsecuregcm.configuration.UnidentifiedDeliveryConfiguration;
import org.whispersystems.textsecuregcm.configuration.UsernameConfiguration;
import org.whispersystems.textsecuregcm.configuration.VoiceVerificationConfiguration;
@@ -65,6 +65,11 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private StripeConfiguration stripe;
@NotNull
@Valid
@JsonProperty
private BraintreeConfiguration braintree;
@NotNull
@Valid
@JsonProperty
@@ -75,11 +80,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private DynamoDbTables dynamoDbTables;
@NotNull
@Valid
@JsonProperty
private TwilioConfiguration twilio;
@NotNull
@Valid
@JsonProperty
@@ -195,6 +195,11 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private RecaptchaConfiguration recaptcha;
@Valid
@NotNull
@JsonProperty
private HCaptchaConfiguration hCaptcha;
@Valid
@NotNull
@JsonProperty
@@ -210,6 +215,11 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private PaymentsServiceConfiguration paymentsService;
@Valid
@NotNull
@JsonProperty
private ArtServiceConfiguration artService;
@Valid
@NotNull
@JsonProperty
@@ -225,11 +235,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private AppConfigConfiguration appConfig;
@Valid
@NotNull
@JsonProperty
private DonationConfiguration donation;
@Valid
@NotNull
@JsonProperty
@@ -243,12 +248,7 @@ public class WhisperServerConfiguration extends Configuration {
@Valid
@JsonProperty
@NotNull
private BoostConfiguration boost;
@Valid
@JsonProperty
@NotNull
private GiftConfiguration gift;
private OneTimeDonationConfiguration oneTimeDonations;
@Valid
@NotNull
@@ -277,6 +277,10 @@ public class WhisperServerConfiguration extends Configuration {
return stripe;
}
public BraintreeConfiguration getBraintree() {
return braintree;
}
public DynamoDbClientConfiguration getDynamoDbClientConfiguration() {
return dynamoDbClientConfiguration;
}
@@ -289,6 +293,10 @@ public class WhisperServerConfiguration extends Configuration {
return recaptcha;
}
public HCaptchaConfiguration getHCaptchaConfiguration() {
return hCaptcha;
}
public VoiceVerificationConfiguration getVoiceVerificationConfiguration() {
return voiceVerification;
}
@@ -297,10 +305,6 @@ public class WhisperServerConfiguration extends Configuration {
return webSocket;
}
public TwilioConfiguration getTwilioConfiguration() {
return twilio;
}
public AwsAttachmentsConfiguration getAwsAttachmentsConfiguration() {
return awsAttachments;
}
@@ -407,6 +411,10 @@ public class WhisperServerConfiguration extends Configuration {
return paymentsService;
}
public ArtServiceConfiguration getArtServiceConfiguration() {
return artService;
}
public ZkConfig getZkConfig() {
return zkConfig;
}
@@ -419,10 +427,6 @@ public class WhisperServerConfiguration extends Configuration {
return appConfig;
}
public DonationConfiguration getDonationConfiguration() {
return donation;
}
public BadgesConfiguration getBadges() {
return badges;
}
@@ -431,12 +435,8 @@ public class WhisperServerConfiguration extends Configuration {
return subscription;
}
public BoostConfiguration getBoost() {
return boost;
}
public GiftConfiguration getGift() {
return gift;
public OneTimeDonationConfiguration getOneTimeDonations() {
return oneTimeDonations;
}
public ReportMessageConfiguration getReportMessageConfiguration() {

View File

@@ -83,6 +83,8 @@ import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener;
import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter;
import org.whispersystems.textsecuregcm.badges.ResourceBundleLevelTranslator;
import org.whispersystems.textsecuregcm.captcha.CaptchaChecker;
import org.whispersystems.textsecuregcm.captcha.HCaptchaClient;
import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.controllers.AccountController;
@@ -104,19 +106,19 @@ import org.whispersystems.textsecuregcm.controllers.RemoteConfigController;
import org.whispersystems.textsecuregcm.controllers.SecureBackupController;
import org.whispersystems.textsecuregcm.controllers.SecureStorageController;
import org.whispersystems.textsecuregcm.controllers.StickerController;
import org.whispersystems.textsecuregcm.controllers.ArtController;
import org.whispersystems.textsecuregcm.controllers.SubscriptionController;
import org.whispersystems.textsecuregcm.controllers.VoiceVerificationController;
import org.whispersystems.textsecuregcm.currency.CoinMarketCapClient;
import org.whispersystems.textsecuregcm.currency.CurrencyConversionManager;
import org.whispersystems.textsecuregcm.currency.FixerClient;
import org.whispersystems.textsecuregcm.currency.FtxClient;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.filters.ContentLengthFilter;
import org.whispersystems.textsecuregcm.filters.RequestStatisticsFilter;
import org.whispersystems.textsecuregcm.filters.RemoteDeprecationFilter;
import org.whispersystems.textsecuregcm.filters.TimestampResponseFilter;
import org.whispersystems.textsecuregcm.limits.DynamicRateLimiters;
import org.whispersystems.textsecuregcm.limits.PushChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeOptionManager;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.mappers.CompletionExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.DeviceLimitExceededExceptionMapper;
@@ -154,7 +156,7 @@ import org.whispersystems.textsecuregcm.push.ProvisioningManager;
import org.whispersystems.textsecuregcm.push.PushLatencyManager;
import org.whispersystems.textsecuregcm.push.PushNotificationManager;
import org.whispersystems.textsecuregcm.push.ReceiptSender;
import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.captcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.redis.ConnectionEventLogger;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.ReplicatedJedisPool;
@@ -163,11 +165,7 @@ import org.whispersystems.textsecuregcm.s3.PolicySigner;
import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sms.SmsSender;
import org.whispersystems.textsecuregcm.sms.TwilioSmsSender;
import org.whispersystems.textsecuregcm.sms.TwilioVerifyExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRules;
import org.whispersystems.textsecuregcm.storage.AccountCleaner;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
@@ -205,6 +203,7 @@ import org.whispersystems.textsecuregcm.storage.ReportMessageManager;
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
import org.whispersystems.textsecuregcm.storage.SubscriptionManager;
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
import org.whispersystems.textsecuregcm.subscriptions.BraintreeManager;
import org.whispersystems.textsecuregcm.subscriptions.StripeManager;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
@@ -227,6 +226,7 @@ import org.whispersystems.textsecuregcm.workers.SetUserDiscoverabilityCommand;
import org.whispersystems.textsecuregcm.workers.ZkParamsCommand;
import org.whispersystems.websocket.WebSocketResourceProviderFactory;
import org.whispersystems.websocket.setup.WebSocketEnvironment;
import reactor.core.scheduler.Schedulers;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
@@ -333,7 +333,14 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
config.getAppConfig().getConfigurationName(),
DynamicConfiguration.class);
Accounts accounts = new Accounts(dynamicConfigurationManager,
BlockingQueue<Runnable> messageDeletionQueue = new LinkedBlockingQueue<>();
Metrics.gaugeCollectionSize(name(getClass(), "messageDeletionQueueSize"), Collections.emptyList(),
messageDeletionQueue);
ExecutorService messageDeletionAsyncExecutor = environment.lifecycle()
.executorService(name(getClass(), "messageDeletionAsyncExecutor-%d")).maxThreads(16)
.workQueue(messageDeletionQueue).build();
Accounts accounts = new Accounts(
dynamoDbClient,
dynamoDbAsyncClient,
config.getDynamoDbTables().getAccounts().getTableName(),
@@ -348,9 +355,10 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
Profiles profiles = new Profiles(dynamoDbClient, dynamoDbAsyncClient,
config.getDynamoDbTables().getProfiles().getTableName());
Keys keys = new Keys(dynamoDbClient, config.getDynamoDbTables().getKeys().getTableName());
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(dynamoDbClient,
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(dynamoDbClient, dynamoDbAsyncClient,
config.getDynamoDbTables().getMessages().getTableName(),
config.getDynamoDbTables().getMessages().getExpiration());
config.getDynamoDbTables().getMessages().getExpiration(),
messageDeletionAsyncExecutor);
RemoteConfigs remoteConfigs = new RemoteConfigs(dynamoDbClient,
config.getDynamoDbTables().getRemoteConfig().getTableName());
PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(dynamoDbClient,
@@ -363,8 +371,13 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
VerificationCodeStore pendingDevices = new VerificationCodeStore(dynamoDbClient,
config.getDynamoDbTables().getPendingDevices().getTableName());
RedisClientFactory pubSubClientFactory = new RedisClientFactory("pubsub_cache", config.getPubsubCacheConfiguration().getUrl(), config.getPubsubCacheConfiguration().getReplicaUrls(), config.getPubsubCacheConfiguration().getCircuitBreakerConfiguration());
ReplicatedJedisPool pubsubClient = pubSubClientFactory.getRedisClientPool();
reactor.util.Metrics.MicrometerConfiguration.useRegistry(Metrics.globalRegistry);
Schedulers.enableMetrics();
RedisClientFactory pubSubClientFactory = new RedisClientFactory("pubsub_cache",
config.getPubsubCacheConfiguration().getUrl(), config.getPubsubCacheConfiguration().getReplicaUrls(),
config.getPubsubCacheConfiguration().getCircuitBreakerConfiguration());
ReplicatedJedisPool pubsubClient = pubSubClientFactory.getRedisClientPool();
MicrometerOptions options = MicrometerOptions.builder().build();
ClientResources redisClientResources = ClientResources.builder()
@@ -378,9 +391,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
FaultTolerantRedisCluster pushSchedulerCluster = new FaultTolerantRedisCluster("push_scheduler", config.getPushSchedulerCluster(), redisClientResources);
FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters", config.getRateLimitersCluster(), redisClientResources);
BlockingQueue<Runnable> keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(10_000);
final BlockingQueue<Runnable> keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(100_000);
Metrics.gaugeCollectionSize(name(getClass(), "keyspaceNotificationDispatchQueueSize"), Collections.emptyList(), keyspaceNotificationDispatchQueue);
final ArrayBlockingQueue<Runnable> receiptSenderQueue = new ArrayBlockingQueue<>(10_000);
final BlockingQueue<Runnable> receiptSenderQueue = new LinkedBlockingQueue<>();
Metrics.gaugeCollectionSize(name(getClass(), "receiptSenderQueue"), Collections.emptyList(), receiptSenderQueue);
final BlockingQueue<Runnable> fcmSenderQueue = new LinkedBlockingQueue<>();
@@ -394,15 +407,17 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
ExecutorService fcmSenderExecutor = environment.lifecycle().executorService(name(getClass(), "fcmSender-%d")).maxThreads(32).minThreads(32).workQueue(fcmSenderQueue).build();
ExecutorService backupServiceExecutor = environment.lifecycle().executorService(name(getClass(), "backupService-%d")).maxThreads(1).minThreads(1).build();
ExecutorService storageServiceExecutor = environment.lifecycle().executorService(name(getClass(), "storageService-%d")).maxThreads(1).minThreads(1).build();
ExecutorService accountDeletionExecutor = environment.lifecycle().executorService(name(getClass(), "accountCleaner-%d")).maxThreads(16).minThreads(16).build();
// TODO: generally speaking this is a DynamoDB I/O executor for the accounts table; we should eventually have a general executor for speaking to the accounts table, but most of the server is still synchronous so this isn't widely useful yet
ExecutorService batchIdentityCheckExecutor = environment.lifecycle().executorService(name(getClass(), "batchIdentityCheck-%d")).minThreads(32).maxThreads(32).build();
ExecutorService multiRecipientMessageExecutor = environment.lifecycle()
.executorService(name(getClass(), "multiRecipientMessage-%d")).minThreads(64).maxThreads(64).build();
ExecutorService stripeExecutor = environment.lifecycle().executorService(name(getClass(), "stripe-%d")).
maxThreads(availableProcessors). // mostly this is IO bound so tying to number of processors is tenuous at best
minThreads(availableProcessors). // mostly this is IO bound so tying to number of processors is tenuous at best
allowCoreThreadTimeOut(true).
ExecutorService subscriptionProcessorExecutor = environment.lifecycle()
.executorService(name(getClass(), "subscriptionProcessor-%d"))
.maxThreads(availableProcessors) // mostly this is IO bound so tying to number of processors is tenuous at best
.minThreads(availableProcessors) // mostly this is IO bound so tying to number of processors is tenuous at best
.allowCoreThreadTimeOut(true).
build();
ExecutorService receiptSenderExecutor = environment.lifecycle()
.executorService(name(getClass(), "receiptSender-%d"))
@@ -425,8 +440,13 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
config.getAdminEventLoggingConfiguration().projectId(),
config.getAdminEventLoggingConfiguration().logName());
StripeManager stripeManager = new StripeManager(config.getStripe().getApiKey(), stripeExecutor,
config.getStripe().getIdempotencyKeyGenerator(), config.getStripe().getBoostDescription());
StripeManager stripeManager = new StripeManager(config.getStripe().apiKey(), subscriptionProcessorExecutor,
config.getStripe().idempotencyKeyGenerator(), config.getStripe().boostDescription(), config.getStripe()
.supportedCurrencies());
BraintreeManager braintreeManager = new BraintreeManager(config.getBraintree().merchantId(),
config.getBraintree().publicKey(), config.getBraintree().privateKey(), config.getBraintree().environment(),
config.getBraintree().supportedCurrencies(), config.getBraintree().merchantAccounts(),
config.getBraintree().graphqlUrl(), config.getBraintree().circuitBreaker(), subscriptionProcessorExecutor);
ExternalServiceCredentialGenerator directoryCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenSharedSecret(),
@@ -440,17 +460,17 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(dynamicConfigurationManager);
TwilioVerifyExperimentEnrollmentManager verifyExperimentEnrollmentManager = new TwilioVerifyExperimentEnrollmentManager(
config.getVoiceVerificationConfiguration(), experimentEnrollmentManager);
ExternalServiceCredentialGenerator storageCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getSecureStorageServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
ExternalServiceCredentialGenerator backupCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getSecureBackupServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
ExternalServiceCredentialGenerator paymentsCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getPaymentsServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
ExternalServiceCredentialGenerator artCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getArtServiceConfiguration().getUserAuthenticationTokenSharedSecret(),
config.getArtServiceConfiguration().getUserAuthenticationTokenUserIdSecret(),
true, false, false);
AbusiveHostRules abusiveHostRules = new AbusiveHostRules(rateLimitersCluster, dynamicConfigurationManager);
RegistrationServiceClient registrationServiceClient = new RegistrationServiceClient(config.getRegistrationServiceConfiguration().getHost(), config.getRegistrationServiceConfiguration().getPort(), config.getRegistrationServiceConfiguration().getApiKey(), config.getRegistrationServiceConfiguration().getRegistrationCaCertificate(), registrationCallbackExecutor);
SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor, config.getSecureBackupServiceConfiguration());
SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator, storageServiceExecutor, config.getSecureStorageServiceConfiguration());
@@ -458,12 +478,15 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DirectoryQueue directoryQueue = new DirectoryQueue(config.getDirectoryConfiguration().getSqsConfiguration());
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
StoredVerificationCodeManager pendingDevicesManager = new StoredVerificationCodeManager(pendingDevices);
ProfilesManager profilesManager = new ProfilesManager(profiles, cacheCluster);
MessagesCache messagesCache = new MessagesCache(messagesCluster, messagesCluster, keyspaceNotificationDispatchExecutor);
PushLatencyManager pushLatencyManager = new PushLatencyManager(metricsCluster, dynamicConfigurationManager);
ReportMessageManager reportMessageManager = new ReportMessageManager(reportMessageDynamoDb, rateLimitersCluster, config.getReportMessageConfiguration().getCounterTtl());
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, reportMessageManager);
UsernameGenerator usernameGenerator = new UsernameGenerator(config.getUsername());
ProfilesManager profilesManager = new ProfilesManager(profiles, cacheCluster);
MessagesCache messagesCache = new MessagesCache(messagesCluster, messagesCluster, Clock.systemUTC(),
keyspaceNotificationDispatchExecutor, messageDeletionAsyncExecutor);
PushLatencyManager pushLatencyManager = new PushLatencyManager(metricsCluster, dynamicConfigurationManager);
ReportMessageManager reportMessageManager = new ReportMessageManager(reportMessageDynamoDb, rateLimitersCluster,
config.getReportMessageConfiguration().getCounterTtl());
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, reportMessageManager,
messageDeletionAsyncExecutor);
UsernameGenerator usernameGenerator = new UsernameGenerator(config.getUsername());
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
deletedAccountsLockDynamoDbClient, config.getDynamoDbTables().getDeletedAccountsLock().getTableName());
AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster,
@@ -499,20 +522,21 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
AccountAuthenticator accountAuthenticator = new AccountAuthenticator(accountsManager);
DisabledPermittedAccountAuthenticator disabledPermittedAccountAuthenticator = new DisabledPermittedAccountAuthenticator(accountsManager);
TwilioSmsSender twilioSmsSender = new TwilioSmsSender(config.getTwilioConfiguration(), dynamicConfigurationManager);
SmsSender smsSender = new SmsSender(twilioSmsSender);
MessageSender messageSender = new MessageSender(clientPresenceManager, messagesManager, pushNotificationManager, pushLatencyManager);
ReceiptSender receiptSender = new ReceiptSender(accountsManager, messageSender, receiptSenderExecutor);
TurnTokenGenerator turnTokenGenerator = new TurnTokenGenerator(dynamicConfigurationManager);
RecaptchaClient recaptchaClient = new RecaptchaClient(
config.getRecaptchaConfiguration().getProjectPath(),
config.getRecaptchaConfiguration().getCredentialConfigurationJson(),
dynamicConfigurationManager);
HttpClient hcaptchaHttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).connectTimeout(Duration.ofSeconds(10)).build();
HCaptchaClient hCaptchaClient = new HCaptchaClient(config.getHCaptchaConfiguration().apiKey(), hcaptchaHttpClient, dynamicConfigurationManager);
CaptchaChecker captchaChecker = new CaptchaChecker(List.of(recaptchaClient, hCaptchaClient));
PushChallengeManager pushChallengeManager = new PushChallengeManager(pushNotificationManager, pushChallengeDynamoDb);
RateLimitChallengeManager rateLimitChallengeManager = new RateLimitChallengeManager(pushChallengeManager,
recaptchaClient, dynamicRateLimiters);
RateLimitChallengeOptionManager rateLimitChallengeOptionManager =
new RateLimitChallengeOptionManager(dynamicRateLimiters, dynamicConfigurationManager);
captchaChecker, dynamicRateLimiters);
MessagePersister messagePersister = new MessagePersister(messagesCache, messagesManager, accountsManager, dynamicConfigurationManager, Duration.ofMinutes(config.getMessageCacheConfiguration().getPersistDelayMinutes()));
ChangeNumberManager changeNumberManager = new ChangeNumberManager(messageSender, accountsManager);
@@ -548,7 +572,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
new AccountDatabaseCrawlerCache(cacheCluster, AccountDatabaseCrawlerCache.ACCOUNT_CLEANER_PREFIX);
AccountDatabaseCrawler accountCleanerAccountDatabaseCrawler = new AccountDatabaseCrawler("Account cleaner crawler",
accountsManager,
accountCleanerAccountDatabaseCrawlerCache, List.of(new AccountCleaner(accountsManager)),
accountCleanerAccountDatabaseCrawlerCache, List.of(new AccountCleaner(accountsManager, accountDeletionExecutor)),
config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
config.getAccountDatabaseCrawlerConfiguration().getChunkIntervalMs()
);
@@ -571,10 +595,11 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DeletedAccountsTableCrawler deletedAccountsTableCrawler = new DeletedAccountsTableCrawler(deletedAccountsManager, deletedAccountsDirectoryReconcilers, cacheCluster, recurringJobExecutor);
HttpClient currencyClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).connectTimeout(Duration.ofSeconds(10)).build();
FixerClient fixerClient = new FixerClient(currencyClient, config.getPaymentsServiceConfiguration().getFixerApiKey());
FtxClient ftxClient = new FtxClient(currencyClient);
CurrencyConversionManager currencyManager = new CurrencyConversionManager(fixerClient, ftxClient, config.getPaymentsServiceConfiguration().getPaymentCurrencies());
HttpClient currencyClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).connectTimeout(Duration.ofSeconds(10)).build();
FixerClient fixerClient = new FixerClient(currencyClient, config.getPaymentsServiceConfiguration().getFixerApiKey());
CoinMarketCapClient coinMarketCapClient = new CoinMarketCapClient(currencyClient, config.getPaymentsServiceConfiguration().getCoinMarketCapApiKey(), config.getPaymentsServiceConfiguration().getCoinMarketCapCurrencyIds());
CurrencyConversionManager currencyManager = new CurrencyConversionManager(fixerClient, coinMarketCapClient,
cacheCluster, config.getPaymentsServiceConfiguration().getPaymentCurrencies(), Clock.systemUTC());
environment.lifecycle().manage(apnSender);
environment.lifecycle().manage(apnPushNotificationScheduler);
@@ -617,7 +642,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
.addFilter("RemoteDeprecationFilter", new RemoteDeprecationFilter(dynamicConfigurationManager))
.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/*");
environment.jersey().register(new ContentLengthFilter(TrafficSource.HTTP));
environment.jersey().register(new RequestStatisticsFilter(TrafficSource.HTTP));
environment.jersey().register(MultiRecipientMessageProvider.class);
environment.jersey().register(new MetricsApplicationEventListener(TrafficSource.HTTP));
environment.jersey()
@@ -637,21 +662,24 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
webSocketEnvironment.setConnectListener(
new AuthenticatedConnectListener(receiptSender, messagesManager, pushNotificationManager,
clientPresenceManager, websocketScheduledExecutor));
webSocketEnvironment.jersey().register(new WebsocketRefreshApplicationEventListener(accountsManager, clientPresenceManager));
webSocketEnvironment.jersey().register(new ContentLengthFilter(TrafficSource.WEBSOCKET));
webSocketEnvironment.jersey()
.register(new WebsocketRefreshApplicationEventListener(accountsManager, clientPresenceManager));
webSocketEnvironment.jersey().register(new RequestStatisticsFilter(TrafficSource.WEBSOCKET));
webSocketEnvironment.jersey().register(MultiRecipientMessageProvider.class);
webSocketEnvironment.jersey().register(new MetricsApplicationEventListener(TrafficSource.WEBSOCKET));
webSocketEnvironment.jersey().register(new KeepAliveController(clientPresenceManager));
// these should be common, but use @Auth DisabledPermittedAccount, which isnt supported yet on websocket
environment.jersey().register(
new AccountController(pendingAccountsManager, accountsManager, abusiveHostRules, rateLimiters,
smsSender, registrationServiceClient, dynamicConfigurationManager, turnTokenGenerator, config.getTestDevices(),
recaptchaClient, pushNotificationManager, verifyExperimentEnrollmentManager,
changeNumberManager, backupCredentialsGenerator, experimentEnrollmentManager));
new AccountController(pendingAccountsManager, accountsManager, rateLimiters,
registrationServiceClient, dynamicConfigurationManager, turnTokenGenerator, config.getTestDevices(),
captchaChecker, pushNotificationManager, changeNumberManager, backupCredentialsGenerator,
clientPresenceManager, clock));
environment.jersey().register(new KeysController(rateLimiters, keys, accountsManager));
final List<Object> commonControllers = Lists.newArrayList(
new ArtController(rateLimiters, artCredentialsGenerator),
new AttachmentControllerV2(rateLimiters, config.getAwsAttachmentsConfiguration().getAccessKey(), config.getAwsAttachmentsConfiguration().getAccessSecret(), config.getAwsAttachmentsConfiguration().getRegion(), config.getAwsAttachmentsConfiguration().getBucket()),
new AttachmentControllerV3(rateLimiters, config.getGcpAttachmentsConfiguration().getDomain(), config.getGcpAttachmentsConfiguration().getEmail(), config.getGcpAttachmentsConfiguration().getMaxSizeInBytes(), config.getGcpAttachmentsConfiguration().getPathPrefix(), config.getGcpAttachmentsConfiguration().getRsaSigningKey()),
new CertificateController(new CertificateGenerator(config.getDeliveryCertificate().getCertificate(), config.getDeliveryCertificate().getPrivateKey(), config.getDeliveryCertificate().getExpiresDays()), zkAuthOperations, clock),
@@ -660,22 +688,26 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
new DirectoryController(directoryCredentialsGenerator),
new DirectoryV2Controller(directoryV2CredentialsGenerator),
new DonationController(clock, zkReceiptOperations, redeemedReceiptsManager, accountsManager, config.getBadges(),
ReceiptCredentialPresentation::new, stripeExecutor, config.getDonationConfiguration(), config.getStripe()),
ReceiptCredentialPresentation::new),
new MessageController(rateLimiters, messageSender, receiptSender, accountsManager, deletedAccountsManager, messagesManager, pushNotificationManager, reportMessageManager, multiRecipientMessageExecutor),
new PaymentsController(currencyManager, paymentsCredentialsGenerator),
new ProfileController(clock, rateLimiters, accountsManager, profilesManager, dynamicConfigurationManager, profileBadgeConverter, config.getBadges(), cdnS3Client, profileCdnPolicyGenerator, profileCdnPolicySigner, config.getCdnConfiguration().getBucket(), zkProfileOperations, batchIdentityCheckExecutor),
new ProfileController(clock, rateLimiters, accountsManager, profilesManager, dynamicConfigurationManager,
profileBadgeConverter, config.getBadges(), cdnS3Client, profileCdnPolicyGenerator, profileCdnPolicySigner,
config.getCdnConfiguration().getBucket(), zkProfileOperations, batchIdentityCheckExecutor),
new ProvisioningController(rateLimiters, provisioningManager),
new RemoteConfigController(remoteConfigsManager, adminEventLogger, config.getRemoteConfigConfiguration().getAuthorizedTokens(), config.getRemoteConfigConfiguration().getGlobalConfig()),
new RemoteConfigController(remoteConfigsManager, adminEventLogger,
config.getRemoteConfigConfiguration().getAuthorizedTokens(),
config.getRemoteConfigConfiguration().getGlobalConfig()),
new SecureBackupController(backupCredentialsGenerator),
new SecureStorageController(storageCredentialsGenerator),
new StickerController(rateLimiters, config.getCdnConfiguration().getAccessKey(),
config.getCdnConfiguration().getAccessSecret(), config.getCdnConfiguration().getRegion(),
config.getCdnConfiguration().getBucket())
);
if (config.getSubscription() != null && config.getBoost() != null) {
commonControllers.add(new SubscriptionController(clock, config.getSubscription(), config.getBoost(),
config.getGift(), subscriptionManager, stripeManager, zkReceiptOperations, issuedReceiptsManager,
profileBadgeConverter, resourceBundleLevelTranslator));
if (config.getSubscription() != null && config.getOneTimeDonations() != null) {
commonControllers.add(new SubscriptionController(clock, config.getSubscription(), config.getOneTimeDonations(),
subscriptionManager, stripeManager, braintreeManager, zkReceiptOperations, issuedReceiptsManager, profileBadgeConverter,
resourceBundleLevelTranslator));
}
for (Object controller : commonControllers) {

View File

@@ -7,6 +7,7 @@ package org.whispersystems.textsecuregcm.auth;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.codec.binary.Hex;
import org.signal.libsignal.protocol.kdf.HKDF;
import org.whispersystems.textsecuregcm.util.Util;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -32,13 +33,13 @@ public class AuthenticationCredentials {
}
public AuthenticationCredentials(String authenticationToken) {
this.salt = String.valueOf(Math.abs(new SecureRandom().nextInt()));
this.salt = String.valueOf(Util.ensureNonNegativeInt(new SecureRandom().nextInt()));
this.hashedAuthenticationToken = getV2HashedValue(salt, authenticationToken);
}
@VisibleForTesting
public AuthenticationCredentials v1ForTesting(String authenticationToken) {
String salt = String.valueOf(Math.abs(new SecureRandom().nextInt()));
String salt = String.valueOf(Util.ensureNonNegativeInt(new SecureRandom().nextInt()));
return new AuthenticationCredentials(getV1HashedValue(salt, authenticationToken), salt);
}

View File

@@ -30,6 +30,9 @@ public class BaseAccountAuthenticator {
private static final String AUTHENTICATION_SUCCEEDED_TAG_NAME = "succeeded";
private static final String AUTHENTICATION_FAILURE_REASON_TAG_NAME = "reason";
private static final String AUTHENTICATION_ENABLED_REQUIRED_TAG_NAME = "enabledRequired";
private static final String AUTHENTICATION_HAS_STORY_CAPABILITY = "hasStoryCapability";
private static final String STORY_ADOPTION_COUNTER_NAME = name(BaseAccountAuthenticator.class, "storyAdoption");
private static final String DAYS_SINCE_LAST_SEEN_DISTRIBUTION_NAME = name(BaseAccountAuthenticator.class, "daysSinceLastSeen");
private static final String IS_PRIMARY_DEVICE_TAG = "isPrimary";
@@ -67,6 +70,7 @@ public class BaseAccountAuthenticator {
public Optional<AuthenticatedAccount> authenticate(BasicCredentials basicCredentials, boolean enabledRequired) {
boolean succeeded = false;
String failureReason = null;
boolean hasStoryCapability = false;
try {
final UUID accountUuid;
@@ -85,6 +89,8 @@ public class BaseAccountAuthenticator {
return Optional.empty();
}
hasStoryCapability = account.map(Account::isStoriesSupported).orElse(false);
Optional<Device> device = account.get().getDevice(deviceId);
if (device.isEmpty()) {
@@ -132,14 +138,26 @@ public class BaseAccountAuthenticator {
}
Metrics.counter(AUTHENTICATION_COUNTER_NAME, tags).increment();
Tags storyTags = Tags.of(AUTHENTICATION_HAS_STORY_CAPABILITY, String.valueOf(hasStoryCapability));
Metrics.counter(STORY_ADOPTION_COUNTER_NAME, storyTags).increment();
}
}
@VisibleForTesting
public Account updateLastSeen(Account account, Device device) {
final long lastSeenOffsetSeconds = Math.abs(account.getUuid().getLeastSignificantBits()) % ChronoUnit.DAYS.getDuration().toSeconds();
// compute a non-negative integer between 0 and 86400.
long n = Util.ensureNonNegativeLong(account.getUuid().getLeastSignificantBits());
final long lastSeenOffsetSeconds = n % ChronoUnit.DAYS.getDuration().toSeconds();
// produce a truncated timestamp which is either today at UTC midnight
// or yesterday at UTC midnight, based on per-user randomized offset used.
final long todayInMillisWithOffset = Util.todayInMillisGivenOffsetFromNow(clock, Duration.ofSeconds(lastSeenOffsetSeconds).negated());
// only update the device's last seen time when it falls behind the truncated timestamp.
// this ensure a few things:
// (1) each account will only update last-seen at most once per day
// (2) these updates will occur throughout the day rather than all occurring at UTC midnight.
if (device.getLastSeen() < todayInMillisWithOffset) {
Metrics.summary(DAYS_SINCE_LAST_SEEN_DISTRIBUTION_NAME, IS_PRIMARY_DEVICE_TAG, String.valueOf(device.isMaster()))
.record(Duration.ofMillis(todayInMillisWithOffset - device.getLastSeen()).toDays());

View File

@@ -9,9 +9,9 @@ import com.google.common.annotations.VisibleForTesting;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.Clock;
import java.util.HexFormat;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Hex;
import org.whispersystems.textsecuregcm.util.Util;
public class ExternalServiceCredentialGenerator {
@@ -20,33 +20,50 @@ public class ExternalServiceCredentialGenerator {
private final byte[] userIdKey;
private final boolean usernameDerivation;
private final boolean prependUsername;
private final boolean truncateKey;
private final Clock clock;
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey) {
this(key, userIdKey, true, true);
this(key, userIdKey, true, true, true);
}
public ExternalServiceCredentialGenerator(byte[] key, boolean prependUsername) {
this(key, new byte[0], false, prependUsername);
this(key, prependUsername, true);
}
public ExternalServiceCredentialGenerator(byte[] key, boolean prependUsername, boolean truncateKey) {
this(key, new byte[0], false, prependUsername, truncateKey);
}
@VisibleForTesting
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation) {
this(key, userIdKey, usernameDerivation, true);
this(key, userIdKey, usernameDerivation, true, true);
}
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation,
boolean prependUsername) {
this(key, userIdKey, usernameDerivation, prependUsername, Clock.systemUTC());
this(key, userIdKey, usernameDerivation, prependUsername, true, Clock.systemUTC());
}
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation,
boolean prependUsername, boolean truncateKey) {
this(key, userIdKey, usernameDerivation, prependUsername, truncateKey, Clock.systemUTC());
}
@VisibleForTesting
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation,
boolean prependUsername, Clock clock) {
this(key, userIdKey, usernameDerivation, prependUsername, true, clock);
}
@VisibleForTesting
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation,
boolean prependUsername, boolean truncateKey, Clock clock) {
this.key = key;
this.userIdKey = userIdKey;
this.usernameDerivation = usernameDerivation;
this.prependUsername = prependUsername;
this.truncateKey = truncateKey;
this.clock = clock;
}
@@ -55,14 +72,17 @@ public class ExternalServiceCredentialGenerator {
String username = getUserId(identity, mac, usernameDerivation);
long currentTimeSeconds = clock.millis() / 1000;
String prefix = username + ":" + currentTimeSeconds;
String output = Hex.encodeHexString(Util.truncate(getHmac(key, prefix.getBytes(), mac), 10));
byte[] prefixMac = getHmac(key, prefix.getBytes(), mac);
final HexFormat hex = HexFormat.of();
String output = hex.formatHex(truncateKey ? Util.truncate(prefixMac, 10) : prefixMac);
String token = (prependUsername ? prefix : currentTimeSeconds) + ":" + output;
return new ExternalServiceCredentials(username, token);
}
private String getUserId(String number, Mac mac, boolean usernameDerivation) {
if (usernameDerivation) return Hex.encodeHexString(Util.truncate(getHmac(userIdKey, number.getBytes(), mac), 10));
final HexFormat hex = HexFormat.of();
if (usernameDerivation) return hex.formatHex(Util.truncate(getHmac(userIdKey, number.getBytes(), mac), 10));
else return number;
}

View File

@@ -21,6 +21,20 @@ public class StoredRegistrationLock {
private final long lastSeen;
/**
* @return milliseconds since the last time the account was seen.
*/
private long timeSinceLastSeen() {
return System.currentTimeMillis() - lastSeen;
}
/**
* @return true if the registration lock and salt are both set.
*/
private boolean hasLockAndSalt() {
return registrationLock.isPresent() && registrationLockSalt.isPresent();
}
public StoredRegistrationLock(Optional<String> registrationLock, Optional<String> registrationLockSalt, long lastSeen) {
this.registrationLock = registrationLock;
this.registrationLockSalt = registrationLockSalt;
@@ -28,24 +42,22 @@ public class StoredRegistrationLock {
}
public boolean requiresClientRegistrationLock() {
return registrationLock.isPresent() && registrationLockSalt.isPresent() && System.currentTimeMillis() - lastSeen < TimeUnit.DAYS.toMillis(7);
boolean hasTimeRemaining = getTimeRemaining() >= 0;
return hasLockAndSalt() && hasTimeRemaining;
}
public boolean needsFailureCredentials() {
return registrationLock.isPresent() && registrationLockSalt.isPresent();
return hasLockAndSalt();
}
public long getTimeRemaining() {
return TimeUnit.DAYS.toMillis(7) - (System.currentTimeMillis() - lastSeen);
return TimeUnit.DAYS.toMillis(7) - timeSinceLastSeen();
}
public boolean verify(@Nullable String clientRegistrationLock) {
if (Util.isEmpty(clientRegistrationLock)) {
return false;
}
if (registrationLock.isPresent() && registrationLockSalt.isPresent() && !Util.isEmpty(clientRegistrationLock)) {
return new AuthenticationCredentials(registrationLock.get(), registrationLockSalt.get()).verify(clientRegistrationLock);
if (hasLockAndSalt() && Util.nonEmpty(clientRegistrationLock)) {
AuthenticationCredentials credentials = new AuthenticationCredentials(registrationLock.get(), registrationLockSalt.get());
return credentials.verify(clientRegistrationLock);
} else {
return false;
}

View File

@@ -10,6 +10,7 @@ import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfigurati
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicTurnConfiguration;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.Pair;
import org.whispersystems.textsecuregcm.util.Util;
import org.whispersystems.textsecuregcm.util.WeightedRandomSelect;
import javax.crypto.Mac;
@@ -36,7 +37,7 @@ public class TurnTokenGenerator {
List<String> urls = urls(e164);
Mac mac = Mac.getInstance("HmacSHA1");
long validUntilSeconds = (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1)) / 1000;
long user = Math.abs(new SecureRandom().nextInt());
long user = Util.ensureNonNegativeInt(new SecureRandom().nextInt());
String userTime = validUntilSeconds + ":" + user;
mac.init(new SecretKeySpec(key, "HmacSHA1"));

View File

@@ -0,0 +1,27 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.captcha;
/**
* A captcha assessment
*
* @param valid whether the captcha was passed
* @param score string representation of the risk level
*/
public record AssessmentResult(boolean valid, String score) {
public static AssessmentResult invalid() {
return new AssessmentResult(false, "");
}
/**
* Map a captcha score in [0.0, 1.0] to a low cardinality discrete space in [0, 100] suitable for use in metrics
*/
static String scoreString(final float score) {
final int x = Math.round(score * 10); // [0, 10]
return Integer.toString(x * 10); // [0, 100] in increments of 10
}
}

View File

@@ -0,0 +1,76 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.captcha;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.ws.rs.BadRequestException;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
public class CaptchaChecker {
private static final String ASSESSMENTS_COUNTER_NAME = name(RecaptchaClient.class, "assessments");
@VisibleForTesting
static final String SEPARATOR = ".";
private final Map<String, CaptchaClient> captchaClientMap;
public CaptchaChecker(final List<CaptchaClient> captchaClients) {
this.captchaClientMap = captchaClients.stream()
.collect(Collectors.toMap(CaptchaClient::scheme, Function.identity()));
}
/**
* Check if a solved captcha should be accepted
* <p>
*
* @param input expected to contain a prefix indicating the captcha scheme, sitekey, token, and action. The expected
* format is {@code version-prefix.sitekey.[action.]token}
* @param ip IP of the solver
* @return An {@link AssessmentResult} indicating whether the solution should be accepted, and a score that can be
* used for metrics
* @throws IOException if there is an error validating the captcha with the underlying service
* @throws BadRequestException if input is not in the expected format
*/
public AssessmentResult verify(final String input, final String ip) throws IOException {
/*
* For action to be optional, there is a strong assumption that the token will never contain a {@value SEPARATOR}.
* Observation suggests {@code token} is base-64 encoded. In practice, an action should always be present, but we
* dont need to be strict.
*/
final String[] parts = input.split("\\" + SEPARATOR, 4);
// we allow missing actions, if we're missing 1 part, assume it's the action
if (parts.length < 3) {
throw new BadRequestException("too few parts");
}
int idx = 0;
final String prefix = parts[idx++];
final String siteKey = parts[idx++];
final String action = parts.length == 3 ? null : parts[idx++];
final String token = parts[idx];
final CaptchaClient client = this.captchaClientMap.get(prefix);
if (client == null) {
throw new BadRequestException("invalid captcha scheme");
}
final AssessmentResult result = client.verify(siteKey, action, token, ip);
Metrics.counter(ASSESSMENTS_COUNTER_NAME,
"action", String.valueOf(action),
"valid", String.valueOf(result.valid()),
"score", result.score(),
"provider", prefix)
.increment();
return result;
}
}

View File

@@ -0,0 +1,33 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.captcha;
import javax.annotation.Nullable;
import java.io.IOException;
public interface CaptchaClient {
/**
* @return the identifying captcha scheme that this CaptchaClient handles
*/
String scheme();
/**
* Verify a provided captcha solution
*
* @param siteKey identifying string for the captcha service
* @param action an optional action indicating the purpose of the captcha
* @param token the captcha solution that will be verified
* @param ip the ip of the captcha solve
* @return An {@link AssessmentResult} indicating whether the solution should be accepted
* @throws IOException if the underlying captcha provider returns an error
*/
AssessmentResult verify(
final String siteKey,
final @Nullable String action,
final String token,
final String ip) throws IOException;
}

View File

@@ -0,0 +1,114 @@
/*
* Copyright 2021-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.captcha;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import javax.annotation.Nullable;
import javax.ws.rs.core.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicCaptchaConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.SystemMapper;
public class HCaptchaClient implements CaptchaClient {
private static final Logger logger = LoggerFactory.getLogger(HCaptchaClient.class);
private static final String PREFIX = "signal-hcaptcha";
private static final String ASSESSMENT_REASON_COUNTER_NAME = name(HCaptchaClient.class, "assessmentReason");
private static final String INVALID_REASON_COUNTER_NAME = name(HCaptchaClient.class, "invalidReason");
private final String apiKey;
private final HttpClient client;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
public HCaptchaClient(
final String apiKey,
final HttpClient client,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
this.apiKey = apiKey;
this.client = client;
this.dynamicConfigurationManager = dynamicConfigurationManager;
}
@Override
public String scheme() {
return PREFIX;
}
@Override
public AssessmentResult verify(final String siteKey, final @Nullable String action, final String token,
final String ip)
throws IOException {
final DynamicCaptchaConfiguration config = dynamicConfigurationManager.getConfiguration().getCaptchaConfiguration();
if (!config.isAllowHCaptcha()) {
logger.warn("Received request to verify an hCaptcha, but hCaptcha is not enabled");
return AssessmentResult.invalid();
}
final String body = String.format("response=%s&secret=%s&remoteip=%s",
URLEncoder.encode(token, StandardCharsets.UTF_8),
URLEncoder.encode(this.apiKey, StandardCharsets.UTF_8),
ip);
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("https://hcaptcha.com/siteverify"))
.header("Content-Type", "application/x-www-form-urlencoded")
.POST(HttpRequest.BodyPublishers.ofString(body))
.build();
HttpResponse<String> response;
try {
response = this.client.send(request, HttpResponse.BodyHandlers.ofString());
} catch (InterruptedException e) {
throw new IOException(e);
}
if (response.statusCode() != Response.Status.OK.getStatusCode()) {
logger.warn("failure submitting token to hCaptcha (code={}): {}", response.statusCode(), response);
throw new IOException("hCaptcha http failure : " + response.statusCode());
}
final HCaptchaResponse hCaptchaResponse = SystemMapper.getMapper()
.readValue(response.body(), HCaptchaResponse.class);
logger.debug("received hCaptcha response: {}", hCaptchaResponse);
if (!hCaptchaResponse.success) {
for (String errorCode : hCaptchaResponse.errorCodes) {
Metrics.counter(INVALID_REASON_COUNTER_NAME,
"action", String.valueOf(action),
"reason", errorCode).increment();
}
return AssessmentResult.invalid();
}
// hcaptcha uses the inverse scheme of recaptcha (for hcaptcha, a low score is less risky)
float score = 1.0f - hCaptchaResponse.score;
if (score < 0.0f || score > 1.0f) {
logger.error("Invalid score {} from hcaptcha response {}", hCaptchaResponse.score, hCaptchaResponse);
return AssessmentResult.invalid();
}
final String scoreString = AssessmentResult.scoreString(score);
for (String reason : hCaptchaResponse.scoreReasons) {
Metrics.counter(ASSESSMENT_REASON_COUNTER_NAME,
"action", String.valueOf(action),
"reason", reason,
"score", scoreString).increment();
}
return new AssessmentResult(score >= config.getScoreFloor().floatValue(), scoreString);
}
}

View File

@@ -0,0 +1,57 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.captcha;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
/**
* Verify response returned by hcaptcha
* <p>
* see <a href="https://docs.hcaptcha.com/#verify-the-user-response-server-side">...</a>
*/
public class HCaptchaResponse {
@JsonProperty
boolean success;
@JsonProperty(value = "challenge-ts")
Duration challengeTs;
@JsonProperty
String hostname;
@JsonProperty
boolean credit;
@JsonProperty(value = "error-codes")
List<String> errorCodes = Collections.emptyList();
@JsonProperty
float score;
@JsonProperty(value = "score-reasons")
List<String> scoreReasons = Collections.emptyList();
public HCaptchaResponse() {
}
@Override
public String toString() {
return "HCaptchaResponse{" +
"success=" + success +
", challengeTs=" + challengeTs +
", hostname='" + hostname + '\'' +
", credit=" + credit +
", errorCodes=" + errorCodes +
", score=" + score +
", scoreReasons=" + scoreReasons +
'}';
}
}

View File

@@ -3,15 +3,15 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.recaptcha;
package org.whispersystems.textsecuregcm.captcha;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.api.gax.rpc.ApiException;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.recaptchaenterprise.v1.RecaptchaEnterpriseServiceClient;
import com.google.cloud.recaptchaenterprise.v1.RecaptchaEnterpriseServiceSettings;
import com.google.common.annotations.VisibleForTesting;
import com.google.recaptchaenterprise.v1.Assessment;
import com.google.recaptchaenterprise.v1.Event;
import com.google.recaptchaenterprise.v1.RiskAnalysis;
@@ -21,22 +21,18 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.ws.rs.BadRequestException;
import org.apache.commons.lang3.StringUtils;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicCaptchaConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
public class RecaptchaClient {
public class RecaptchaClient implements CaptchaClient {
private static final Logger log = LoggerFactory.getLogger(RecaptchaClient.class);
@VisibleForTesting
static final String SEPARATOR = ".";
@VisibleForTesting
static final String V2_PREFIX = "signal-recaptcha-v2" + RecaptchaClient.SEPARATOR;
private static final String ASSESSMENTS_COUNTER_NAME = name(RecaptchaClient.class, "assessments");
private static final String V2_PREFIX = "signal-recaptcha-v2";
private static final String INVALID_REASON_COUNTER_NAME = name(RecaptchaClient.class, "invalidReason");
private static final String ASSESSMENT_REASON_COUNTER_NAME = name(RecaptchaClient.class, "assessmentReason");
@@ -61,57 +57,20 @@ public class RecaptchaClient {
}
}
/**
* Parses the sitekey, token, and action (if any) from {@code input}. The expected input format is: {@code [version
* prefix.]sitekey.[action.]token}.
* <p>
* For action to be optional, there is a strong assumption that the token will never contain a {@value SEPARATOR}.
* Observation suggests {@code token} is base-64 encoded. In practice, an action should always be present, but we
* dont need to be strict.
*/
static String[] parseInputToken(final String input) {
String[] parts = StringUtils.removeStart(input, V2_PREFIX).split("\\" + SEPARATOR, 3);
if (parts.length == 1) {
throw new BadRequestException("too few parts");
}
if (parts.length == 2) {
// we got some parts, assume it is action that is missing
return new String[]{parts[0], null, parts[1]};
}
return parts;
@Override
public String scheme() {
return V2_PREFIX;
}
/**
* A captcha assessment
*
* @param valid whether the captcha was passed
* @param score string representation of the risk level
*/
public record AssessmentResult(boolean valid, String score) {
public static AssessmentResult invalid() {
return new AssessmentResult(false, "");
@Override
public org.whispersystems.textsecuregcm.captcha.AssessmentResult verify(final String sitekey,
final @Nullable String expectedAction,
final String token, final String ip) throws IOException {
final DynamicCaptchaConfiguration config = dynamicConfigurationManager.getConfiguration().getCaptchaConfiguration();
if (!config.isAllowRecaptcha()) {
log.warn("Received request to verify a recaptcha, but recaptcha is not enabled");
return AssessmentResult.invalid();
}
}
/*
* recaptcha enterprise scores are from [0.0, 1.0] in increments of .1
* map to [0, 100] for easier interpretation
*/
@VisibleForTesting
static String scoreString(final float score) {
return Integer.toString((int) (score * 100));
}
public AssessmentResult verify(final String input, final String ip) {
final String[] parts = parseInputToken(input);
final String sitekey = parts[0];
final String expectedAction = parts[1];
final String token = parts[2];
Event.Builder eventBuilder = Event.newBuilder()
.setSiteKey(sitekey)
@@ -123,32 +82,30 @@ public class RecaptchaClient {
}
final Event event = eventBuilder.build();
final Assessment assessment = client.createAssessment(projectPath, Assessment.newBuilder().setEvent(event).build());
Metrics.counter(ASSESSMENTS_COUNTER_NAME,
"action", String.valueOf(expectedAction),
"valid", String.valueOf(assessment.getTokenProperties().getValid()))
.increment();
final Assessment assessment;
try {
assessment = client.createAssessment(projectPath, Assessment.newBuilder().setEvent(event).build());
} catch (ApiException e) {
throw new IOException(e);
}
if (assessment.getTokenProperties().getValid()) {
final float score = assessment.getRiskAnalysis().getScore();
log.debug("assessment for {} was valid, score: {}", expectedAction, score);
for (RiskAnalysis.ClassificationReason reason : assessment.getRiskAnalysis().getReasonsList()) {
Metrics.counter(ASSESSMENT_REASON_COUNTER_NAME,
"action", String.valueOf(expectedAction),
"score", scoreString(score),
"reason", reason.name())
"action", String.valueOf(expectedAction),
"score", AssessmentResult.scoreString(score),
"reason", reason.name())
.increment();
}
return new AssessmentResult(
score >=
dynamicConfigurationManager.getConfiguration().getCaptchaConfiguration().getScoreFloor().floatValue(),
scoreString(score));
score >= config.getScoreFloor().floatValue(),
AssessmentResult.scoreString(score));
} else {
Metrics.counter(INVALID_REASON_COUNTER_NAME,
"action", String.valueOf(expectedAction),
"reason", assessment.getTokenProperties().getInvalidReason().name())
"action", String.valueOf(expectedAction),
"reason", assessment.getTokenProperties().getInvalidReason().name())
.increment();
return AssessmentResult.invalid();
}

View File

@@ -0,0 +1,41 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import java.time.Duration;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
public class ArtServiceConfiguration {
@NotEmpty
@JsonProperty
private String userAuthenticationTokenSharedSecret;
@NotEmpty
@JsonProperty
private String userAuthenticationTokenUserIdSecret;
@JsonProperty
@NotNull
private Duration tokenExpiration = Duration.ofDays(1);
public byte[] getUserAuthenticationTokenSharedSecret() throws DecoderException {
return Hex.decodeHex(userAuthenticationTokenSharedSecret.toCharArray());
}
public byte[] getUserAuthenticationTokenUserIdSecret() throws DecoderException {
return Hex.decodeHex(userAuthenticationTokenUserIdSecret.toCharArray());
}
public Duration getTokenExpiration() {
return tokenExpiration;
}
}

View File

@@ -1,58 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.math.BigDecimal;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import javax.validation.Valid;
import javax.validation.constraints.DecimalMin;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import org.whispersystems.textsecuregcm.util.ExactlySize;
public class BoostConfiguration {
private final long level;
private final Duration expiration;
private final Map<String, List<BigDecimal>> currencies;
private final String badge;
@JsonCreator
public BoostConfiguration(
@JsonProperty("level") long level,
@JsonProperty("expiration") Duration expiration,
@JsonProperty("currencies") Map<String, List<BigDecimal>> currencies,
@JsonProperty("badge") String badge) {
this.level = level;
this.expiration = expiration;
this.currencies = currencies;
this.badge = badge;
}
public long getLevel() {
return level;
}
@NotNull
public Duration getExpiration() {
return expiration;
}
@Valid
@NotNull
public Map<@NotEmpty String, @Valid @ExactlySize(6) List<@DecimalMin("0.01") @NotNull BigDecimal>> getCurrencies() {
return currencies;
}
@NotEmpty
public String getBadge() {
return badge;
}
}

View File

@@ -0,0 +1,44 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import java.util.Map;
import java.util.Set;
import javax.validation.Valid;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
/**
* @param merchantId the Braintree merchant ID
* @param publicKey the Braintree API public key
* @param privateKey the Braintree API private key
* @param environment the Braintree environment ("production" or "sandbox")
* @param supportedCurrencies the set of supported currencies
* @param graphqlUrl the Braintree GraphQL URl to use (this must match the environment)
* @param merchantAccounts merchant account within the merchant for processing individual currencies
* @param circuitBreaker configuration for the circuit breaker used by the GraphQL HTTP client
*/
public record BraintreeConfiguration(@NotBlank String merchantId,
@NotBlank String publicKey,
@NotBlank String privateKey,
@NotBlank String environment,
@NotEmpty Set<@NotBlank String> supportedCurrencies,
@NotBlank String graphqlUrl,
@NotEmpty Map<String, String> merchantAccounts,
@NotNull
@Valid
CircuitBreakerConfiguration circuitBreaker) {
public BraintreeConfiguration {
if (circuitBreaker == null) {
// Its a little counter-intuitive, but this compact constructor allows a default value
// to be used when one isnt specified (e.g. in YAML), allowing the field to still be
// validated as @NotNull
circuitBreaker = new CircuitBreakerConfiguration();
}
}
}

View File

@@ -5,7 +5,9 @@
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotEmpty;
import java.util.List;
public class DirectoryServerConfiguration {
@@ -23,7 +25,7 @@ public class DirectoryServerConfiguration {
@NotEmpty
@JsonProperty
private String replicationCaCertificate;
private List<@NotBlank String> replicationCaCertificates;
public String getReplicationName() {
return replicationName;
@@ -37,8 +39,8 @@ public class DirectoryServerConfiguration {
return replicationPassword;
}
public String getReplicationCaCertificate() {
return replicationCaCertificate;
public List<String> getReplicationCaCertificates() {
return replicationCaCertificates;
}
}

View File

@@ -1,78 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import java.util.Set;
import javax.validation.Valid;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
public class DonationConfiguration {
private String uri;
private String description;
private Set<String> supportedCurrencies;
private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
private RetryConfiguration retry = new RetryConfiguration();
@JsonProperty
@NotEmpty
public String getUri() {
return uri;
}
@VisibleForTesting
public void setUri(final String uri) {
this.uri = uri;
}
@JsonProperty
public String getDescription() {
return description;
}
@VisibleForTesting
public void setDescription(final String description) {
this.description = description;
}
@JsonProperty
@NotEmpty
public Set<String> getSupportedCurrencies() {
return supportedCurrencies;
}
@VisibleForTesting
public void setSupportedCurrencies(final Set<String> supportedCurrencies) {
this.supportedCurrencies = supportedCurrencies;
}
@JsonProperty
@NotNull
@Valid
public CircuitBreakerConfiguration getCircuitBreaker() {
return circuitBreaker;
}
@VisibleForTesting
public void setCircuitBreaker(final CircuitBreakerConfiguration circuitBreaker) {
this.circuitBreaker = circuitBreaker;
}
@JsonProperty
@NotNull
@Valid
public RetryConfiguration getRetry() {
return retry;
}
@VisibleForTesting
public void setRetry(final RetryConfiguration retry) {
this.retry = retry;
}
}

View File

@@ -1,21 +0,0 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import java.math.BigDecimal;
import java.time.Duration;
import java.util.Map;
import javax.validation.Valid;
import javax.validation.constraints.DecimalMin;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
public record GiftConfiguration(
long level,
@NotNull Duration expiration,
@Valid @NotNull Map<@NotEmpty String, @DecimalMin("0.01") @NotNull BigDecimal> currencies,
@NotEmpty String badge) {
}

View File

@@ -0,0 +1,11 @@
/*
* Copyright 2021-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import javax.validation.constraints.NotBlank;
public record HCaptchaConfiguration(@NotBlank String apiKey) {
}

View File

@@ -0,0 +1,31 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import java.time.Duration;
import java.util.Map;
import javax.validation.Valid;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Positive;
/**
* @param boost configuration for individual donations
* @param gift configuration for gift donations
* @param currencies map of lower-cased ISO 3 currency codes and the suggested donation amounts in that currency
*/
public record OneTimeDonationConfiguration(@Valid ExpiringLevelConfiguration boost,
@Valid ExpiringLevelConfiguration gift,
Map<String, @Valid OneTimeDonationCurrencyConfiguration> currencies) {
/**
* @param badge the numeric donation level ID
* @param level the badge ID associated with the level
* @param expiration the duration after which the level expires
*/
public record ExpiringLevelConfiguration(@NotEmpty String badge, @Positive long level, Duration expiration) {
}
}

View File

@@ -0,0 +1,30 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import java.math.BigDecimal;
import java.util.List;
import javax.validation.Valid;
import javax.validation.constraints.DecimalMin;
import javax.validation.constraints.NotNull;
import org.whispersystems.textsecuregcm.util.ExactlySize;
/**
* One-time donation configuration for a given currency
*
* @param minimum the minimum amount permitted to be charged in this currency
* @param gift the suggested gift donation amount
* @param boosts the list of suggested one-time donation amounts
*/
public record OneTimeDonationCurrencyConfiguration(
@NotNull @DecimalMin("0.01") BigDecimal minimum,
@NotNull @DecimalMin("0.01") BigDecimal gift,
@Valid
@ExactlySize(6)
@NotNull
List<@NotNull @DecimalMin("0.01") BigDecimal> boosts) {
}

View File

@@ -9,8 +9,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotEmpty;
import java.util.List;
import java.util.Map;
public class PaymentsServiceConfiguration {
@@ -18,6 +20,14 @@ public class PaymentsServiceConfiguration {
@JsonProperty
private String userAuthenticationTokenSharedSecret;
@NotBlank
@JsonProperty
private String coinMarketCapApiKey;
@JsonProperty
@NotEmpty
private Map<@NotBlank String, Integer> coinMarketCapCurrencyIds;
@NotEmpty
@JsonProperty
private String fixerApiKey;
@@ -30,6 +40,14 @@ public class PaymentsServiceConfiguration {
return Hex.decodeHex(userAuthenticationTokenSharedSecret.toCharArray());
}
public String getCoinMarketCapApiKey() {
return coinMarketCapApiKey;
}
public Map<String, Integer> getCoinMarketCapCurrencyIds() {
return coinMarketCapCurrencyIds;
}
public String getFixerApiKey() {
return fixerApiKey;
}

View File

@@ -56,6 +56,9 @@ public class RateLimitsConfiguration {
@JsonProperty
private RateLimitConfiguration stickerPack = new RateLimitConfiguration(50, 20 / (24.0 * 60.0));
@JsonProperty
private RateLimitConfiguration artPack = new RateLimitConfiguration(50, 20 / (24.0 * 60.0));
@JsonProperty
private RateLimitConfiguration usernameLookup = new RateLimitConfiguration(100, 100 / (24.0 * 60.0));
@@ -135,6 +138,10 @@ public class RateLimitsConfiguration {
return stickerPack;
}
public RateLimitConfiguration getArtPack() {
return artPack;
}
public RateLimitConfiguration getUsernameLookup() {
return usernameLookup;
}

View File

@@ -13,6 +13,7 @@ import javax.validation.constraints.NotNull;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import java.util.List;
public class SecureBackupServiceConfiguration {
@@ -24,9 +25,9 @@ public class SecureBackupServiceConfiguration {
@JsonProperty
private String uri;
@NotBlank
@NotEmpty
@JsonProperty
private String backupCaCertificate;
private List<@NotBlank String> backupCaCertificates;
@NotNull
@Valid
@@ -52,12 +53,12 @@ public class SecureBackupServiceConfiguration {
}
@VisibleForTesting
public void setBackupCaCertificate(final String backupCaCertificate) {
this.backupCaCertificate = backupCaCertificate;
public void setBackupCaCertificates(final List<String> backupCaCertificates) {
this.backupCaCertificates = backupCaCertificates;
}
public String getBackupCaCertificate() {
return backupCaCertificate;
public List<String> getBackupCaCertificates() {
return backupCaCertificates;
}
public CircuitBreakerConfiguration getCircuitBreakerConfiguration() {

View File

@@ -13,6 +13,7 @@ import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import java.util.List;
public class SecureStorageServiceConfiguration {
@@ -24,9 +25,9 @@ public class SecureStorageServiceConfiguration {
@JsonProperty
private String uri;
@NotBlank
@NotEmpty
@JsonProperty
private String storageCaCertificate;
private List<@NotBlank String> storageCaCertificates;
@NotNull
@Valid
@@ -52,12 +53,12 @@ public class SecureStorageServiceConfiguration {
}
@VisibleForTesting
public void setStorageCaCertificate(final String certificatePem) {
this.storageCaCertificate = certificatePem;
public void setStorageCaCertificates(final List<String> certificatePem) {
this.storageCaCertificates = certificatePem;
}
public String getStorageCaCertificate() {
return storageCaCertificate;
public List<String> getStorageCaCertificates() {
return storageCaCertificates;
}
public CircuitBreakerConfiguration getCircuitBreakerConfiguration() {

View File

@@ -5,38 +5,13 @@
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Set;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotEmpty;
public class StripeConfiguration {
public record StripeConfiguration(@NotBlank String apiKey,
@NotEmpty byte[] idempotencyKeyGenerator,
@NotBlank String boostDescription,
@NotEmpty Set<@NotBlank String> supportedCurrencies) {
private final String apiKey;
private final byte[] idempotencyKeyGenerator;
private final String boostDescription;
@JsonCreator
public StripeConfiguration(
@JsonProperty("apiKey") final String apiKey,
@JsonProperty("idempotencyKeyGenerator") final byte[] idempotencyKeyGenerator,
@JsonProperty("boostDescription") final String boostDescription) {
this.apiKey = apiKey;
this.idempotencyKeyGenerator = idempotencyKeyGenerator;
this.boostDescription = boostDescription;
}
@NotEmpty
public String getApiKey() {
return apiKey;
}
@NotEmpty
public byte[] getIdempotencyKeyGenerator() {
return idempotencyKeyGenerator;
}
@NotEmpty
public String getBoostDescription() {
return boostDescription;
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2021 Signal Messenger, LLC
* Copyright 2021-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
@@ -15,16 +15,13 @@ import javax.validation.constraints.NotNull;
public class SubscriptionLevelConfiguration {
private final String badge;
private final String product;
private final Map<String, SubscriptionPriceConfiguration> prices;
@JsonCreator
public SubscriptionLevelConfiguration(
@JsonProperty("badge") @NotEmpty String badge,
@JsonProperty("product") @NotEmpty String product,
@JsonProperty("prices") @Valid Map<@NotEmpty String, @NotNull @Valid SubscriptionPriceConfiguration> prices) {
this.badge = badge;
this.product = product;
this.prices = prices;
}
@@ -32,10 +29,6 @@ public class SubscriptionLevelConfiguration {
return badge;
}
public String getProduct() {
return product;
}
public Map<String, SubscriptionPriceConfiguration> getPrices() {
return prices;
}

View File

@@ -1,159 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.google.common.annotations.VisibleForTesting;
import java.util.Collections;
import java.util.Map;
import javax.validation.Valid;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
public class TwilioConfiguration {
@NotEmpty
private String accountId;
@NotEmpty
private String accountToken;
@NotEmpty
private String localDomain;
@NotEmpty
private String messagingServiceSid;
@NotEmpty
private String nanpaMessagingServiceSid;
@NotEmpty
private String verifyServiceSid;
@NotNull
@Valid
private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
@NotNull
@Valid
private RetryConfiguration retry = new RetryConfiguration();
@Valid
private TwilioVerificationTextConfiguration defaultClientVerificationTexts;
@Valid
private Map<String,TwilioVerificationTextConfiguration> regionalClientVerificationTexts = Collections.emptyMap();
@NotEmpty
private String androidAppHash;
@NotEmpty
private String verifyServiceFriendlyName;
public String getAccountId() {
return accountId;
}
@VisibleForTesting
public void setAccountId(String accountId) {
this.accountId = accountId;
}
public String getAccountToken() {
return accountToken;
}
@VisibleForTesting
public void setAccountToken(String accountToken) {
this.accountToken = accountToken;
}
public String getLocalDomain() {
return localDomain;
}
@VisibleForTesting
public void setLocalDomain(String localDomain) {
this.localDomain = localDomain;
}
public String getMessagingServiceSid() {
return messagingServiceSid;
}
@VisibleForTesting
public void setMessagingServiceSid(String messagingServiceSid) {
this.messagingServiceSid = messagingServiceSid;
}
public String getNanpaMessagingServiceSid() {
return nanpaMessagingServiceSid;
}
@VisibleForTesting
public void setNanpaMessagingServiceSid(String nanpaMessagingServiceSid) {
this.nanpaMessagingServiceSid = nanpaMessagingServiceSid;
}
public String getVerifyServiceSid() {
return verifyServiceSid;
}
@VisibleForTesting
public void setVerifyServiceSid(String verifyServiceSid) {
this.verifyServiceSid = verifyServiceSid;
}
public CircuitBreakerConfiguration getCircuitBreaker() {
return circuitBreaker;
}
@VisibleForTesting
public void setCircuitBreaker(CircuitBreakerConfiguration circuitBreaker) {
this.circuitBreaker = circuitBreaker;
}
public RetryConfiguration getRetry() {
return retry;
}
@VisibleForTesting
public void setRetry(RetryConfiguration retry) {
this.retry = retry;
}
public TwilioVerificationTextConfiguration getDefaultClientVerificationTexts() {
return defaultClientVerificationTexts;
}
@VisibleForTesting
public void setDefaultClientVerificationTexts(TwilioVerificationTextConfiguration defaultClientVerificationTexts) {
this.defaultClientVerificationTexts = defaultClientVerificationTexts;
}
public Map<String,TwilioVerificationTextConfiguration> getRegionalClientVerificationTexts() {
return regionalClientVerificationTexts;
}
@VisibleForTesting
public void setRegionalClientVerificationTexts(final Map<String,TwilioVerificationTextConfiguration> regionalClientVerificationTexts) {
this.regionalClientVerificationTexts = regionalClientVerificationTexts;
}
public String getAndroidAppHash() {
return androidAppHash;
}
public void setAndroidAppHash(String androidAppHash) {
this.androidAppHash = androidAppHash;
}
public void setVerifyServiceFriendlyName(String serviceFriendlyName) {
this.verifyServiceFriendlyName = serviceFriendlyName;
}
public String getVerifyServiceFriendlyName() {
return verifyServiceFriendlyName;
}
}

View File

@@ -1,36 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.google.common.annotations.VisibleForTesting;
import javax.validation.constraints.NotEmpty;
public class TwilioCountrySenderIdConfiguration {
@NotEmpty
private String countryCode;
@NotEmpty
private String senderId;
public String getCountryCode() {
return countryCode;
}
@VisibleForTesting
public void setCountryCode(String countryCode) {
this.countryCode = countryCode;
}
public String getSenderId() {
return senderId;
}
@VisibleForTesting
public void setSenderId(String senderId) {
this.senderId = senderId;
}
}

View File

@@ -1,67 +0,0 @@
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.constraints.NotEmpty;
public class TwilioVerificationTextConfiguration {
@JsonProperty
@NotEmpty
private String ios;
@JsonProperty
@NotEmpty
private String androidNg;
@JsonProperty
@NotEmpty
private String android202001;
@JsonProperty
@NotEmpty
private String android202103;
@JsonProperty
@NotEmpty
private String generic;
public String getIosText() {
return ios;
}
public void setIosText(String ios) {
this.ios = ios;
}
public String getAndroidNgText() {
return androidNg;
}
public void setAndroidNgText(final String androidNg) {
this.androidNg = androidNg;
}
public String getAndroid202001Text() {
return android202001;
}
public void setAndroid202001Text(final String android202001) {
this.android202001 = android202001;
}
public String getAndroid202103Text() {
return android202103;
}
public void setAndroid202103Text(final String android202103) {
this.android202103 = android202103;
}
public String getGenericText() {
return generic;
}
public void setGenericText(final String generic) {
this.generic = generic;
}
}

View File

@@ -1,3 +1,8 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration.dynamic;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -17,6 +22,12 @@ public class DynamicCaptchaConfiguration {
@NotNull
private BigDecimal scoreFloor;
@JsonProperty
private boolean allowHCaptcha = false;
@JsonProperty
private boolean allowRecaptcha = true;
@JsonProperty
@NotNull
private Set<String> signupCountryCodes = Collections.emptySet();
@@ -46,4 +57,22 @@ public class DynamicCaptchaConfiguration {
public Set<String> getSignupRegions() {
return signupRegions;
}
public boolean isAllowHCaptcha() {
return allowHCaptcha;
}
public boolean isAllowRecaptcha() {
return allowRecaptcha;
}
@VisibleForTesting
public void setAllowHCaptcha(final boolean allowHCaptcha) {
this.allowHCaptcha = allowHCaptcha;
}
@VisibleForTesting
public void setScoreFloor(final BigDecimal scoreFloor) {
this.scoreFloor = scoreFloor;
}
}

View File

@@ -29,10 +29,6 @@ public class DynamicConfiguration {
@Valid
private DynamicPaymentsConfiguration payments = new DynamicPaymentsConfiguration();
@JsonProperty
@Valid
private DynamicTwilioConfiguration twilio = new DynamicTwilioConfiguration();
@JsonProperty
@Valid
private DynamicCaptchaConfiguration captcha = new DynamicCaptchaConfiguration();
@@ -86,15 +82,6 @@ public class DynamicConfiguration {
return payments;
}
public DynamicTwilioConfiguration getTwilioConfiguration() {
return twilio;
}
@VisibleForTesting
public void setTwilioConfiguration(DynamicTwilioConfiguration twilioConfiguration) {
this.twilio = twilioConfiguration;
}
public DynamicCaptchaConfiguration getCaptchaConfiguration() {
return captcha;
}

View File

@@ -1,23 +0,0 @@
package org.whispersystems.textsecuregcm.configuration.dynamic;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import javax.validation.constraints.NotNull;
import java.util.Collections;
import java.util.List;
public class DynamicTwilioConfiguration {
@JsonProperty
@NotNull
private List<String> numbers = Collections.emptyList();
public List<String> getNumbers() {
return numbers;
}
@VisibleForTesting
public void setNumbers(List<String> numbers) {
this.numbers = numbers;
}
}

View File

@@ -11,23 +11,24 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.annotation.Timed;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HttpHeaders;
import com.google.i18n.phonenumbers.NumberParseException;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.google.i18n.phonenumbers.Phonenumber;
import io.dropwizard.auth.Auth;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import java.io.IOException;
import java.security.SecureRandom;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import javax.annotation.Nullable;
import javax.servlet.http.HttpServletRequest;
import javax.validation.Valid;
@@ -65,6 +66,8 @@ import org.whispersystems.textsecuregcm.auth.StoredRegistrationLock;
import org.whispersystems.textsecuregcm.auth.StoredVerificationCode;
import org.whispersystems.textsecuregcm.auth.TurnToken;
import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
import org.whispersystems.textsecuregcm.captcha.AssessmentResult;
import org.whispersystems.textsecuregcm.captcha.CaptchaChecker;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicCaptchaConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
@@ -83,19 +86,15 @@ import org.whispersystems.textsecuregcm.entities.ReserveUsernameResponse;
import org.whispersystems.textsecuregcm.entities.StaleDevices;
import org.whispersystems.textsecuregcm.entities.UsernameRequest;
import org.whispersystems.textsecuregcm.entities.UsernameResponse;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.limits.RateLimiter;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil;
import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
import org.whispersystems.textsecuregcm.push.PushNotification;
import org.whispersystems.textsecuregcm.push.PushNotificationManager;
import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.registration.ClientType;
import org.whispersystems.textsecuregcm.registration.MessageTransport;
import org.whispersystems.textsecuregcm.registration.RegistrationServiceClient;
import org.whispersystems.textsecuregcm.sms.SmsSender;
import org.whispersystems.textsecuregcm.sms.TwilioVerifyExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRules;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.ChangeNumberManager;
@@ -105,14 +104,13 @@ import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
import org.whispersystems.textsecuregcm.storage.UsernameNotAvailableException;
import org.whispersystems.textsecuregcm.storage.UsernameReservationNotFoundException;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.ForwardedIpUtil;
import org.whispersystems.textsecuregcm.util.HeaderUtils;
import org.whispersystems.textsecuregcm.util.Hex;
import org.whispersystems.textsecuregcm.util.ImpossiblePhoneNumberException;
import org.whispersystems.textsecuregcm.util.NonNormalizedPhoneNumberException;
import org.whispersystems.textsecuregcm.util.Optionals;
import org.whispersystems.textsecuregcm.util.UsernameGenerator;
import org.whispersystems.textsecuregcm.util.Util;
import org.whispersystems.textsecuregcm.util.VerificationCode;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Path("/v1/accounts")
@@ -120,8 +118,6 @@ public class AccountController {
private final Logger logger = LoggerFactory.getLogger(AccountController.class);
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Meter blockedHostMeter = metricRegistry.meter(name(AccountController.class, "blocked_host" ));
private final Meter countryFilterApplicable = metricRegistry.meter(name(AccountController.class, "country_filter_applicable"));
private final Meter countryFilteredHostMeter = metricRegistry.meter(name(AccountController.class, "country_limited_host" ));
private final Meter rateLimitedHostMeter = metricRegistry.meter(name(AccountController.class, "rate_limited_host" ));
private final Meter rateLimitedPrefixMeter = metricRegistry.meter(name(AccountController.class, "rate_limited_prefix" ));
@@ -133,12 +129,16 @@ public class AccountController {
private static final String CAPTCHA_ATTEMPT_COUNTER_NAME = name(AccountController.class, "captcha");
private static final String CHALLENGE_ISSUED_COUNTER_NAME = name(AccountController.class, "challengeIssued");
private static final String TWILIO_VERIFY_ERROR_COUNTER_NAME = name(AccountController.class, "twilioVerifyError");
private static final String TWILIO_VERIFY_UNDELIVERED_COUNTER_NAME = name(AccountController.class, "twilioUndelivered");
private static final DistributionSummary REREGISTRATION_IDLE_DAYS_DISTRIBUTION_NAME = DistributionSummary
.builder(name(AccountController.class, "reregistrationIdleDays"))
.publishPercentiles(0.75, 0.95, 0.99, 0.999)
.distributionStatisticExpiry(Duration.ofHours(2))
.register(Metrics.globalRegistry);
private static final String INVALID_ACCEPT_LANGUAGE_COUNTER_NAME = name(AccountController.class, "invalidAcceptLanguage");
private static final String NONSTANDARD_USERNAME_COUNTER_NAME = name(AccountController.class, "nonStandardUsername");
private static final String LOCKED_ACCOUNT_COUNTER_NAME = name(AccountController.class, "lockedAccount");
private static final String CHALLENGE_PRESENT_TAG_NAME = "present";
private static final String CHALLENGE_MATCH_TAG_NAME = "matches";
private static final String COUNTRY_CODE_TAG_NAME = "countryCode";
@@ -151,62 +151,77 @@ public class AccountController {
private static final String REGION_CODE_TAG_NAME = "regionCode";
private static final String VERIFICATION_TRANSPORT_TAG_NAME = "transport";
private static final String SCORE_TAG_NAME = "score";
private static final String LOCK_REASON_TAG_NAME = "lockReason";
private static final String ALREADY_LOCKED_TAG_NAME = "alreadyLocked";
private final StoredVerificationCodeManager pendingAccounts;
private final AccountsManager accounts;
private final AbusiveHostRules abusiveHostRules;
private final RateLimiters rateLimiters;
private final SmsSender smsSender;
private final RegistrationServiceClient registrationServiceClient;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
private final TurnTokenGenerator turnTokenGenerator;
private final Map<String, Integer> testDevices;
private final RecaptchaClient recaptchaClient;
private final CaptchaChecker captchaChecker;
private final PushNotificationManager pushNotificationManager;
private final ExternalServiceCredentialGenerator backupServiceCredentialGenerator;
private final TwilioVerifyExperimentEnrollmentManager verifyExperimentEnrollmentManager;
private final ExperimentEnrollmentManager experimentEnrollmentManager;
private final ChangeNumberManager changeNumberManager;
private final Clock clock;
@VisibleForTesting
static final String REGISTRATION_SERVICE_EXPERIMENT_NAME = "registration-service";
private final ClientPresenceManager clientPresenceManager;
@VisibleForTesting
static final Duration REGISTRATION_RPC_TIMEOUT = Duration.ofSeconds(15);
public AccountController(StoredVerificationCodeManager pendingAccounts,
AccountsManager accounts,
AbusiveHostRules abusiveHostRules,
RateLimiters rateLimiters,
SmsSender smsSenderFactory,
RegistrationServiceClient registrationServiceClient,
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
TurnTokenGenerator turnTokenGenerator,
Map<String, Integer> testDevices,
RecaptchaClient recaptchaClient,
PushNotificationManager pushNotificationManager,
TwilioVerifyExperimentEnrollmentManager verifyExperimentEnrollmentManager,
ChangeNumberManager changeNumberManager,
ExternalServiceCredentialGenerator backupServiceCredentialGenerator,
final ExperimentEnrollmentManager experimentEnrollmentManager)
{
this.pendingAccounts = pendingAccounts;
this.accounts = accounts;
this.abusiveHostRules = abusiveHostRules;
this.rateLimiters = rateLimiters;
this.smsSender = smsSenderFactory;
this.registrationServiceClient = registrationServiceClient;
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.testDevices = testDevices;
this.turnTokenGenerator = turnTokenGenerator;
this.recaptchaClient = recaptchaClient;
this.pushNotificationManager = pushNotificationManager;
this.verifyExperimentEnrollmentManager = verifyExperimentEnrollmentManager;
public AccountController(
StoredVerificationCodeManager pendingAccounts,
AccountsManager accounts,
RateLimiters rateLimiters,
RegistrationServiceClient registrationServiceClient,
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
TurnTokenGenerator turnTokenGenerator,
Map<String, Integer> testDevices,
CaptchaChecker captchaChecker,
PushNotificationManager pushNotificationManager,
ChangeNumberManager changeNumberManager,
ExternalServiceCredentialGenerator backupServiceCredentialGenerator,
ClientPresenceManager clientPresenceManager,
Clock clock
) {
this.pendingAccounts = pendingAccounts;
this.accounts = accounts;
this.rateLimiters = rateLimiters;
this.registrationServiceClient = registrationServiceClient;
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.testDevices = testDevices;
this.turnTokenGenerator = turnTokenGenerator;
this.captchaChecker = captchaChecker;
this.pushNotificationManager = pushNotificationManager;
this.backupServiceCredentialGenerator = backupServiceCredentialGenerator;
this.changeNumberManager = changeNumberManager;
this.experimentEnrollmentManager = experimentEnrollmentManager;
this.clientPresenceManager = clientPresenceManager;
this.clock = clock;
}
@VisibleForTesting
public AccountController(
StoredVerificationCodeManager pendingAccounts,
AccountsManager accounts,
RateLimiters rateLimiters,
RegistrationServiceClient registrationServiceClient,
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
TurnTokenGenerator turnTokenGenerator,
Map<String, Integer> testDevices,
CaptchaChecker captchaChecker,
PushNotificationManager pushNotificationManager,
ChangeNumberManager changeNumberManager,
ExternalServiceCredentialGenerator backupServiceCredentialGenerator
) {
this(pendingAccounts, accounts, rateLimiters,
registrationServiceClient, dynamicConfigurationManager, turnTokenGenerator, testDevices, captchaChecker,
pushNotificationManager, changeNumberManager,
backupServiceCredentialGenerator, null, Clock.systemUTC());
}
@Timed
@@ -229,7 +244,7 @@ public class AccountController {
String pushChallenge = generatePushChallenge();
StoredVerificationCode storedVerificationCode =
new StoredVerificationCode(null, System.currentTimeMillis(), pushChallenge, null, null);
new StoredVerificationCode(null, clock.millis(), pushChallenge, null, null);
pendingAccounts.store(number, storedVerificationCode);
pushNotificationManager.sendRegistrationChallengeNotification(pushToken, tokenType, storedVerificationCode.pushCode());
@@ -244,25 +259,26 @@ public class AccountController {
@Produces(MediaType.APPLICATION_JSON)
public Response createAccount(@PathParam("transport") String transport,
@PathParam("number") String number,
@HeaderParam("X-Forwarded-For") String forwardedFor,
@HeaderParam("User-Agent") String userAgent,
@HeaderParam("Accept-Language") Optional<String> acceptLanguage,
@HeaderParam(HttpHeaders.X_FORWARDED_FOR) String forwardedFor,
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent,
@HeaderParam(HttpHeaders.ACCEPT_LANGUAGE) Optional<String> acceptLanguage,
@QueryParam("client") Optional<String> client,
@QueryParam("captcha") Optional<String> captcha,
@QueryParam("challenge") Optional<String> pushChallenge)
throws RateLimitExceededException, ImpossiblePhoneNumberException, NonNormalizedPhoneNumberException {
throws RateLimitExceededException, ImpossiblePhoneNumberException, NonNormalizedPhoneNumberException, IOException {
Util.requireNormalizedNumber(number);
final String sourceHost = ForwardedIpUtil.getMostRecentProxy(forwardedFor).orElseThrow();
final String sourceHost = HeaderUtils.getMostRecentProxy(forwardedFor).orElseThrow();
final Optional<StoredVerificationCode> maybeStoredVerificationCode = pendingAccounts.getCodeForNumber(number);
final String countryCode = Util.getCountryCode(number);
final String region = Util.getRegion(number);
// if there's a captcha, assess it, otherwise check if we need a captcha
final Optional<RecaptchaClient.AssessmentResult> assessmentResult = captcha
.map(captchaToken -> recaptchaClient.verify(captchaToken, sourceHost));
final Optional<AssessmentResult> assessmentResult = captcha.isPresent()
? Optional.of(captchaChecker.verify(captcha.get(), sourceHost))
: Optional.empty();
assessmentResult.ifPresent(result ->
Metrics.counter(CAPTCHA_ATTEMPT_COUNTER_NAME, Tags.of(
@@ -304,127 +320,6 @@ public class AccountController {
default -> throw new WebApplicationException(Response.status(422).build());
}
if (experimentEnrollmentManager.isEnrolled(number, REGISTRATION_SERVICE_EXPERIMENT_NAME)) {
sendVerificationCodeViaRegistrationService(number,
maybeStoredVerificationCode,
acceptLanguage,
client,
transport);
} else {
sendVerificationCodeViaTwilioSender(number,
maybeStoredVerificationCode,
acceptLanguage,
userAgent,
client,
transport,
assessmentResult);
}
Metrics.counter(ACCOUNT_CREATE_COUNTER_NAME, Tags.of(
UserAgentTagUtil.getPlatformTag(userAgent),
Tag.of(COUNTRY_CODE_TAG_NAME, Util.getCountryCode(number)),
Tag.of(REGION_TAG_NAME, Util.getRegion(number)),
Tag.of(VERIFICATION_TRANSPORT_TAG_NAME, transport)))
.increment();
return Response.ok().build();
}
private void sendVerificationCodeViaTwilioSender(final String number,
final Optional<StoredVerificationCode> maybeStoredVerificationCode,
final Optional<String> acceptLanguage,
final String userAgent,
final Optional<String> client,
final String transport,
final Optional<RecaptchaClient.AssessmentResult> assessmentResult) {
final VerificationCode verificationCode = generateVerificationCode(number);
final StoredVerificationCode storedVerificationCode = new StoredVerificationCode(verificationCode.getVerificationCode(),
System.currentTimeMillis(),
maybeStoredVerificationCode.map(StoredVerificationCode::pushCode).orElse(null),
maybeStoredVerificationCode.map(StoredVerificationCode::twilioVerificationSid).orElse(null),
maybeStoredVerificationCode.map(StoredVerificationCode::sessionId).orElse(null));
pendingAccounts.store(number, storedVerificationCode);
List<Locale.LanguageRange> languageRanges;
try {
languageRanges = acceptLanguage.map(Locale.LanguageRange::parse).orElse(Collections.emptyList());
} catch (final IllegalArgumentException e) {
logger.debug("Could not get acceptable languages; Accept-Language: {}; User-Agent: {}",
acceptLanguage.orElse(""),
userAgent,
e);
Metrics.counter(INVALID_ACCEPT_LANGUAGE_COUNTER_NAME, Tags.of(UserAgentTagUtil.getPlatformTag(userAgent))).increment();
languageRanges = Collections.emptyList();
}
final boolean enrolledInVerifyExperiment = verifyExperimentEnrollmentManager.isEnrolled(client, number, languageRanges, transport);
final CompletableFuture<Optional<String>> sendVerificationWithTwilioVerifyFuture;
if (testDevices.containsKey(number)) {
// noop
sendVerificationWithTwilioVerifyFuture = CompletableFuture.completedFuture(Optional.empty());
} else if (transport.equals("sms")) {
if (enrolledInVerifyExperiment) {
sendVerificationWithTwilioVerifyFuture = smsSender.deliverSmsVerificationWithTwilioVerify(number, client, verificationCode.getVerificationCode(), languageRanges);
} else {
smsSender.deliverSmsVerification(number, client, verificationCode.getVerificationCodeDisplay());
sendVerificationWithTwilioVerifyFuture = CompletableFuture.completedFuture(Optional.empty());
}
} else if (transport.equals("voice")) {
if (enrolledInVerifyExperiment) {
sendVerificationWithTwilioVerifyFuture = smsSender.deliverVoxVerificationWithTwilioVerify(number, verificationCode.getVerificationCode(), languageRanges);
} else {
smsSender.deliverVoxVerification(number, verificationCode.getVerificationCode(), languageRanges);
sendVerificationWithTwilioVerifyFuture = CompletableFuture.completedFuture(Optional.empty());
}
} else {
sendVerificationWithTwilioVerifyFuture = CompletableFuture.completedFuture(Optional.empty());
}
sendVerificationWithTwilioVerifyFuture.whenComplete((maybeVerificationSid, throwable) -> {
if (throwable != null) {
Metrics.counter(TWILIO_VERIFY_ERROR_COUNTER_NAME).increment();
logger.warn("Error with Twilio Verify", throwable);
return;
}
if (enrolledInVerifyExperiment && maybeVerificationSid.isEmpty() && assessmentResult.isPresent()) {
final String countryCode = Util.getCountryCode(number);
final String region = Util.getRegion(number);
Metrics.counter(TWILIO_VERIFY_UNDELIVERED_COUNTER_NAME, Tags.of(
Tag.of(COUNTRY_CODE_TAG_NAME, countryCode),
Tag.of(REGION_TAG_NAME, region),
UserAgentTagUtil.getPlatformTag(userAgent),
Tag.of(SCORE_TAG_NAME, assessmentResult.get().score())))
.increment();
}
maybeVerificationSid.ifPresent(twilioVerificationSid -> {
StoredVerificationCode storedVerificationCodeWithVerificationSid = new StoredVerificationCode(
storedVerificationCode.code(),
storedVerificationCode.timestamp(),
storedVerificationCode.pushCode(),
twilioVerificationSid,
storedVerificationCode.sessionId());
pendingAccounts.store(number, storedVerificationCodeWithVerificationSid);
});
});
}
private void sendVerificationCodeViaRegistrationService(final String number,
final Optional<StoredVerificationCode> maybeStoredVerificationCode,
final Optional<String> acceptLanguage,
final Optional<String> client,
final String transport) {
final Phonenumber.PhoneNumber phoneNumber;
try {
@@ -455,12 +350,21 @@ public class AccountController {
messageTransport, clientType, acceptLanguage.orElse(null), REGISTRATION_RPC_TIMEOUT).join();
final StoredVerificationCode storedVerificationCode = new StoredVerificationCode(null,
System.currentTimeMillis(),
clock.millis(),
maybeStoredVerificationCode.map(StoredVerificationCode::pushCode).orElse(null),
null,
sessionId);
pendingAccounts.store(number, storedVerificationCode);
Metrics.counter(ACCOUNT_CREATE_COUNTER_NAME, Tags.of(
UserAgentTagUtil.getPlatformTag(userAgent),
Tag.of(COUNTRY_CODE_TAG_NAME, Util.getCountryCode(number)),
Tag.of(REGION_TAG_NAME, Util.getRegion(number)),
Tag.of(VERIFICATION_TRANSPORT_TAG_NAME, transport)))
.increment();
return Response.ok().build();
}
@Timed
@@ -469,9 +373,9 @@ public class AccountController {
@Produces(MediaType.APPLICATION_JSON)
@Path("/code/{verification_code}")
public AccountIdentityResponse verifyAccount(@PathParam("verification_code") String verificationCode,
@HeaderParam("Authorization") BasicAuthorizationHeader authorizationHeader,
@HeaderParam("X-Signal-Agent") String signalAgent,
@HeaderParam("User-Agent") String userAgent,
@HeaderParam(HttpHeaders.AUTHORIZATION) BasicAuthorizationHeader authorizationHeader,
@HeaderParam(HeaderUtils.X_SIGNAL_AGENT) String signalAgent,
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent,
@QueryParam("transfer") Optional<Boolean> availableForTransfer,
@NotNull @Valid AccountAttributes accountAttributes)
throws RateLimitExceededException, InterruptedException {
@@ -497,12 +401,14 @@ public class AccountController {
throw new WebApplicationException(Response.status(403).build());
}
maybeStoredVerificationCode.map(StoredVerificationCode::twilioVerificationSid)
.ifPresent(
verificationSid -> smsSender.reportVerificationSucceeded(verificationSid, userAgent, "registration"));
Optional<Account> existingAccount = accounts.getByE164(number);
existingAccount.ifPresent(account -> {
Instant accountLastSeen = Instant.ofEpochMilli(account.getLastSeen());
Duration timeSinceLastSeen = Duration.between(accountLastSeen, Instant.now());
REREGISTRATION_IDLE_DAYS_DISTRIBUTION_NAME.record(timeSinceLastSeen.toDays());
});
if (existingAccount.isPresent()) {
verifyRegistrationLock(existingAccount.get(), accountAttributes.getRegistrationLock());
}
@@ -537,7 +443,7 @@ public class AccountController {
@Produces(MediaType.APPLICATION_JSON)
public AccountIdentityResponse changeNumber(@Auth final AuthenticatedAccount authenticatedAccount,
@NotNull @Valid final ChangePhoneNumberRequest request,
@HeaderParam("User-Agent") String userAgent)
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent)
throws RateLimitExceededException, InterruptedException, ImpossiblePhoneNumberException, NonNormalizedPhoneNumberException {
if (!authenticatedAccount.getAuthenticatedDevice().isMaster()) {
@@ -552,16 +458,15 @@ public class AccountController {
rateLimiters.getVerifyLimiter().validate(number);
final Optional<StoredVerificationCode> storedVerificationCode = pendingAccounts.getCodeForNumber(number);
final boolean codeVerified = pendingAccounts.getCodeForNumber(number).map(storedVerificationCode ->
registrationServiceClient.checkVerificationCode(storedVerificationCode.sessionId(),
request.code(), REGISTRATION_RPC_TIMEOUT).join())
.orElse(false);
if (storedVerificationCode.isEmpty() || !storedVerificationCode.get().isValid(request.code())) {
if (!codeVerified) {
throw new ForbiddenException();
}
storedVerificationCode.map(StoredVerificationCode::twilioVerificationSid)
.ifPresent(
verificationSid -> smsSender.reportVerificationSucceeded(verificationSid, userAgent, "changeNumber"));
final Optional<Account> existingAccount = accounts.getByE164(number);
if (existingAccount.isPresent()) {
@@ -727,7 +632,7 @@ public class AccountController {
@Produces(MediaType.APPLICATION_JSON)
@ChangesDeviceEnabledState
public void setAccountAttributes(@Auth DisabledPermittedAuthenticatedAccount disabledPermittedAuth,
@HeaderParam("X-Signal-Agent") String userAgent,
@HeaderParam(HeaderUtils.X_SIGNAL_AGENT) String userAgent,
@NotNull @Valid AccountAttributes attributes) {
Account account = disabledPermittedAuth.getAccount();
long deviceId = disabledPermittedAuth.getAuthenticatedDevice().getId();
@@ -783,7 +688,7 @@ public class AccountController {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public ReserveUsernameResponse reserveUsername(@Auth AuthenticatedAccount auth,
@HeaderParam("X-Signal-Agent") String userAgent,
@HeaderParam(HeaderUtils.X_SIGNAL_AGENT) String userAgent,
@NotNull @Valid ReserveUsernameRequest usernameRequest) throws RateLimitExceededException {
rateLimiters.getUsernameReserveLimiter().validate(auth.getAccount().getUuid());
@@ -805,7 +710,7 @@ public class AccountController {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public UsernameResponse confirmUsername(@Auth AuthenticatedAccount auth,
@HeaderParam("X-Signal-Agent") String userAgent,
@HeaderParam(HeaderUtils.X_SIGNAL_AGENT) String userAgent,
@NotNull @Valid ConfirmUsernameRequest confirmRequest) throws RateLimitExceededException {
rateLimiters.getUsernameSetLimiter().validate(auth.getAccount().getUuid());
@@ -829,7 +734,7 @@ public class AccountController {
@Consumes(MediaType.APPLICATION_JSON)
public UsernameResponse setUsername(
@Auth AuthenticatedAccount auth,
@HeaderParam("X-Signal-Agent") String userAgent,
@HeaderParam(HeaderUtils.X_SIGNAL_AGENT) String userAgent,
@NotNull @Valid UsernameRequest usernameRequest) throws RateLimitExceededException {
rateLimiters.getUsernameSetLimiter().validate(auth.getAccount().getUuid());
checkUsername(usernameRequest.existingUsername(), userAgent);
@@ -851,8 +756,8 @@ public class AccountController {
@Path("/username/{username}")
@Produces(MediaType.APPLICATION_JSON)
public AccountIdentifierResponse lookupUsername(
@HeaderParam("X-Signal-Agent") final String userAgent,
@HeaderParam("X-Forwarded-For") final String forwardedFor,
@HeaderParam(HeaderUtils.X_SIGNAL_AGENT) final String userAgent,
@HeaderParam(HttpHeaders.X_FORWARDED_FOR) final String forwardedFor,
@PathParam("username") final String username,
@Context final HttpServletRequest request) throws RateLimitExceededException {
@@ -875,7 +780,7 @@ public class AccountController {
@HEAD
@Path("/account/{uuid}")
public Response accountExists(
@HeaderParam("X-Forwarded-For") final String forwardedFor,
@HeaderParam(HttpHeaders.X_FORWARDED_FOR) final String forwardedFor,
@PathParam("uuid") final UUID uuid,
@Context HttpServletRequest request) throws RateLimitExceededException {
@@ -893,7 +798,7 @@ public class AccountController {
}
private void rateLimitByClientIp(final RateLimiter rateLimiter, final String forwardedFor) throws RateLimitExceededException {
final String mostRecentProxy = ForwardedIpUtil.getMostRecentProxy(forwardedFor)
final String mostRecentProxy = HeaderUtils.getMostRecentProxy(forwardedFor)
.orElseThrow(() -> {
// Missing/malformed Forwarded-For, so we cannot check for a rate-limit.
// This shouldn't happen, so conservatively assume we're over the rate-limit
@@ -917,25 +822,51 @@ public class AccountController {
rateLimiters.getPinLimiter().validate(existingAccount.getNumber());
}
final String phoneNumber = existingAccount.getNumber();
if (!existingRegistrationLock.verify(clientRegistrationLock)) {
// At this point, the client verified ownership of the phone number but doesnt have the reglock PIN.
// Freezing the existing account credentials will definitively start the reglock timeout.
// Until the timeout, the current reglock can still be supplied,
// along with phone number verification, to restore access.
/* boolean alreadyLocked = existingAccount.hasLockedCredentials();
Metrics.counter(LOCKED_ACCOUNT_COUNTER_NAME,
LOCK_REASON_TAG_NAME, "verifiedNumberFailedReglock",
ALREADY_LOCKED_TAG_NAME, Boolean.toString(alreadyLocked))
.increment();
final Account updatedAccount;
if (!alreadyLocked) {
updatedAccount = accounts.update(existingAccount, Account::lockAuthenticationCredentials);
} else {
updatedAccount = existingAccount;
}
List<Long> deviceIds = updatedAccount.getDevices().stream().map(Device::getId).toList();
clientPresenceManager.disconnectAllPresences(updatedAccount.getUuid(), deviceIds); */
throw new WebApplicationException(Response.status(423)
.entity(new RegistrationLockFailure(existingRegistrationLock.getTimeRemaining(),
existingRegistrationLock.needsFailureCredentials() ? existingBackupCredentials : null))
.build());
}
rateLimiters.getPinLimiter().clear(existingAccount.getNumber());
rateLimiters.getPinLimiter().clear(phoneNumber);
}
}
private boolean pushChallengeMatches(
@VisibleForTesting
static boolean pushChallengeMatches(
final String number,
final Optional<String> pushChallenge,
final Optional<StoredVerificationCode> storedVerificationCode) {
final String countryCode = Util.getCountryCode(number);
final String region = Util.getRegion(number);
Optional<String> storedPushChallenge = storedVerificationCode.map(StoredVerificationCode::pushCode);
boolean match = Optionals.zipWith(pushChallenge, storedPushChallenge, String::equals).orElse(false);
final Optional<String> storedPushChallenge = storedVerificationCode.map(StoredVerificationCode::pushCode);
final boolean match = Optionals.zipWith(pushChallenge, storedPushChallenge, String::equals).orElse(false);
Metrics.counter(PUSH_CHALLENGE_COUNTER_NAME,
COUNTRY_CODE_TAG_NAME, countryCode,
REGION_TAG_NAME, region,
@@ -943,6 +874,7 @@ public class AccountController {
CHALLENGE_PRESENT_TAG_NAME, Boolean.toString(pushChallenge.isPresent()),
CHALLENGE_MATCH_TAG_NAME, Boolean.toString(match))
.increment();
return match;
}
@@ -964,26 +896,12 @@ public class AccountController {
boolean countryFiltered = captchaConfig.getSignupCountryCodes().contains(countryCode) ||
captchaConfig.getSignupRegions().contains(region);
if (abusiveHostRules.isBlocked(sourceHost)) {
blockedHostMeter.mark();
logger.info("Blocked host: {}, {}, {} ({})", transport, number, sourceHost, forwardedFor);
if (countryFiltered) {
// this host was caught in the abusiveHostRules filter, but
// would be caught by country filter as well
countryFilterApplicable.mark();
}
return true;
}
try {
rateLimiters.getSmsVoiceIpLimiter().validate(sourceHost);
} catch (RateLimitExceededException e) {
logger.info("Rate limit exceeded: {}, {}, {} ({})", transport, number, sourceHost, forwardedFor);
rateLimitedHostMeter.mark();
if (shouldAutoBlock(sourceHost)) {
logger.info("Auto-block: {}", sourceHost);
abusiveHostRules.setBlockedHost(sourceHost);
}
return true;
}
@@ -992,10 +910,7 @@ public class AccountController {
} catch (RateLimitExceededException e) {
logger.info("Prefix rate limit exceeded: {}, {}, {} ({})", transport, number, sourceHost, forwardedFor);
rateLimitedPrefixMeter.mark();
if (shouldAutoBlock(sourceHost)) {
logger.info("Auto-block: {}", sourceHost);
abusiveHostRules.setBlockedHost(sourceHost);
}
return true;
}
@@ -1003,6 +918,7 @@ public class AccountController {
countryFilteredHostMeter.mark();
return true;
}
return false;
}
@@ -1022,27 +938,6 @@ public class AccountController {
}
}
private boolean shouldAutoBlock(String sourceHost) {
try {
rateLimiters.getAutoBlockLimiter().validate(sourceHost);
} catch (RateLimitExceededException e) {
return true;
}
return false;
}
@VisibleForTesting protected
VerificationCode generateVerificationCode(String number) {
if (testDevices.containsKey(number)) {
return new VerificationCode(testDevices.get(number));
}
SecureRandom random = new SecureRandom();
int randomInt = 100000 + random.nextInt(900000);
return new VerificationCode(randomInt);
}
private String generatePushChallenge() {
SecureRandom random = new SecureRandom();
byte[] challenge = new byte[16];

View File

@@ -0,0 +1,41 @@
/*
* Copyright 2013-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.controllers;
import com.codahale.metrics.annotation.Timed;
import io.dropwizard.auth.Auth;
import java.util.UUID;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
@Path("/v1/art")
public class ArtController {
private final ExternalServiceCredentialGenerator artServiceCredentialGenerator;
private final RateLimiters rateLimiters;
public ArtController(RateLimiters rateLimiters,
ExternalServiceCredentialGenerator artServiceCredentialGenerator) {
this.artServiceCredentialGenerator = artServiceCredentialGenerator;
this.rateLimiters = rateLimiters;
}
@Timed
@GET
@Path("/auth")
@Produces(MediaType.APPLICATION_JSON)
public ExternalServiceCredentials getAuth(@Auth AuthenticatedAccount auth)
throws RateLimitExceededException {
final UUID uuid = auth.getAccount().getUuid();
rateLimiters.getArtPackLimiter().validate(uuid);
return artServiceCredentialGenerator.generateFor(uuid.toString());
}
}

View File

@@ -24,6 +24,7 @@ import java.util.Optional;
import java.util.UUID;
import javax.annotation.Nonnull;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -66,14 +67,13 @@ public class CertificateController {
@Produces(MediaType.APPLICATION_JSON)
@Path("/delivery")
public DeliveryCertificate getDeliveryCertificate(@Auth AuthenticatedAccount auth,
@QueryParam("includeE164") Optional<Boolean> maybeIncludeE164)
@QueryParam("includeE164") @DefaultValue("true") boolean includeE164)
throws InvalidKeyException {
if (Util.isEmpty(auth.getAccount().getIdentityKey())) {
throw new WebApplicationException(Response.Status.BAD_REQUEST);
}
final boolean includeE164 = maybeIncludeE164.orElse(true);
Metrics.counter(GENERATE_DELIVERY_CERTIFICATE_COUNTER_NAME, INCLUDE_E164_TAG_NAME, String.valueOf(includeE164))
.increment();

View File

@@ -8,9 +8,11 @@ package org.whispersystems.textsecuregcm.controllers;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import com.codahale.metrics.annotation.Timed;
import com.google.common.net.HttpHeaders;
import io.dropwizard.auth.Auth;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
import java.io.IOException;
import java.util.NoSuchElementException;
import javax.validation.Valid;
import javax.ws.rs.Consumes;
@@ -19,7 +21,6 @@ import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
@@ -29,7 +30,7 @@ import org.whispersystems.textsecuregcm.entities.AnswerRecaptchaChallengeRequest
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil;
import org.whispersystems.textsecuregcm.push.NotPushRegisteredException;
import org.whispersystems.textsecuregcm.util.ForwardedIpUtil;
import org.whispersystems.textsecuregcm.util.HeaderUtils;
@Path("/v1/challenge")
public class ChallengeController {
@@ -49,8 +50,8 @@ public class ChallengeController {
@Consumes(MediaType.APPLICATION_JSON)
public Response handleChallengeResponse(@Auth final AuthenticatedAccount auth,
@Valid final AnswerChallengeRequest answerRequest,
@HeaderParam("X-Forwarded-For") final String forwardedFor,
@HeaderParam(HttpHeaders.USER_AGENT) final String userAgent) throws RateLimitExceededException {
@HeaderParam(HttpHeaders.X_FORWARDED_FOR) final String forwardedFor,
@HeaderParam(HttpHeaders.USER_AGENT) final String userAgent) throws RateLimitExceededException, IOException {
Tags tags = Tags.of(UserAgentTagUtil.getPlatformTag(userAgent));
@@ -64,7 +65,7 @@ public class ChallengeController {
try {
final AnswerRecaptchaChallengeRequest recaptchaChallengeRequest = (AnswerRecaptchaChallengeRequest) answerRequest;
final String mostRecentProxy = ForwardedIpUtil.getMostRecentProxy(forwardedFor).orElseThrow();
final String mostRecentProxy = HeaderUtils.getMostRecentProxy(forwardedFor).orElseThrow();
rateLimitChallengeManager.answerRecaptchaChallenge(auth.getAccount(), recaptchaChallengeRequest.getCaptcha(),
mostRecentProxy, userAgent);

View File

@@ -6,6 +6,7 @@ package org.whispersystems.textsecuregcm.controllers;
import com.codahale.metrics.annotation.Timed;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HttpHeaders;
import io.dropwizard.auth.Auth;
import java.security.SecureRandom;
import java.util.LinkedList;
@@ -148,8 +149,8 @@ public class DeviceController {
@Path("/{verification_code}")
@ChangesDeviceEnabledState
public DeviceResponse verifyDeviceToken(@PathParam("verification_code") String verificationCode,
@HeaderParam("Authorization") BasicAuthorizationHeader authorizationHeader,
@HeaderParam("User-Agent") String userAgent,
@HeaderParam(HttpHeaders.AUTHORIZATION) BasicAuthorizationHeader authorizationHeader,
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent,
@NotNull @Valid AccountAttributes accountAttributes,
@Context ContainerRequest containerRequest)
throws RateLimitExceededException, DeviceLimitExceededException {
@@ -187,7 +188,7 @@ public class DeviceController {
}
final DeviceCapabilities capabilities = accountAttributes.getCapabilities();
if (capabilities != null && isCapabilityDowngrade(account.get(), capabilities, userAgent)) {
if (capabilities != null && isCapabilityDowngrade(account.get(), capabilities)) {
throw new WebApplicationException(Response.status(409).build());
}
@@ -235,44 +236,16 @@ public class DeviceController {
return new VerificationCode(randomInt);
}
private boolean isCapabilityDowngrade(Account account, DeviceCapabilities capabilities, String userAgent) {
private boolean isCapabilityDowngrade(Account account, DeviceCapabilities capabilities) {
boolean isDowngrade = false;
// TODO stories capability
// isDowngrade |= account.isStoriesSupported() && !capabilities.isStories();
isDowngrade |= account.isStoriesSupported() && !capabilities.isStories();
isDowngrade |= account.isPniSupported() && !capabilities.isPni();
isDowngrade |= account.isChangeNumberSupported() && !capabilities.isChangeNumber();
isDowngrade |= account.isAnnouncementGroupSupported() && !capabilities.isAnnouncementGroup();
isDowngrade |= account.isSenderKeySupported() && !capabilities.isSenderKey();
isDowngrade |= account.isGv1MigrationSupported() && !capabilities.isGv1Migration();
isDowngrade |= account.isGiftBadgesSupported() && !capabilities.isGiftBadges();
if (account.isGroupsV2Supported()) {
try {
switch (UserAgentUtil.parseUserAgentString(userAgent).getPlatform()) {
case DESKTOP:
case ANDROID: {
if (!capabilities.isGv2_3()) {
isDowngrade = true;
}
break;
}
case IOS: {
if (!capabilities.isGv2_2() && !capabilities.isGv2_3()) {
isDowngrade = true;
}
break;
}
}
} catch (final UnrecognizedUserAgentException e) {
// If we can't parse the UA string, the client is for sure too old to support groups V2
isDowngrade = true;
}
}
return isDowngrade;
}
}

View File

@@ -6,30 +6,13 @@
package org.whispersystems.textsecuregcm.controllers;
import com.codahale.metrics.annotation.Timed;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.dropwizard.auth.Auth;
import io.dropwizard.util.Strings;
import java.net.URI;
import java.net.http.HttpClient.Redirect;
import java.net.http.HttpClient.Version;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executor;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinPool.ManagedBlocker;
import java.util.function.Function;
@@ -52,18 +35,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration;
import org.whispersystems.textsecuregcm.configuration.DonationConfiguration;
import org.whispersystems.textsecuregcm.configuration.StripeConfiguration;
import org.whispersystems.textsecuregcm.entities.ApplePayAuthorizationRequest;
import org.whispersystems.textsecuregcm.entities.ApplePayAuthorizationResponse;
import org.whispersystems.textsecuregcm.entities.RedeemReceiptRequest;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
import org.whispersystems.textsecuregcm.http.FormDataBodyPublisher;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountBadge;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.RedeemedReceiptsManager;
import org.whispersystems.textsecuregcm.util.SystemMapper;
@Path("/v1/donation")
public class DonationController {
@@ -80,11 +56,6 @@ public class DonationController {
private final AccountsManager accountsManager;
private final BadgesConfiguration badgesConfiguration;
private final ReceiptCredentialPresentationFactory receiptCredentialPresentationFactory;
private final URI uri;
private final String apiKey;
private final String description;
private final Set<String> supportedCurrencies;
private final FaultTolerantHttpClient httpClient;
public DonationController(
@Nonnull final Clock clock,
@@ -92,30 +63,13 @@ public class DonationController {
@Nonnull final RedeemedReceiptsManager redeemedReceiptsManager,
@Nonnull final AccountsManager accountsManager,
@Nonnull final BadgesConfiguration badgesConfiguration,
@Nonnull final ReceiptCredentialPresentationFactory receiptCredentialPresentationFactory,
@Nonnull final Executor httpClientExecutor,
@Nonnull final DonationConfiguration configuration,
@Nonnull final StripeConfiguration stripeConfiguration) {
@Nonnull final ReceiptCredentialPresentationFactory receiptCredentialPresentationFactory) {
this.clock = Objects.requireNonNull(clock);
this.serverZkReceiptOperations = Objects.requireNonNull(serverZkReceiptOperations);
this.redeemedReceiptsManager = Objects.requireNonNull(redeemedReceiptsManager);
this.accountsManager = Objects.requireNonNull(accountsManager);
this.badgesConfiguration = Objects.requireNonNull(badgesConfiguration);
this.receiptCredentialPresentationFactory = Objects.requireNonNull(receiptCredentialPresentationFactory);
this.uri = URI.create(configuration.getUri());
this.apiKey = stripeConfiguration.getApiKey();
this.description = configuration.getDescription();
this.supportedCurrencies = configuration.getSupportedCurrencies();
this.httpClient = FaultTolerantHttpClient.newBuilder()
.withCircuitBreaker(configuration.getCircuitBreaker())
.withRetry(configuration.getRetry())
.withVersion(Version.HTTP_2)
.withConnectTimeout(Duration.ofSeconds(10))
.withRedirect(Redirect.NEVER)
.withExecutor(Objects.requireNonNull(httpClientExecutor))
.withName("donation")
.withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_3)
.build();
}
@Timed
@@ -188,55 +142,4 @@ public class DonationController {
}).thenCompose(Function.identity());
}
@Timed
@POST
@Path("/authorize-apple-pay")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public CompletableFuture<Response> getApplePayAuthorization(@Auth AuthenticatedAccount auth, @NotNull @Valid ApplePayAuthorizationRequest request) {
if (!supportedCurrencies.contains(request.getCurrency())) {
return CompletableFuture.completedFuture(Response.status(422).build());
}
final Map<String, String> formData = new HashMap<>();
formData.put("amount", Long.toString(request.getAmount()));
formData.put("currency", request.getCurrency());
if (!Strings.isNullOrEmpty(description)) {
formData.put("description", description);
}
final HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(uri)
.POST(FormDataBodyPublisher.of(formData))
.header("Authorization", "Basic " + Base64.getEncoder().encodeToString(
(apiKey + ":").getBytes(StandardCharsets.UTF_8)))
.header("Content-Type", "application/x-www-form-urlencoded")
.build();
return httpClient.sendAsync(httpRequest, BodyHandlers.ofString())
.thenApply(this::processApplePayAuthorizationRemoteResponse);
}
private Response processApplePayAuthorizationRemoteResponse(HttpResponse<String> response) {
ObjectMapper mapper = SystemMapper.getMapper();
if (response.statusCode() >= 200 && response.statusCode() < 300 &&
MediaType.APPLICATION_JSON.equalsIgnoreCase(response.headers().firstValue("Content-Type").orElse(null))) {
try {
final JsonNode jsonResponse = mapper.readTree(response.body());
final String id = jsonResponse.get("id").asText(null);
final String clientSecret = jsonResponse.get("client_secret").asText(null);
if (Strings.isNullOrEmpty(id) || Strings.isNullOrEmpty(clientSecret)) {
logger.warn("missing fields in json response in donation controller");
return Response.status(500).build();
}
final String responseJson = mapper.writeValueAsString(new ApplePayAuthorizationResponse(id, clientSecret));
return Response.ok(responseJson, MediaType.APPLICATION_JSON_TYPE).build();
} catch (JsonProcessingException e) {
logger.warn("json processing error in donation controller", e);
return Response.status(500).build();
}
} else {
logger.warn("unexpected response code returned to donation controller");
return Response.status(500).build();
}
}
}

View File

@@ -7,6 +7,7 @@ package org.whispersystems.textsecuregcm.controllers;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.annotation.Timed;
import com.google.common.net.HttpHeaders;
import io.dropwizard.auth.Auth;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
@@ -94,7 +95,7 @@ public class KeysController {
public void setKeys(@Auth final DisabledPermittedAuthenticatedAccount disabledPermittedAuth,
@NotNull @Valid final PreKeyState preKeys,
@QueryParam("identity") final Optional<String> identityType,
@HeaderParam("User-Agent") String userAgent) {
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent) {
Account account = disabledPermittedAuth.getAccount();
Device device = disabledPermittedAuth.getAuthenticatedDevice();
boolean updateAccount = false;
@@ -151,7 +152,7 @@ public class KeysController {
@HeaderParam(OptionalAccess.UNIDENTIFIED) Optional<Anonymous> accessKey,
@PathParam("identifier") UUID targetUuid,
@PathParam("device_id") String deviceId,
@HeaderParam("User-Agent") String userAgent)
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent)
throws RateLimitExceededException {
if (!auth.isPresent() && !accessKey.isPresent()) {

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.controllers;
@@ -8,6 +8,7 @@ import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.annotation.Timed;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HttpHeaders;
import com.google.protobuf.ByteString;
import io.dropwizard.auth.Auth;
import io.dropwizard.util.DataSize;
@@ -16,6 +17,7 @@ import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import java.security.MessageDigest;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
@@ -30,6 +32,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
@@ -97,6 +100,7 @@ import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException;
import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
import org.whispersystems.textsecuregcm.websocket.WebSocketConnection;
import org.whispersystems.websocket.Stories;
import reactor.core.scheduler.Schedulers;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Path("/v1/messages")
@@ -163,8 +167,8 @@ public class MessageController {
@FilterAbusiveMessages
public Response sendMessage(@Auth Optional<AuthenticatedAccount> source,
@HeaderParam(OptionalAccess.UNIDENTIFIED) Optional<Anonymous> accessKey,
@HeaderParam("User-Agent") String userAgent,
@HeaderParam("X-Forwarded-For") String forwardedFor,
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent,
@HeaderParam(HttpHeaders.X_FORWARDED_FOR) String forwardedFor,
@PathParam("destination") UUID destinationUuid,
@QueryParam("story") boolean isStory,
@NotNull @Valid IncomingMessageList messages)
@@ -321,8 +325,8 @@ public class MessageController {
@FilterAbusiveMessages
public Response sendMultiRecipientMessage(
@HeaderParam(OptionalAccess.UNIDENTIFIED) @Nullable CombinedUnidentifiedSenderAccessKeys accessKeys,
@HeaderParam("User-Agent") String userAgent,
@HeaderParam("X-Forwarded-For") String forwardedFor,
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent,
@HeaderParam(HttpHeaders.X_FORWARDED_FOR) String forwardedFor,
@QueryParam("online") boolean online,
@QueryParam("ts") long timestamp,
@QueryParam("urgent") @DefaultValue("true") final boolean isUrgent,
@@ -481,47 +485,48 @@ public class MessageController {
@Timed
@GET
@Produces(MediaType.APPLICATION_JSON)
public OutgoingMessageEntityList getPendingMessages(@Auth AuthenticatedAccount auth,
public CompletableFuture<OutgoingMessageEntityList> getPendingMessages(@Auth AuthenticatedAccount auth,
@HeaderParam(Stories.X_SIGNAL_RECEIVE_STORIES) String receiveStoriesHeader,
@HeaderParam("User-Agent") String userAgent) {
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent) {
boolean shouldReceiveStories = Stories.parseReceiveStoriesHeader(receiveStoriesHeader);
pushNotificationManager.handleMessagesRetrieved(auth.getAccount(), auth.getAuthenticatedDevice(), userAgent);
final OutgoingMessageEntityList outgoingMessages;
{
final Pair<List<Envelope>, Boolean> messagesAndHasMore = messagesManager.getMessagesForDevice(
auth.getAccount().getUuid(),
auth.getAuthenticatedDevice().getId(),
false);
return messagesManager.getMessagesForDevice(
auth.getAccount().getUuid(),
auth.getAuthenticatedDevice().getId(),
false)
.map(messagesAndHasMore -> {
Stream<Envelope> envelopes = messagesAndHasMore.first().stream();
if (!shouldReceiveStories) {
envelopes = envelopes.filter(e -> !e.getStory());
}
Stream<Envelope> envelopes = messagesAndHasMore.first().stream();
if (!shouldReceiveStories) {
envelopes = envelopes.filter(e -> !e.getStory());
}
final OutgoingMessageEntityList messages = new OutgoingMessageEntityList(envelopes
.map(OutgoingMessageEntity::fromEnvelope)
.peek(
outgoingMessageEntity -> MessageMetrics.measureAccountOutgoingMessageUuidMismatches(auth.getAccount(),
outgoingMessageEntity))
.collect(Collectors.toList()),
messagesAndHasMore.second());
outgoingMessages = new OutgoingMessageEntityList(envelopes
.map(OutgoingMessageEntity::fromEnvelope)
.peek(outgoingMessageEntity -> MessageMetrics.measureAccountOutgoingMessageUuidMismatches(auth.getAccount(),
outgoingMessageEntity))
.collect(Collectors.toList()),
messagesAndHasMore.second());
}
String platform;
{
String platform;
try {
platform = UserAgentUtil.parseUserAgentString(userAgent).getPlatform().name().toLowerCase();
} catch (final UnrecognizedUserAgentException ignored) {
platform = "unrecognized";
}
try {
platform = UserAgentUtil.parseUserAgentString(userAgent).getPlatform().name().toLowerCase();
} catch (final UnrecognizedUserAgentException ignored) {
platform = "unrecognized";
}
Metrics.summary(OUTGOING_MESSAGE_LIST_SIZE_BYTES_DISTRIBUTION_NAME, "platform", platform)
.record(estimateMessageListSizeBytes(messages));
Metrics.summary(OUTGOING_MESSAGE_LIST_SIZE_BYTES_DISTRIBUTION_NAME, "platform", platform).record(estimateMessageListSizeBytes(outgoingMessages));
}
return outgoingMessages;
return messages;
})
.timeout(Duration.ofSeconds(5))
.subscribeOn(Schedulers.boundedElastic())
.toFuture();
}
private static long estimateMessageListSizeBytes(final OutgoingMessageEntityList messageList) {
@@ -538,25 +543,29 @@ public class MessageController {
@Timed
@DELETE
@Path("/uuid/{uuid}")
public void removePendingMessage(@Auth AuthenticatedAccount auth, @PathParam("uuid") UUID uuid) {
messagesManager.delete(
auth.getAccount().getUuid(),
auth.getAuthenticatedDevice().getId(),
uuid,
null).ifPresent(deletedMessage -> {
public CompletableFuture<Void> removePendingMessage(@Auth AuthenticatedAccount auth, @PathParam("uuid") UUID uuid) {
return messagesManager.delete(
auth.getAccount().getUuid(),
auth.getAuthenticatedDevice().getId(),
uuid,
null)
.thenAccept(maybeDeletedMessage -> {
maybeDeletedMessage.ifPresent(deletedMessage -> {
WebSocketConnection.recordMessageDeliveryDuration(deletedMessage.getTimestamp(), auth.getAuthenticatedDevice());
WebSocketConnection.recordMessageDeliveryDuration(deletedMessage.getTimestamp(),
auth.getAuthenticatedDevice());
if (deletedMessage.hasSourceUuid() && deletedMessage.getType() != Type.SERVER_DELIVERY_RECEIPT) {
try {
receiptSender.sendReceipt(
UUID.fromString(deletedMessage.getDestinationUuid()), auth.getAuthenticatedDevice().getId(),
UUID.fromString(deletedMessage.getSourceUuid()), deletedMessage.getTimestamp());
} catch (Exception e) {
logger.warn("Failed to send delivery receipt", e);
}
}
});
if (deletedMessage.hasSourceUuid() && deletedMessage.getType() != Type.SERVER_DELIVERY_RECEIPT) {
try {
receiptSender.sendReceipt(
UUID.fromString(deletedMessage.getDestinationUuid()), auth.getAuthenticatedDevice().getId(),
UUID.fromString(deletedMessage.getSourceUuid()), deletedMessage.getTimestamp());
} catch (Exception e) {
logger.warn("Failed to send delivery receipt", e);
}
}
});
});
}
@Timed

View File

@@ -317,7 +317,7 @@ public class ProfileController {
@Auth Optional<AuthenticatedAccount> auth,
@HeaderParam(OptionalAccess.UNIDENTIFIED) Optional<Anonymous> accessKey,
@Context ContainerRequestContext containerRequestContext,
@HeaderParam("User-Agent") String userAgent,
@HeaderParam(HttpHeaders.USER_AGENT) String userAgent,
@PathParam("identifier") UUID identifier,
@QueryParam("ca") boolean useCaCertificate)
throws RateLimitExceededException {

View File

@@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.entities.UserRemoteConfigList;
import org.whispersystems.textsecuregcm.storage.RemoteConfig;
import org.whispersystems.textsecuregcm.storage.RemoteConfigsManager;
import org.whispersystems.textsecuregcm.util.Conversions;
import org.whispersystems.textsecuregcm.util.Util;
@Path("/v1/config")
public class RemoteConfigController {
@@ -133,7 +134,7 @@ public class RemoteConfigController {
digest.update(bb.array());
byte[] hash = digest.digest(hashKey);
int bucket = (int)(Math.abs(Conversions.byteArrayToLong(hash)) % 100);
int bucket = (int)(Util.ensureNonNegativeLong(Conversions.byteArrayToLong(hash)) % 100);
return bucket < configPercentage;
}

View File

@@ -12,7 +12,8 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Strings;
import com.google.common.annotations.VisibleForTesting;
import com.stripe.exception.StripeException;
import com.stripe.model.Charge;
import com.stripe.model.Charge.Outcome;
import com.stripe.model.Invoice;
@@ -28,8 +29,10 @@ import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -37,6 +40,7 @@ import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.crypto.Mac;
@@ -46,6 +50,7 @@ import javax.validation.constraints.Min;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.ClientErrorException;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
@@ -78,8 +83,8 @@ import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.badges.BadgeTranslator;
import org.whispersystems.textsecuregcm.badges.LevelTranslator;
import org.whispersystems.textsecuregcm.configuration.BoostConfiguration;
import org.whispersystems.textsecuregcm.configuration.GiftConfiguration;
import org.whispersystems.textsecuregcm.configuration.OneTimeDonationConfiguration;
import org.whispersystems.textsecuregcm.configuration.OneTimeDonationCurrencyConfiguration;
import org.whispersystems.textsecuregcm.configuration.SubscriptionConfiguration;
import org.whispersystems.textsecuregcm.configuration.SubscriptionLevelConfiguration;
import org.whispersystems.textsecuregcm.configuration.SubscriptionPriceConfiguration;
@@ -89,6 +94,7 @@ import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil;
import org.whispersystems.textsecuregcm.storage.IssuedReceiptsManager;
import org.whispersystems.textsecuregcm.storage.SubscriptionManager;
import org.whispersystems.textsecuregcm.storage.SubscriptionManager.GetResult;
import org.whispersystems.textsecuregcm.subscriptions.BraintreeManager;
import org.whispersystems.textsecuregcm.subscriptions.PaymentMethod;
import org.whispersystems.textsecuregcm.subscriptions.ProcessorCustomer;
import org.whispersystems.textsecuregcm.subscriptions.StripeManager;
@@ -103,38 +109,114 @@ public class SubscriptionController {
private final Clock clock;
private final SubscriptionConfiguration subscriptionConfiguration;
private final BoostConfiguration boostConfiguration;
private final GiftConfiguration giftConfiguration;
private final OneTimeDonationConfiguration oneTimeDonationConfiguration;
private final SubscriptionManager subscriptionManager;
private final StripeManager stripeManager;
private final BraintreeManager braintreeManager;
private final ServerZkReceiptOperations zkReceiptOperations;
private final IssuedReceiptsManager issuedReceiptsManager;
private final BadgeTranslator badgeTranslator;
private final LevelTranslator levelTranslator;
private final Map<String, CurrencyConfiguration> currencyConfiguration;
private static final String INVALID_ACCEPT_LANGUAGE_COUNTER_NAME = name(SubscriptionController.class, "invalidAcceptLanguage");
public SubscriptionController(
@Nonnull Clock clock,
@Nonnull SubscriptionConfiguration subscriptionConfiguration,
@Nonnull BoostConfiguration boostConfiguration,
@Nonnull GiftConfiguration giftConfiguration,
@Nonnull OneTimeDonationConfiguration oneTimeDonationConfiguration,
@Nonnull SubscriptionManager subscriptionManager,
@Nonnull StripeManager stripeManager,
@Nonnull BraintreeManager braintreeManager,
@Nonnull ServerZkReceiptOperations zkReceiptOperations,
@Nonnull IssuedReceiptsManager issuedReceiptsManager,
@Nonnull BadgeTranslator badgeTranslator,
@Nonnull LevelTranslator levelTranslator) {
this.clock = Objects.requireNonNull(clock);
this.subscriptionConfiguration = Objects.requireNonNull(subscriptionConfiguration);
this.boostConfiguration = Objects.requireNonNull(boostConfiguration);
this.giftConfiguration = Objects.requireNonNull(giftConfiguration);
this.oneTimeDonationConfiguration = Objects.requireNonNull(oneTimeDonationConfiguration);
this.subscriptionManager = Objects.requireNonNull(subscriptionManager);
this.stripeManager = Objects.requireNonNull(stripeManager);
this.braintreeManager = Objects.requireNonNull(braintreeManager);
this.zkReceiptOperations = Objects.requireNonNull(zkReceiptOperations);
this.issuedReceiptsManager = Objects.requireNonNull(issuedReceiptsManager);
this.badgeTranslator = Objects.requireNonNull(badgeTranslator);
this.levelTranslator = Objects.requireNonNull(levelTranslator);
this.currencyConfiguration = buildCurrencyConfiguration(this.oneTimeDonationConfiguration,
this.subscriptionConfiguration, List.of(stripeManager, braintreeManager));
}
private static Map<String, CurrencyConfiguration> buildCurrencyConfiguration(
OneTimeDonationConfiguration oneTimeDonationConfiguration,
SubscriptionConfiguration subscriptionConfiguration,
List<SubscriptionProcessorManager> subscriptionProcessorManagers) {
return oneTimeDonationConfiguration.currencies()
.entrySet().stream()
.collect(Collectors.toMap(Entry::getKey, currencyAndConfig -> {
final String currency = currencyAndConfig.getKey();
final OneTimeDonationCurrencyConfiguration currencyConfig = currencyAndConfig.getValue();
final Map<String, List<BigDecimal>> oneTimeLevelsToSuggestedAmounts = Map.of(
String.valueOf(oneTimeDonationConfiguration.boost().level()), currencyConfig.boosts(),
String.valueOf(oneTimeDonationConfiguration.gift().level()), List.of(currencyConfig.gift())
);
final Map<String, BigDecimal> subscriptionLevelsToAmounts = subscriptionConfiguration.getLevels()
.entrySet().stream()
.filter(levelIdAndConfig -> levelIdAndConfig.getValue().getPrices().containsKey(currency))
.collect(Collectors.toMap(
levelIdAndConfig -> String.valueOf(levelIdAndConfig.getKey()),
levelIdAndConfig -> levelIdAndConfig.getValue().getPrices().get(currency).getAmount()));
final List<String> supportedPaymentMethods = Arrays.stream(PaymentMethod.values())
.filter(paymentMethod -> subscriptionProcessorManagers.stream()
.anyMatch(manager -> manager.getSupportedCurrencies().contains(currency)
&& manager.supportsPaymentMethod(paymentMethod)))
.map(PaymentMethod::name)
.collect(Collectors.toList());
if (supportedPaymentMethods.isEmpty()) {
throw new RuntimeException("Configuration has currency with no processor support: " + currency);
}
return new CurrencyConfiguration(currencyConfig.minimum(), oneTimeLevelsToSuggestedAmounts,
subscriptionLevelsToAmounts, supportedPaymentMethods);
}));
}
@VisibleForTesting
GetSubscriptionConfigurationResponse buildGetSubscriptionConfigurationResponse(List<Locale> acceptableLanguages) {
final Map<String, LevelConfiguration> levels = new HashMap<>();
subscriptionConfiguration.getLevels().forEach((levelId, levelConfig) -> {
final LevelConfiguration levelConfiguration = new LevelConfiguration(
levelTranslator.translate(acceptableLanguages, levelConfig.getBadge()),
badgeTranslator.translate(acceptableLanguages, levelConfig.getBadge()));
levels.put(String.valueOf(levelId), levelConfiguration);
});
final Badge boostBadge = badgeTranslator.translate(acceptableLanguages,
oneTimeDonationConfiguration.boost().badge());
levels.put(String.valueOf(oneTimeDonationConfiguration.boost().level()),
new LevelConfiguration(
boostBadge.getName(),
// NB: the one-time badges are PurchasableBadge, which has a `duration` field
new PurchasableBadge(
boostBadge,
oneTimeDonationConfiguration.boost().expiration())));
final Badge giftBadge = badgeTranslator.translate(acceptableLanguages, oneTimeDonationConfiguration.gift().badge());
levels.put(String.valueOf(oneTimeDonationConfiguration.gift().level()),
new LevelConfiguration(
giftBadge.getName(),
new PurchasableBadge(
giftBadge,
oneTimeDonationConfiguration.gift().expiration())));
return new GetSubscriptionConfigurationResponse(currencyConfiguration, levels);
}
@Timed
@@ -150,21 +232,22 @@ public class SubscriptionController {
if (getResult == GetResult.NOT_STORED || getResult == GetResult.PASSWORD_MISMATCH) {
throw new NotFoundException();
}
String customerId = getResult.record.customerId;
if (Strings.isNullOrEmpty(customerId)) {
throw new InternalServerErrorException("no customer id found");
}
return stripeManager.getCustomer(customerId).thenCompose(customer -> {
if (customer == null) {
throw new InternalServerErrorException("no customer record found for id " + customerId);
}
return stripeManager.listNonCanceledSubscriptions(customer);
}).thenCompose(subscriptions -> {
@SuppressWarnings("unchecked")
CompletableFuture<Subscription>[] futures = (CompletableFuture<Subscription>[]) subscriptions.stream()
.map(stripeManager::cancelSubscriptionAtEndOfCurrentPeriod).toArray(CompletableFuture[]::new);
return CompletableFuture.allOf(futures);
});
return getResult.record.getProcessorCustomer()
.map(processorCustomer -> stripeManager.getCustomer(processorCustomer.customerId())
.thenCompose(customer -> {
if (customer == null) {
throw new InternalServerErrorException(
"no customer record found for id " + processorCustomer.customerId());
}
return stripeManager.listNonCanceledSubscriptions(customer);
}).thenCompose(subscriptions -> {
@SuppressWarnings("unchecked")
CompletableFuture<Subscription>[] futures = (CompletableFuture<Subscription>[]) subscriptions.stream()
.map(stripeManager::cancelSubscriptionAtEndOfCurrentPeriod).toArray(CompletableFuture[]::new);
return CompletableFuture.allOf(futures);
}))
// a missing customer ID is OK; it means the subscriber never started to add a payment method
.orElseGet(() -> CompletableFuture.completedFuture(null));
})
.thenCompose(unused -> subscriptionManager.canceledAt(requestData.subscriberUser, requestData.now))
.thenApply(unused -> Response.ok().build());
@@ -222,19 +305,22 @@ public class SubscriptionController {
return subscriptionManager.get(requestData.subscriberUser, requestData.hmac)
.thenApply(this::requireRecordFromGetResult)
.thenCompose(record -> {
final CompletableFuture<SubscriptionManager.Record> updatedRecordFuture;
if (record.customerId == null) {
updatedRecordFuture = subscriptionProcessorManager.createCustomer(requestData.subscriberUser)
.thenApply(ProcessorCustomer::customerId)
.thenCompose(customerId -> subscriptionManager.updateProcessorAndCustomerId(record,
new ProcessorCustomer(customerId,
subscriptionProcessorManager.getProcessor()), Instant.now()));
} else {
updatedRecordFuture = CompletableFuture.completedFuture(record);
}
final CompletableFuture<SubscriptionManager.Record> updatedRecordFuture =
record.getProcessorCustomer()
.map(ignored -> CompletableFuture.completedFuture(record))
.orElseGet(() -> subscriptionProcessorManager.createCustomer(requestData.subscriberUser)
.thenApply(ProcessorCustomer::customerId)
.thenCompose(customerId -> subscriptionManager.setProcessorAndCustomerId(record,
new ProcessorCustomer(customerId, subscriptionProcessorManager.getProcessor()),
Instant.now())));
return updatedRecordFuture.thenCompose(
updatedRecord -> subscriptionProcessorManager.createPaymentMethodSetupToken(updatedRecord.customerId));
updatedRecord -> {
final String customerId = updatedRecord.getProcessorCustomer()
.orElseThrow(() -> new InternalServerErrorException("record should not be missing customer"))
.customerId();
return subscriptionProcessorManager.createPaymentMethodSetupToken(customerId);
});
})
.thenApply(
token -> Response.ok(new CreatePaymentMethodResponse(token, subscriptionProcessorManager.getProcessor()))
@@ -244,6 +330,14 @@ public class SubscriptionController {
private SubscriptionProcessorManager getManagerForPaymentMethod(PaymentMethod paymentMethod) {
return switch (paymentMethod) {
case CARD -> stripeManager;
case PAYPAL -> braintreeManager;
};
}
private SubscriptionProcessorManager getManagerForProcessor(SubscriptionProcessor processor) {
return switch (processor) {
case STRIPE -> stripeManager;
case BRAINTREE -> braintreeManager;
};
}
@@ -259,10 +353,15 @@ public class SubscriptionController {
RequestData requestData = RequestData.process(authenticatedAccount, subscriberId, clock);
return subscriptionManager.get(requestData.subscriberUser, requestData.hmac)
.thenApply(this::requireRecordFromGetResult)
.thenCompose(record -> stripeManager.setDefaultPaymentMethodForCustomer(record.customerId, paymentMethodId))
.thenCompose(record -> record.getProcessorCustomer()
.map(processorCustomer -> stripeManager.setDefaultPaymentMethodForCustomer(processorCustomer.customerId(),
paymentMethodId))
.orElseThrow(() ->
// a missing customer ID indicates the client made requests out of order,
// and needs to call create_payment_method to create a customer for the given payment method
new ClientErrorException(Status.CONFLICT)))
.thenApply(customer -> Response.ok().build());
}
public static class SetSubscriptionLevelSuccessResponse {
private final long level;
@@ -285,6 +384,7 @@ public class SubscriptionController {
public enum Type {
UNSUPPORTED_LEVEL,
UNSUPPORTED_CURRENCY,
PAYMENT_REQUIRES_ACTION,
}
private final Type type;
@@ -356,15 +456,38 @@ public class SubscriptionController {
if (record.subscriptionId == null) {
long lastSubscriptionCreatedAt =
record.subscriptionCreatedAt != null ? record.subscriptionCreatedAt.getEpochSecond() : 0;
// we don't have one yet so create it and then record the subscription id
//
// this relies on stripe's idempotency key to avoid creating more than one subscription if the client
// retries this request
return stripeManager.createSubscription(record.customerId, priceConfiguration.getId(), level,
lastSubscriptionCreatedAt)
.thenCompose(subscription -> subscriptionManager.subscriptionCreated(
requestData.subscriberUser, subscription.getId(), requestData.now, level)
.thenApply(unused -> subscription));
return record.getProcessorCustomer()
.map(processorCustomer ->
// we don't have a subscription yet so create it and then record the subscription id
//
// this relies on stripe's idempotency key to avoid creating more than one subscription if the client
// retries this request
stripeManager.createSubscription(processorCustomer.customerId(), priceConfiguration.getId(), level,
lastSubscriptionCreatedAt)
.exceptionally(e -> {
if (e.getCause() instanceof StripeException stripeException
&& stripeException.getCode().equals("subscription_payment_intent_requires_action")) {
throw new BadRequestException(Response.status(Status.BAD_REQUEST)
.entity(new SetSubscriptionLevelErrorResponse(List.of(
new SetSubscriptionLevelErrorResponse.Error(
SetSubscriptionLevelErrorResponse.Error.Type.PAYMENT_REQUIRES_ACTION, null
)
))).build());
}
if (e instanceof RuntimeException re) {
throw re;
}
throw new CompletionException(e);
})
.thenCompose(subscription -> subscriptionManager.subscriptionCreated(
requestData.subscriberUser, subscription.getId(), requestData.now, level)
.thenApply(unused -> subscription)))
.orElseThrow(() ->
// a missing customer ID indicates the client made requests out of order,
// and needs to call create_payment_method to create a customer for the given payment method
new ClientErrorException(Status.CONFLICT));
} else {
// we already have a subscription in our records so let's check the level and change it if needed
return stripeManager.getSubscription(record.subscriptionId).thenCompose(
@@ -428,10 +551,58 @@ public class SubscriptionController {
}
}
/**
* Comprehensive configuration for subscriptions and one-time donations
*
* @param currencies map of lower-cased ISO 3 currency codes to minimums and level-specific scalar amounts
* @param levels map of numeric level IDs to level-specific configuration
*/
public record GetSubscriptionConfigurationResponse(Map<String, CurrencyConfiguration> currencies,
Map<String, LevelConfiguration> levels) {
}
/**
* Configuration for a currency - use to present appropriate client interfaces
*
* @param minimum the minimum amount that may be submitted for a one-time donation in the currency
* @param oneTime map of numeric one-time donation level IDs to the list of default amounts to be
* presented
* @param subscription map of numeric subscription level IDs to the amount charged for that level
* @param supportedPaymentMethods the payment methods that support the given currency
*/
public record CurrencyConfiguration(BigDecimal minimum, Map<String, List<BigDecimal>> oneTime,
Map<String, BigDecimal> subscription,
List<String> supportedPaymentMethods) {
}
/**
* Configuration for a donation level - use to present appropriate client interfaces
*
* @param name the localized name for the level
* @param badge the displayable badge associated with the level
*/
public record LevelConfiguration(String name, Badge badge) {
}
@Timed
@GET
@Path("/configuration")
@Produces(MediaType.APPLICATION_JSON)
public CompletableFuture<Response> getConfiguration(@Context ContainerRequestContext containerRequestContext) {
return CompletableFuture.supplyAsync(() -> {
List<Locale> acceptableLanguages = getAcceptableLanguagesForRequest(containerRequestContext);
return Response.ok(buildGetSubscriptionConfigurationResponse(acceptableLanguages)).build();
});
}
@Timed
@GET
@Path("/levels")
@Produces(MediaType.APPLICATION_JSON)
@Deprecated // use /configuration
public CompletableFuture<Response> getLevels(@Context ContainerRequestContext containerRequestContext) {
return CompletableFuture.supplyAsync(() -> {
List<Locale> acceptableLanguages = getAcceptableLanguagesForRequest(containerRequestContext);
@@ -479,16 +650,21 @@ public class SubscriptionController {
@GET
@Path("/boost/badges")
@Produces(MediaType.APPLICATION_JSON)
@Deprecated // use /configuration
public CompletableFuture<Response> getBoostBadges(@Context ContainerRequestContext containerRequestContext) {
return CompletableFuture.supplyAsync(() -> {
long boostLevel = boostConfiguration.getLevel();
String boostBadge = boostConfiguration.getBadge();
long giftLevel = giftConfiguration.level();
String giftBadge = giftConfiguration.badge();
long boostLevel = oneTimeDonationConfiguration.boost().level();
String boostBadge = oneTimeDonationConfiguration.boost().badge();
long giftLevel = oneTimeDonationConfiguration.gift().level();
String giftBadge = oneTimeDonationConfiguration.gift().badge();
List<Locale> acceptableLanguages = getAcceptableLanguagesForRequest(containerRequestContext);
GetBoostBadgesResponse getBoostBadgesResponse = new GetBoostBadgesResponse(Map.of(
boostLevel, new GetBoostBadgesResponse.Level(new PurchasableBadge(badgeTranslator.translate(acceptableLanguages, boostBadge), boostConfiguration.getExpiration())),
giftLevel, new GetBoostBadgesResponse.Level(new PurchasableBadge(badgeTranslator.translate(acceptableLanguages, giftBadge), giftConfiguration.expiration()))));
boostLevel, new GetBoostBadgesResponse.Level(
new PurchasableBadge(badgeTranslator.translate(acceptableLanguages, boostBadge),
oneTimeDonationConfiguration.boost().expiration())),
giftLevel, new GetBoostBadgesResponse.Level(
new PurchasableBadge(badgeTranslator.translate(acceptableLanguages, giftBadge),
oneTimeDonationConfiguration.gift().expiration()))));
return Response.ok(getBoostBadgesResponse).build();
});
}
@@ -497,28 +673,48 @@ public class SubscriptionController {
@GET
@Path("/boost/amounts")
@Produces(MediaType.APPLICATION_JSON)
@Deprecated // use /configuration
public CompletableFuture<Response> getBoostAmounts() {
return CompletableFuture.supplyAsync(() -> Response.ok(
boostConfiguration.getCurrencies().entrySet().stream().collect(
Collectors.toMap(entry -> entry.getKey().toUpperCase(Locale.ROOT), Entry::getValue))).build());
oneTimeDonationConfiguration.currencies().entrySet().stream().collect(
Collectors.toMap(entry -> entry.getKey().toUpperCase(Locale.ROOT), entry -> entry.getValue().boosts())))
.build());
}
@Timed
@GET
@Path("/boost/amounts/gift")
@Produces(MediaType.APPLICATION_JSON)
@Deprecated // use /configuration
public CompletableFuture<Response> getGiftAmounts() {
return CompletableFuture.supplyAsync(() -> Response.ok(
giftConfiguration.currencies().entrySet().stream().collect(
Collectors.toMap(entry -> entry.getKey().toUpperCase(Locale.ROOT), Entry::getValue))).build());
oneTimeDonationConfiguration.currencies().entrySet().stream().collect(
Collectors.toMap(entry -> entry.getKey().toUpperCase(Locale.ROOT), entry -> entry.getValue().gift())))
.build());
}
public static class CreateBoostRequest {
@NotEmpty @ExactlySize(3) public String currency;
@Min(1) public long amount;
@NotEmpty
@ExactlySize(3)
public String currency;
@Min(1)
public long amount;
public Long level;
}
public static class CreatePayPalBoostRequest extends CreateBoostRequest {
@NotEmpty
public String returnUrl;
@NotEmpty
public String cancelUrl;
}
record CreatePayPalBoostResponse(String approvalUrl, String paymentId) {
}
public static class CreateBoostResponse {
private final String clientSecret;
@@ -541,23 +737,125 @@ public class SubscriptionController {
@Produces(MediaType.APPLICATION_JSON)
public CompletableFuture<Response> createBoostPaymentIntent(@NotNull @Valid CreateBoostRequest request) {
return CompletableFuture.runAsync(() -> {
if (request.level == null) {
request.level = boostConfiguration.getLevel();
}
if (request.level == giftConfiguration.level()) {
BigDecimal amountConfigured = giftConfiguration.currencies().get(request.currency.toLowerCase(Locale.ROOT));
if (amountConfigured == null || stripeManager.convertConfiguredAmountToStripeAmount(request.currency, amountConfigured).compareTo(BigDecimal.valueOf(request.amount)) != 0) {
throw new WebApplicationException(Response.status(Status.CONFLICT).entity(Map.of("error", "level_amount_mismatch")).build());
}
}
})
if (request.level == null) {
request.level = oneTimeDonationConfiguration.boost().level();
}
BigDecimal amount = BigDecimal.valueOf(request.amount);
if (request.level == oneTimeDonationConfiguration.gift().level()) {
BigDecimal amountConfigured = oneTimeDonationConfiguration.currencies()
.get(request.currency.toLowerCase(Locale.ROOT)).gift();
if (amountConfigured == null ||
stripeManager.convertConfiguredAmountToStripeAmount(request.currency, amountConfigured)
.compareTo(amount) != 0) {
throw new WebApplicationException(
Response.status(Status.CONFLICT).entity(Map.of("error", "level_amount_mismatch")).build());
}
}
validateRequestCurrencyAmount(request, amount, stripeManager);
})
.thenCompose(unused -> stripeManager.createPaymentIntent(request.currency, request.amount, request.level))
.thenApply(paymentIntent -> Response.ok(new CreateBoostResponse(paymentIntent.getClientSecret())).build());
}
/**
* Validates that the currency and amount in the request are supported by the {@code manager} and exceed the minimum
* permitted amount
*
* @throws BadRequestException indicates validation failed. Inspect {@code response.error} for details
*/
private void validateRequestCurrencyAmount(CreateBoostRequest request, BigDecimal amount,
SubscriptionProcessorManager manager) {
if (!manager.supportsCurrency(request.currency.toLowerCase(Locale.ROOT))) {
throw new BadRequestException(Response.status(Status.BAD_REQUEST)
.entity(Map.of("error", "unsupported_currency")).build());
}
BigDecimal minCurrencyAmountMajorUnits = oneTimeDonationConfiguration.currencies()
.get(request.currency.toLowerCase(Locale.ROOT)).minimum();
BigDecimal minCurrencyAmountMinorUnits = stripeManager.convertConfiguredAmountToStripeAmount(request.currency,
minCurrencyAmountMajorUnits);
if (minCurrencyAmountMinorUnits.compareTo(amount) > 0) {
throw new BadRequestException(Response.status(Status.BAD_REQUEST)
.entity(Map.of(
"error", "amount_below_currency_minimum",
"minimum", minCurrencyAmountMajorUnits.toString())).build());
}
}
@Timed
@POST
@Path("/boost/paypal/create")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public CompletableFuture<Response> createPayPalBoost(@NotNull @Valid CreatePayPalBoostRequest request,
@Context ContainerRequestContext containerRequestContext) {
return CompletableFuture.runAsync(() -> {
if (request.level == null) {
request.level = oneTimeDonationConfiguration.boost().level();
}
validateRequestCurrencyAmount(request, BigDecimal.valueOf(request.amount), braintreeManager);
})
.thenCompose(unused -> {
final Locale locale = getAcceptableLanguagesForRequest(containerRequestContext).stream()
.filter(l -> !"*".equals(l.getLanguage()))
.findFirst()
.orElse(Locale.US);
return braintreeManager.createOneTimePayment(request.currency.toUpperCase(Locale.ROOT), request.amount,
locale.toLanguageTag(),
request.returnUrl, request.cancelUrl);
})
.thenApply(approvalDetails -> Response.ok(
new CreatePayPalBoostResponse(approvalDetails.approvalUrl(), approvalDetails.paymentId())).build());
}
public static class ConfirmPayPalBoostRequest extends CreateBoostRequest {
@NotEmpty
public String payerId;
@NotEmpty
public String paymentId; // PAYID-…
@NotEmpty
public String paymentToken; // EC-…
}
record ConfirmPayPalBoostResponse(String paymentId) {
}
@Timed
@POST
@Path("/boost/paypal/confirm")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public CompletableFuture<Response> confirmPayPalBoost(@NotNull @Valid ConfirmPayPalBoostRequest request) {
return CompletableFuture.runAsync(() -> {
if (request.level == null) {
request.level = oneTimeDonationConfiguration.boost().level();
}
})
.thenCompose(unused -> braintreeManager.captureOneTimePayment(request.payerId, request.paymentId,
request.paymentToken, request.currency, request.amount, request.level))
.thenApply(chargeSuccessDetails -> Response.ok(
new ConfirmPayPalBoostResponse(chargeSuccessDetails.paymentId())).build());
}
public static class CreateBoostReceiptCredentialsRequest {
@NotNull public String paymentIntentId;
@NotNull public byte[] receiptCredentialRequest;
/**
* a payment ID from {@link #processor}
*/
@NotNull
public String paymentIntentId;
@NotNull
public byte[] receiptCredentialRequest;
@NotNull
public SubscriptionProcessor processor = SubscriptionProcessor.STRIPE;
}
public static class CreateBoostReceiptCredentialsResponse {
@@ -581,32 +879,38 @@ public class SubscriptionController {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public CompletableFuture<Response> createBoostReceiptCredentials(@NotNull @Valid CreateBoostReceiptCredentialsRequest request) {
return stripeManager.getPaymentIntent(request.paymentIntentId)
.thenCompose(paymentIntent -> {
if (paymentIntent == null) {
final SubscriptionProcessorManager manager = getManagerForProcessor(request.processor);
return manager.getPaymentDetails(request.paymentIntentId)
.thenCompose(paymentDetails -> {
if (paymentDetails == null) {
throw new WebApplicationException(Status.NOT_FOUND);
}
if (StringUtils.equalsIgnoreCase("processing", paymentIntent.getStatus())) {
throw new WebApplicationException(Status.NO_CONTENT);
switch (paymentDetails.status()) {
case PROCESSING -> throw new WebApplicationException(Status.NO_CONTENT);
case SUCCEEDED -> {
}
default -> throw new WebApplicationException(Status.PAYMENT_REQUIRED);
}
if (!StringUtils.equalsIgnoreCase("succeeded", paymentIntent.getStatus())) {
throw new WebApplicationException(Status.PAYMENT_REQUIRED);
}
long level = boostConfiguration.getLevel();
if (paymentIntent.getMetadata() != null) {
String levelMetadata = paymentIntent.getMetadata().getOrDefault("level", Long.toString(boostConfiguration.getLevel()));
long level = oneTimeDonationConfiguration.boost().level();
if (paymentDetails.customMetadata() != null) {
String levelMetadata = paymentDetails.customMetadata()
.getOrDefault("level", Long.toString(oneTimeDonationConfiguration.boost().level()));
try {
level = Long.parseLong(levelMetadata);
} catch (NumberFormatException e) {
logger.error("failed to parse level metadata ({}) on payment intent {}", levelMetadata, paymentIntent.getId(), e);
logger.error("failed to parse level metadata ({}) on payment intent {}", levelMetadata,
paymentDetails.id(), e);
throw new WebApplicationException(Status.INTERNAL_SERVER_ERROR);
}
}
Duration levelExpiration;
if (boostConfiguration.getLevel() == level) {
levelExpiration = boostConfiguration.getExpiration();
} else if (giftConfiguration.level() == level) {
levelExpiration = giftConfiguration.expiration();
if (oneTimeDonationConfiguration.boost().level() == level) {
levelExpiration = oneTimeDonationConfiguration.boost().expiration();
} else if (oneTimeDonationConfiguration.gift().level() == level) {
levelExpiration = oneTimeDonationConfiguration.gift().expiration();
} else {
logger.error("level ({}) returned from payment intent that is unknown to the server", level);
throw new WebApplicationException(Status.INTERNAL_SERVER_ERROR);
@@ -618,9 +922,10 @@ public class SubscriptionController {
throw new BadRequestException("invalid receipt credential request", e);
}
final long finalLevel = level;
return issuedReceiptsManager.recordIssuance(paymentIntent.getId(), receiptCredentialRequest, clock.instant())
return issuedReceiptsManager.recordIssuance(paymentDetails.id(), manager.getProcessor(),
receiptCredentialRequest, clock.instant())
.thenApply(unused -> {
Instant expiration = Instant.ofEpochSecond(paymentIntent.getCreated())
Instant expiration = paymentDetails.created()
.plus(levelExpiration)
.truncatedTo(ChronoUnit.DAYS)
.plus(1, ChronoUnit.DAYS);
@@ -869,7 +1174,8 @@ public class SubscriptionController {
return stripeManager.getLatestInvoiceForSubscription(record.subscriptionId)
.thenCompose(invoice -> convertInvoiceToReceipt(invoice, record.subscriptionId))
.thenCompose(receipt -> issuedReceiptsManager.recordIssuance(
receipt.getInvoiceLineItemId(), receiptCredentialRequest, requestData.now)
receipt.getInvoiceLineItemId(), SubscriptionProcessor.STRIPE, receiptCredentialRequest,
requestData.now)
.thenApply(unused -> receipt))
.thenApply(receipt -> {
ReceiptCredentialResponse receiptCredentialResponse;

View File

@@ -0,0 +1,79 @@
package org.whispersystems.textsecuregcm.currency;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.Map;
public class CoinMarketCapClient {
private final HttpClient httpClient;
private final String apiKey;
private final Map<String, Integer> currencyIdsBySymbol;
private static final Logger logger = LoggerFactory.getLogger(CoinMarketCapClient.class);
record CoinMarketCapResponse(@JsonProperty("data") PriceConversionResponse priceConversionResponse) {};
record PriceConversionResponse(int id, String symbol, Map<String, PriceConversionQuote> quote) {};
record PriceConversionQuote(BigDecimal price) {};
public CoinMarketCapClient(final HttpClient httpClient, final String apiKey, final Map<String, Integer> currencyIdsBySymbol) {
this.httpClient = httpClient;
this.apiKey = apiKey;
this.currencyIdsBySymbol = currencyIdsBySymbol;
}
public BigDecimal getSpotPrice(final String currency, final String base) throws IOException {
if (!currencyIdsBySymbol.containsKey(currency)) {
throw new IllegalArgumentException("No currency ID found for " + currency);
}
final URI quoteUri = URI.create(
String.format("https://pro-api.coinmarketcap.com/v2/tools/price-conversion?amount=1&id=%d&convert=%s",
currencyIdsBySymbol.get(currency), base));
try {
final HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
.GET()
.uri(quoteUri)
.header("X-CMC_PRO_API_KEY", apiKey)
.build(),
HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
logger.warn("CoinMarketCapRequest failed with response: {}", response);
throw new IOException("CoinMarketCap request failed with status code " + response.statusCode());
}
return extractConversionRate(parseResponse(response.body()), base);
} catch (final InterruptedException e) {
throw new IOException("Interrupted while waiting for a response", e);
}
}
@VisibleForTesting
static CoinMarketCapResponse parseResponse(final String responseJson) throws JsonProcessingException {
return SystemMapper.getMapper().readValue(responseJson, CoinMarketCapResponse.class);
}
@VisibleForTesting
static BigDecimal extractConversionRate(final CoinMarketCapResponse response, final String destinationCurrency)
throws IOException {
if (!response.priceConversionResponse().quote.containsKey(destinationCurrency)) {
throw new IOException("Response does not contain conversion rate for " + destinationCurrency);
}
return response.priceConversionResponse().quote.get(destinationCurrency).price();
}
}

View File

@@ -1,47 +1,63 @@
package org.whispersystems.textsecuregcm.currency;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.entities.CurrencyConversionEntity;
import org.whispersystems.textsecuregcm.entities.CurrencyConversionEntityList;
import org.whispersystems.textsecuregcm.util.Util;
import io.dropwizard.lifecycle.Managed;
import io.lettuce.core.SetArgs;
import java.io.IOException;
import java.math.BigDecimal;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import io.dropwizard.lifecycle.Managed;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.entities.CurrencyConversionEntity;
import org.whispersystems.textsecuregcm.entities.CurrencyConversionEntityList;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.Util;
public class CurrencyConversionManager implements Managed {
private static final Logger logger = LoggerFactory.getLogger(CurrencyConversionManager.class);
private static final long FIXER_INTERVAL = TimeUnit.HOURS.toMillis(2);
private static final long FTX_INTERVAL = TimeUnit.MINUTES.toMillis(5);
@VisibleForTesting
static final Duration FIXER_REFRESH_INTERVAL = Duration.ofHours(2);
private static final Duration COIN_MARKET_CAP_REFRESH_INTERVAL = Duration.ofMinutes(5);
@VisibleForTesting
static final String COIN_MARKET_CAP_SHARED_CACHE_CURRENT_KEY = "CurrencyConversionManager::CoinMarketCapCacheCurrent";
private static final String COIN_MARKET_CAP_SHARED_CACHE_DATA_KEY = "CurrencyConversionManager::CoinMarketCapCacheData";
private final FixerClient fixerClient;
private final FtxClient ftxClient;
private final CoinMarketCapClient coinMarketCapClient;
private final FaultTolerantRedisCluster cacheCluster;
private final Clock clock;
private final List<String> currencies;
private AtomicReference<CurrencyConversionEntityList> cached = new AtomicReference<>(null);
private final AtomicReference<CurrencyConversionEntityList> cached = new AtomicReference<>(null);
private long fixerUpdatedTimestamp;
private long ftxUpdatedTimestamp;
private Instant fixerUpdatedTimestamp = Instant.MIN;
private Map<String, BigDecimal> cachedFixerValues;
private Map<String, BigDecimal> cachedFtxValues;
private Map<String, BigDecimal> cachedCoinMarketCapValues;
public CurrencyConversionManager(FixerClient fixerClient, FtxClient ftxClient, List<String> currencies) {
public CurrencyConversionManager(final FixerClient fixerClient,
final CoinMarketCapClient coinMarketCapClient,
final FaultTolerantRedisCluster cacheCluster,
final List<String> currencies,
final Clock clock) {
this.fixerClient = fixerClient;
this.ftxClient = ftxClient;
this.coinMarketCapClient = coinMarketCapClient;
this.cacheCluster = cacheCluster;
this.currencies = currencies;
this.clock = clock;
}
public Optional<CurrencyConversionEntityList> getCurrencyConversions() {
@@ -70,25 +86,55 @@ public class CurrencyConversionManager implements Managed {
@VisibleForTesting
void updateCacheIfNecessary() throws IOException {
if (System.currentTimeMillis() - fixerUpdatedTimestamp > FIXER_INTERVAL || cachedFixerValues == null) {
this.cachedFixerValues = new HashMap<>(fixerClient.getConversionsForBase("USD"));
this.fixerUpdatedTimestamp = System.currentTimeMillis();
if (Duration.between(fixerUpdatedTimestamp, clock.instant()).abs().compareTo(FIXER_REFRESH_INTERVAL) >= 0 || cachedFixerValues == null) {
this.cachedFixerValues = new HashMap<>(fixerClient.getConversionsForBase("USD"));
this.fixerUpdatedTimestamp = clock.instant();
}
if (System.currentTimeMillis() - ftxUpdatedTimestamp > FTX_INTERVAL || cachedFtxValues == null) {
Map<String, BigDecimal> cachedFtxValues = new HashMap<>();
{
final Map<String, BigDecimal> coinMarketCapValuesFromSharedCache = cacheCluster.withCluster(connection -> {
final Map<String, BigDecimal> parsedSharedCacheData = new HashMap<>();
for (String currency : currencies) {
cachedFtxValues.put(currency, ftxClient.getSpotPrice(currency, "USD"));
connection.sync().hgetall(COIN_MARKET_CAP_SHARED_CACHE_DATA_KEY).forEach((currency, conversionRate) ->
parsedSharedCacheData.put(currency, new BigDecimal(conversionRate)));
return parsedSharedCacheData;
});
if (coinMarketCapValuesFromSharedCache != null && !coinMarketCapValuesFromSharedCache.isEmpty()) {
cachedCoinMarketCapValues = coinMarketCapValuesFromSharedCache;
}
}
final boolean shouldUpdateSharedCache = cacheCluster.withCluster(connection ->
"OK".equals(connection.sync().set(COIN_MARKET_CAP_SHARED_CACHE_CURRENT_KEY,
"true",
SetArgs.Builder.nx().ex(COIN_MARKET_CAP_REFRESH_INTERVAL))));
if (shouldUpdateSharedCache || cachedCoinMarketCapValues == null) {
final Map<String, BigDecimal> conversionRatesFromCoinMarketCap = new HashMap<>(currencies.size());
for (final String currency : currencies) {
conversionRatesFromCoinMarketCap.put(currency, coinMarketCapClient.getSpotPrice(currency, "USD"));
}
this.cachedFtxValues = cachedFtxValues;
this.ftxUpdatedTimestamp = System.currentTimeMillis();
cachedCoinMarketCapValues = conversionRatesFromCoinMarketCap;
if (shouldUpdateSharedCache) {
cacheCluster.useCluster(connection -> {
final Map<String, String> sharedCoinMarketCapValues = new HashMap<>();
cachedCoinMarketCapValues.forEach((currency, conversionRate) ->
sharedCoinMarketCapValues.put(currency, conversionRate.toString()));
connection.sync().hset(COIN_MARKET_CAP_SHARED_CACHE_DATA_KEY, sharedCoinMarketCapValues);
});
}
}
List<CurrencyConversionEntity> entities = new LinkedList<>();
for (Map.Entry<String, BigDecimal> currency : cachedFtxValues.entrySet()) {
for (Map.Entry<String, BigDecimal> currency : cachedCoinMarketCapValues.entrySet()) {
BigDecimal usdValue = stripTrailingZerosAfterDecimal(currency.getValue());
Map<String, BigDecimal> values = new HashMap<>();
@@ -101,8 +147,7 @@ public class CurrencyConversionManager implements Managed {
entities.add(new CurrencyConversionEntity(currency.getKey(), values));
}
this.cached.set(new CurrencyConversionEntityList(entities, ftxUpdatedTimestamp));
this.cached.set(new CurrencyConversionEntityList(entities, clock.millis()));
}
private BigDecimal stripTrailingZerosAfterDecimal(BigDecimal bigDecimal) {
@@ -113,15 +158,4 @@ public class CurrencyConversionManager implements Managed {
return n;
}
}
@VisibleForTesting
void setFixerUpdatedTimestamp(long timestamp) {
this.fixerUpdatedTimestamp = timestamp;
}
@VisibleForTesting
void setFtxUpdatedTimestamp(long timestamp) {
this.ftxUpdatedTimestamp = timestamp;
}
}

View File

@@ -1,69 +0,0 @@
package org.whispersystems.textsecuregcm.currency;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
public class FtxClient {
private final HttpClient client;
public FtxClient(HttpClient client) {
this.client = client;
}
public BigDecimal getSpotPrice(String currency, String base) throws FtxException{
try {
URI uri = URI.create("https://ftx.com/api/markets/" + currency + "/" + base);
HttpResponse<String> response = client.send(HttpRequest.newBuilder()
.GET()
.uri(uri)
.build(),
HttpResponse.BodyHandlers.ofString());
if (response.statusCode() < 200 || response.statusCode() >= 300) {
throw new FtxException("Bad response: " + response.statusCode() + " " + response.toString());
}
FtxResponse parsedResponse = SystemMapper.getMapper().readValue(response.body(), FtxResponse.class);
return parsedResponse.result.price;
} catch (IOException | InterruptedException e) {
throw new FtxException(e);
}
}
private static class FtxResponse {
@JsonProperty
private FtxResult result;
}
private static class FtxResult {
@JsonProperty
private BigDecimal price;
}
public static class FtxException extends IOException {
public FtxException(String message) {
super(message);
}
public FtxException(Exception exception) {
super(exception);
}
}
}

View File

@@ -1,42 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Pattern;
import javax.validation.constraints.Size;
public class ApplePayAuthorizationRequest {
private String currency;
private long amount;
@JsonProperty
@NotEmpty
@Size(min=3, max=3)
@Pattern(regexp="[a-z]{3}")
public String getCurrency() {
return currency;
}
public void setCurrency(final String currency) {
this.currency = currency;
}
@JsonProperty
@Min(0)
public long getAmount() {
return amount;
}
@VisibleForTesting
public void setAmount(final long amount) {
this.amount = amount;
}
}

View File

@@ -1,44 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.dropwizard.util.Strings;
import javax.validation.constraints.NotEmpty;
public class ApplePayAuthorizationResponse {
private final String id;
private final String clientSecret;
@JsonCreator
public ApplePayAuthorizationResponse(
@JsonProperty("id") final String id,
@JsonProperty("client_secret") final String clientSecret) {
if (Strings.isNullOrEmpty(id)) {
throw new IllegalArgumentException("id cannot be empty");
}
if (Strings.isNullOrEmpty(clientSecret)) {
throw new IllegalArgumentException("clientSecret cannot be empty");
}
this.id = id;
this.clientSecret = clientSecret;
}
@JsonProperty("id")
@NotEmpty
public String getId() {
return id;
}
@JsonProperty("client_secret")
@NotEmpty
public String getClientSecret() {
return clientSecret;
}
}

View File

@@ -5,6 +5,7 @@
package org.whispersystems.textsecuregcm.entities;
import java.util.Arrays;
import java.util.UUID;
import javax.validation.Valid;
import javax.validation.constraints.Max;
@@ -53,6 +54,37 @@ public class MultiRecipientMessage {
public byte[] getPerRecipientKeyMaterial() {
return perRecipientKeyMaterial;
}
@Override
public boolean equals(final Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
Recipient recipient = (Recipient) o;
if (deviceId != recipient.deviceId)
return false;
if (registrationId != recipient.registrationId)
return false;
if (!uuid.equals(recipient.uuid))
return false;
return Arrays.equals(perRecipientKeyMaterial, recipient.perRecipientKeyMaterial);
}
@Override
public int hashCode() {
int result = uuid.hashCode();
result = 31 * result + (int) (deviceId ^ (deviceId >>> 32));
result = 31 * result + registrationId;
result = 31 * result + Arrays.hashCode(perRecipientKeyMaterial);
return result;
}
public String toString() {
return "Recipient(" + uuid + ", " + deviceId + ", " + registrationId + ", " + Arrays.toString(perRecipientKeyMaterial) + ")";
}
}
@NotNull

View File

@@ -8,85 +8,30 @@ package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.whispersystems.textsecuregcm.storage.Account;
public class UserCapabilities {
public record UserCapabilities(
@JsonProperty("gv1-migration") boolean gv1Migration,
boolean senderKey,
boolean announcementGroup,
boolean changeNumber,
boolean stories,
boolean giftBadges,
boolean paymentActivation,
boolean pni) {
public static UserCapabilities createForAccount(Account account) {
return new UserCapabilities(
account.isGroupsV2Supported(),
account.isGv1MigrationSupported(),
true,
account.isSenderKeySupported(),
account.isAnnouncementGroupSupported(),
account.isChangeNumberSupported(),
account.isStoriesSupported(),
account.isGiftBadgesSupported());
}
account.isGiftBadgesSupported(),
@JsonProperty
private boolean gv2;
// Hardcode payment activation flag to false until all clients support the flow
false,
@JsonProperty("gv1-migration")
private boolean gv1Migration;
@JsonProperty
private boolean senderKey;
@JsonProperty
private boolean announcementGroup;
@JsonProperty
private boolean changeNumber;
@JsonProperty
private boolean stories;
@JsonProperty
private boolean giftBadges;
public UserCapabilities() {
}
public UserCapabilities(final boolean gv2,
boolean gv1Migration,
final boolean senderKey,
final boolean announcementGroup,
final boolean changeNumber,
final boolean stories,
final boolean giftBadges) {
this.gv2 = gv2;
this.gv1Migration = gv1Migration;
this.senderKey = senderKey;
this.announcementGroup = announcementGroup;
this.changeNumber = changeNumber;
this.stories = stories;
this.giftBadges = giftBadges;
}
public boolean isGv2() {
return gv2;
}
public boolean isGv1Migration() {
return gv1Migration;
}
public boolean isSenderKey() {
return senderKey;
}
public boolean isAnnouncementGroup() {
return announcementGroup;
}
public boolean isChangeNumber() {
return changeNumber;
}
public boolean isStories() {
return stories;
}
public boolean isGiftBadges() {
return giftBadges;
// Although originally intended to indicate that clients support phone number identifiers, the scope of this
// flag has expanded to cover phone number privacy in general
account.isPniSupported());
}
}

View File

@@ -1,37 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.filters;
import static com.codahale.metrics.MetricRegistry.name;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.metrics.TrafficSource;
public class ContentLengthFilter implements ContainerRequestFilter {
private static final String CONTENT_LENGTH_DISTRIBUTION_NAME = name(ContentLengthFilter.class, "contentLength");
private static final Logger logger = LoggerFactory.getLogger(ContentLengthFilter.class);
private final TrafficSource trafficSource;
public ContentLengthFilter(final TrafficSource trafficeSource) {
this.trafficSource = trafficeSource;
}
@Override
public void filter(final ContainerRequestContext requestContext) throws IOException {
try {
Metrics.summary(CONTENT_LENGTH_DISTRIBUTION_NAME, "trafficSource", trafficSource.name().toLowerCase()).record(requestContext.getLength());
} catch (final Exception e) {
logger.warn("Error recording content length", e);
}
}
}

View File

@@ -7,6 +7,7 @@ package org.whispersystems.textsecuregcm.filters;
import static com.codahale.metrics.MetricRegistry.name;
import com.google.common.net.HttpHeaders;
import com.vdurmont.semver4j.Semver;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
@@ -63,7 +64,7 @@ public class RemoteDeprecationFilter implements Filter {
boolean shouldBlock = false;
try {
final String userAgentString = ((HttpServletRequest) request).getHeader("User-Agent");
final String userAgentString = ((HttpServletRequest) request).getHeader(HttpHeaders.USER_AGENT);
final UserAgent userAgent = UserAgentUtil.parseUserAgentString(userAgentString);
if (blockedVersionsByPlatform.containsKey(userAgent.getPlatform())) {

View File

@@ -0,0 +1,78 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.filters;
import static com.codahale.metrics.MetricRegistry.name;
import static java.util.Objects.requireNonNull;
import com.google.common.net.HttpHeaders;
import com.google.common.net.InetAddresses;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
import javax.annotation.Nonnull;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.metrics.TrafficSource;
import org.whispersystems.textsecuregcm.util.HeaderUtils;
public class RequestStatisticsFilter implements ContainerRequestFilter {
private static final Logger logger = LoggerFactory.getLogger(RequestStatisticsFilter.class);
private static final String CONTENT_LENGTH_DISTRIBUTION_NAME = name(RequestStatisticsFilter.class, "contentLength");
private static final String IP_VERSION_METRIC = name(RequestStatisticsFilter.class, "ipVersion");
private static final String TRAFFIC_SOURCE_TAG = "trafficSource";
private static final String IP_VERSION_TAG = "ipVersion";
@Nonnull
private final String trafficSourceTag;
public RequestStatisticsFilter(@Nonnull final TrafficSource trafficeSource) {
this.trafficSourceTag = requireNonNull(trafficeSource).name().toLowerCase();
}
@Override
public void filter(final ContainerRequestContext requestContext) throws IOException {
try {
Metrics.summary(CONTENT_LENGTH_DISTRIBUTION_NAME, TRAFFIC_SOURCE_TAG, trafficSourceTag)
.record(requestContext.getLength());
Metrics.counter(IP_VERSION_METRIC, TRAFFIC_SOURCE_TAG, trafficSourceTag, IP_VERSION_TAG, resolveIpVersion(requestContext))
.increment();
} catch (final Exception e) {
logger.warn("Error recording request statistics", e);
}
}
@Nonnull
private static String resolveIpVersion(@Nonnull final ContainerRequestContext ctx) {
return HeaderUtils.getMostRecentProxy(ctx.getHeaderString(HttpHeaders.X_FORWARDED_FOR))
.map(ipString -> {
try {
//noinspection UnstableApiUsage
final InetAddress addr = InetAddresses.forString(ipString);
if (addr instanceof Inet4Address) {
return "IPv4";
}
if (addr instanceof Inet6Address) {
return "IPv6";
}
} catch (IllegalArgumentException e) {
// ignore illegal argument exception
}
return null;
})
.orElse("unresolved");
}
}

View File

@@ -5,11 +5,10 @@
package org.whispersystems.textsecuregcm.filters;
import org.whispersystems.textsecuregcm.util.TimestampHeaderUtil;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.container.ContainerResponseFilter;
import org.whispersystems.textsecuregcm.util.HeaderUtils;
/**
* Injects a timestamp header into all outbound responses.
@@ -18,6 +17,6 @@ public class TimestampResponseFilter implements ContainerResponseFilter {
@Override
public void filter(final ContainerRequestContext requestContext, final ContainerResponseContext responseContext) {
responseContext.getHeaders().add(TimestampHeaderUtil.TIMESTAMP_HEADER, System.currentTimeMillis());
responseContext.getHeaders().add(HeaderUtils.TIMESTAMP_HEADER, System.currentTimeMillis());
}
}

View File

@@ -135,7 +135,7 @@ public class FaultTolerantHttpClient {
return this;
}
public Builder withTrustedServerCertificate(final String certificatePem) throws CertificateException {
public Builder withTrustedServerCertificates(final String... certificatePem) throws CertificateException {
this.trustStore = CertificateUtil.buildKeyStoreForPem(certificatePem);
return this;
}

View File

@@ -5,22 +5,22 @@ import static com.codahale.metrics.MetricRegistry.name;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.whispersystems.textsecuregcm.abuse.RateLimitChallengeListener;
import org.whispersystems.textsecuregcm.captcha.CaptchaChecker;
import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil;
import org.whispersystems.textsecuregcm.push.NotPushRegisteredException;
import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.util.Util;
public class RateLimitChallengeManager {
private final PushChallengeManager pushChallengeManager;
private final RecaptchaClient recaptchaClient;
private final CaptchaChecker captchaChecker;
private final DynamicRateLimiters rateLimiters;
private final List<RateLimitChallengeListener> rateLimitChallengeListeners =
@@ -34,11 +34,11 @@ public class RateLimitChallengeManager {
public RateLimitChallengeManager(
final PushChallengeManager pushChallengeManager,
final RecaptchaClient recaptchaClient,
final CaptchaChecker captchaChecker,
final DynamicRateLimiters rateLimiters) {
this.pushChallengeManager = pushChallengeManager;
this.recaptchaClient = recaptchaClient;
this.captchaChecker = captchaChecker;
this.rateLimiters = rateLimiters;
}
@@ -58,11 +58,11 @@ public class RateLimitChallengeManager {
}
public void answerRecaptchaChallenge(final Account account, final String captcha, final String mostRecentProxyIp, final String userAgent)
throws RateLimitExceededException {
throws RateLimitExceededException, IOException {
rateLimiters.getRecaptchaChallengeAttemptLimiter().validate(account.getUuid());
final boolean challengeSuccess = recaptchaClient.verify(captcha, mostRecentProxyIp).valid();
final boolean challengeSuccess = captchaChecker.verify(captcha, mostRecentProxyIp).valid();
final Tags tags = Tags.of(
Tag.of(SOURCE_COUNTRY_TAG_NAME, Util.getCountryCode(account.getNumber())),

View File

@@ -15,7 +15,6 @@ public class RateLimiters {
private final RateLimiter voiceDestinationDailyLimiter;
private final RateLimiter smsVoiceIpLimiter;
private final RateLimiter smsVoicePrefixLimiter;
private final RateLimiter autoBlockLimiter;
private final RateLimiter verifyLimiter;
private final RateLimiter pinLimiter;
@@ -30,6 +29,8 @@ public class RateLimiters {
private final RateLimiter profileLimiter;
private final RateLimiter stickerPackLimiter;
private final RateLimiter artPackLimiter;
private final RateLimiter usernameLookupLimiter;
private final RateLimiter usernameSetLimiter;
@@ -60,10 +61,6 @@ public class RateLimiters {
config.getSmsVoicePrefix().getBucketSize(),
config.getSmsVoicePrefix().getLeakRatePerMinute());
this.autoBlockLimiter = new RateLimiter(cacheCluster, "autoBlock",
config.getAutoBlock().getBucketSize(),
config.getAutoBlock().getLeakRatePerMinute());
this.verifyLimiter = new LockingRateLimiter(cacheCluster, "verify",
config.getVerifyNumber().getBucketSize(),
config.getVerifyNumber().getLeakRatePerMinute());
@@ -104,6 +101,10 @@ public class RateLimiters {
config.getStickerPack().getBucketSize(),
config.getStickerPack().getLeakRatePerMinute());
this.artPackLimiter = new RateLimiter(cacheCluster, "artPack",
config.getArtPack().getBucketSize(),
config.getArtPack().getLeakRatePerMinute());
this.usernameLookupLimiter = new RateLimiter(cacheCluster, "usernameLookup",
config.getUsernameLookup().getBucketSize(),
config.getUsernameLookup().getLeakRatePerMinute());
@@ -158,10 +159,6 @@ public class RateLimiters {
return smsVoicePrefixLimiter;
}
public RateLimiter getAutoBlockLimiter() {
return autoBlockLimiter;
}
public RateLimiter getVoiceDestinationLimiter() {
return voiceDestinationLimiter;
}
@@ -190,6 +187,10 @@ public class RateLimiters {
return stickerPackLimiter;
}
public RateLimiter getArtPackLimiter() {
return artPackLimiter;
}
public RateLimiter getUsernameLookupLimiter() {
return usernameLookupLimiter;
}

View File

@@ -5,7 +5,9 @@
package org.whispersystems.textsecuregcm.mappers;
import java.util.Optional;
import java.util.concurrent.CompletionException;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
@@ -23,8 +25,25 @@ public class CompletionExceptionMapper implements ExceptionMapper<CompletionExce
final Throwable cause = exception.getCause();
if (cause != null) {
final Class type = cause.getClass();
return providers.getExceptionMapper(type).toResponse(cause);
final ExceptionMapper exceptionMapper = providers.getExceptionMapper(type);
// some exception mappers, like LoggingExceptionMapper, have side effects (e.g., logging)
// so we always build their response…
final Response exceptionMapperResponse = exceptionMapper.toResponse(cause);
final Optional<Response> webApplicationExceptionResponse;
if (cause instanceof WebApplicationException webApplicationException) {
webApplicationExceptionResponse = Optional.of(webApplicationException.getResponse());
} else {
webApplicationExceptionResponse = Optional.empty();
}
// …but if the exception was a WebApplicationException, and provides an entity, we want to keep it
return webApplicationExceptionResponse
.filter(Response::hasEntity)
.orElse(exceptionMapperResponse);
}
return Response.serverError().build();

View File

@@ -7,11 +7,17 @@ package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HttpHeaders;
import com.vdurmont.semver4j.Semver;
import com.vdurmont.semver4j.SemverException;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.glassfish.jersey.server.monitoring.RequestEvent;
import org.glassfish.jersey.server.monitoring.RequestEventListener;
import org.whispersystems.textsecuregcm.util.logging.UriInfoUtil;
@@ -20,12 +26,6 @@ import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException;
import org.whispersystems.textsecuregcm.util.ua.UserAgent;
import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Gathers and reports request-level metrics.
*/
@@ -75,7 +75,7 @@ public class MetricsRequestEventListener implements RequestEventListener {
tags.add(Tag.of(STATUS_CODE_TAG, String.valueOf(event.getContainerResponse().getStatus())));
tags.add(Tag.of(TRAFFIC_SOURCE_TAG, trafficSource.name().toLowerCase()));
final List<String> userAgentValues = event.getContainerRequest().getRequestHeader("User-Agent");
final List<String> userAgentValues = event.getContainerRequest().getRequestHeader(HttpHeaders.USER_AGENT);
// tags.addAll(UserAgentTagUtil.getUserAgentTags(userAgentValues != null ? userAgentValues.stream().findFirst().orElse(null) : null));
tags.add(UserAgentTagUtil.getPlatformTag(userAgentValues != null ? userAgentValues.stream().findFirst().orElse(null) : null));

View File

@@ -1,93 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.metrics;
import static com.codahale.metrics.MetricRegistry.name;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class NstatCounters {
private final Map<String, Long> networkStatistics = new ConcurrentHashMap<>();
private static final String[] NSTAT_COMMAND_LINE = new String[] { "nstat", "--zero", "--json", "--noupdate", "--ignore" };
private static final String[] EXCLUDE_METRIC_NAME_PREFIXES = new String[] { "Icmp", "Udp", "Ip6" };
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
private static final Logger log = LoggerFactory.getLogger(NstatCounters.class);
@VisibleForTesting
static class NetworkStatistics {
private final Map<String, Long> kernelStatistics;
@JsonCreator
private NetworkStatistics(@JsonProperty("kernel") final Map<String, Long> kernelStatistics) {
this.kernelStatistics = kernelStatistics;
}
public Map<String, Long> getKernelStatistics() {
return kernelStatistics;
}
}
public void registerMetrics(final ScheduledExecutorService refreshService, final Duration refreshInterval) {
refreshNetworkStatistics();
networkStatistics.keySet().stream()
.filter(NstatCounters::shouldIncludeMetric)
.forEach(metricName -> Metrics.globalRegistry.more().counter(name(getClass(), "kernel", metricName),
Collections.emptyList(), networkStatistics, statistics -> statistics.get(metricName)));
refreshService.scheduleAtFixedRate(this::refreshNetworkStatistics,
refreshInterval.toMillis(), refreshInterval.toMillis(), TimeUnit.MILLISECONDS);
}
private void refreshNetworkStatistics() {
try {
networkStatistics.putAll(loadNetworkStatistics().getKernelStatistics());
} catch (final InterruptedException | IOException e) {
log.warn("Failed to refresh network statistics", e);
}
}
@VisibleForTesting
static boolean shouldIncludeMetric(final String metricName) {
for (final String prefix : EXCLUDE_METRIC_NAME_PREFIXES) {
if (metricName.startsWith(prefix)) {
return false;
}
}
return true;
}
@VisibleForTesting
static NetworkStatistics loadNetworkStatistics() throws IOException, InterruptedException {
final Process nstatProcess = Runtime.getRuntime().exec(NSTAT_COMMAND_LINE);
if (nstatProcess.waitFor() == 0) {
return OBJECT_MAPPER.readValue(nstatProcess.getInputStream(), NetworkStatistics.class);
} else {
throw new IOException("nstat process did not exit normally");
}
}
}

View File

@@ -107,10 +107,11 @@ public class MultiRecipientMessageProvider implements MessageBodyReader<MultiRec
*
* @return the varint value
*/
private long readVarint(InputStream stream) throws IOException, WebApplicationException {
@VisibleForTesting
public static long readVarint(InputStream stream) throws IOException, WebApplicationException {
boolean hasMore = true;
int currentOffset = 0;
int result = 0;
long result = 0;
while (hasMore) {
if (currentOffset >= 64) {
throw new BadRequestException("varint is too large");
@@ -123,7 +124,7 @@ public class MultiRecipientMessageProvider implements MessageBodyReader<MultiRec
throw new BadRequestException("varint is too large");
}
hasMore = (b & 0x80) != 0;
result |= (b & 0x7F) << currentOffset;
result |= (b & 0x7FL) << currentOffset;
currentOffset += 7;
}
return result;

View File

@@ -13,6 +13,8 @@ import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import io.dropwizard.lifecycle.Managed;
import io.lettuce.core.LettuceFutures;
import io.lettuce.core.RedisFuture;
import io.lettuce.core.ScriptOutputType;
import io.lettuce.core.cluster.SlotHash;
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
@@ -21,6 +23,7 @@ import io.lettuce.core.cluster.models.partitions.RedisClusterNode;
import io.lettuce.core.cluster.pubsub.RedisClusterPubSubAdapter;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -178,16 +181,25 @@ public class ClientPresenceManager extends RedisClusterPubSubAdapter<String, Str
List.of(managerId, String.valueOf(PRESENCE_EXPIRATION_SECONDS)));
}
public void disconnectAllPresences(final UUID accountUuid, final List<Long> deviceIds) {
List<String> presenceKeys = new ArrayList<>();
deviceIds.forEach(deviceId -> {
String presenceKey = getPresenceKey(accountUuid, deviceId);
if (isLocallyPresent(accountUuid, deviceId)) {
displacePresence(presenceKey, false);
}
presenceKeys.add(presenceKey);
});
presenceCluster.useCluster(connection -> {
List<RedisFuture<Long>> futures = presenceKeys.stream().map(key -> connection.async().del(key)).toList();
LettuceFutures.awaitAll(connection.getTimeout(), futures.toArray(new RedisFuture[0]));
});
}
public void disconnectPresence(final UUID accountUuid, final long deviceId) {
final String presenceKey = getPresenceKey(accountUuid, deviceId);
if (isLocallyPresent(accountUuid, deviceId)) {
displacePresence(presenceKey, false);
}
// If connected locally, we still need to clean up the presence key.
// If connected remotely, the other server will get a keyspace message and handle the disconnect
presenceCluster.useCluster(connection -> connection.sync().del(presenceKey));
disconnectAllPresences(accountUuid, List.of(deviceId));
}
private void displacePresence(final String presenceKey, final boolean connectedElsewhere) {

View File

@@ -8,14 +8,11 @@ package org.whispersystems.textsecuregcm.push;
import com.codahale.metrics.InstrumentedExecutorService;
import com.codahale.metrics.SharedMetricRegistries;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.controllers.NoSuchUserException;
import org.whispersystems.textsecuregcm.entities.MessageProtos.Envelope;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.util.Constants;
@@ -37,34 +34,41 @@ public class ReceiptSender {
MetricsUtil.name(ReceiptSender.class, "executor"));
}
public CompletableFuture<Void> sendReceipt(UUID sourceUuid, long sourceDeviceId, UUID destinationUuid, long messageId)
throws NoSuchUserException {
public void sendReceipt(UUID sourceUuid, long sourceDeviceId, UUID destinationUuid, long messageId) {
if (sourceUuid.equals(destinationUuid)) {
return CompletableFuture.completedFuture(null);
return;
}
final Account destinationAccount = accountManager.getByAccountIdentifier(destinationUuid)
.orElseThrow(() -> new NoSuchUserException(destinationUuid));
executor.submit(() -> {
try {
accountManager.getByAccountIdentifier(destinationUuid).ifPresentOrElse(
destinationAccount -> {
final Envelope.Builder message = Envelope.newBuilder()
.setServerTimestamp(System.currentTimeMillis())
.setSourceUuid(sourceUuid.toString())
.setSourceDevice((int) sourceDeviceId)
.setDestinationUuid(destinationUuid.toString())
.setTimestamp(messageId)
.setType(Envelope.Type.SERVER_DELIVERY_RECEIPT)
.setUrgent(false);
final Envelope.Builder message = Envelope.newBuilder()
.setServerTimestamp(System.currentTimeMillis())
.setSourceUuid(sourceUuid.toString())
.setSourceDevice((int) sourceDeviceId)
.setDestinationUuid(destinationUuid.toString())
.setTimestamp(messageId)
.setType(Envelope.Type.SERVER_DELIVERY_RECEIPT)
.setUrgent(false);
for (final Device destinationDevice : destinationAccount.getDevices()) {
try {
messageSender.sendMessage(destinationAccount, destinationDevice, message.build(), false);
} catch (final NotPushRegisteredException e) {
logger.debug("User no longer push registered for delivery receipt: {}", e.getMessage());
} catch (final Exception e) {
logger.warn("Could not send delivery receipt", e);
}
}
},
() -> logger.info("No longer registered: {}", destinationUuid)
);
return CompletableFuture.runAsync(() -> {
for (final Device destinationDevice : destinationAccount.getDevices()) {
try {
messageSender.sendMessage(destinationAccount, destinationDevice, message.build(), false);
} catch (final NotPushRegisteredException e) {
logger.info("User no longer push registered for delivery receipt: " + e.getMessage());
} catch (final Exception e) {
logger.warn("Could not send delivery receipt", e);
}
} catch (final Exception e) {
// this exception is most likely a Dynamo timeout or a Redis timeout/circuit breaker
logger.warn("Could not send delivery receipt", e);
}
}, executor);
});
}
}

View File

@@ -6,6 +6,7 @@
package org.whispersystems.textsecuregcm.redis;
import com.google.common.annotations.VisibleForTesting;
import io.lettuce.core.RedisException;
import io.lettuce.core.RedisNoScriptException;
import io.lettuce.core.ScriptOutputType;
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
@@ -15,9 +16,12 @@ import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import org.apache.commons.codec.binary.Hex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class ClusterLuaScript {
@@ -73,11 +77,31 @@ public class ClusterLuaScript {
execute(connection, keys.toArray(STRING_ARRAY), args.toArray(STRING_ARRAY)));
}
public CompletableFuture<Object> executeAsync(final List<String> keys, final List<String> args) {
return redisCluster.withCluster(connection ->
executeAsync(connection, keys.toArray(STRING_ARRAY), args.toArray(STRING_ARRAY)));
}
public Flux<Object> executeReactive(final List<String> keys, final List<String> args) {
return redisCluster.withCluster(connection ->
executeReactive(connection, keys.toArray(STRING_ARRAY), args.toArray(STRING_ARRAY)));
}
public Object executeBinary(final List<byte[]> keys, final List<byte[]> args) {
return redisCluster.withBinaryCluster(connection ->
execute(connection, keys.toArray(BYTE_ARRAY_ARRAY), args.toArray(BYTE_ARRAY_ARRAY)));
}
public CompletableFuture<Object> executeBinaryAsync(final List<byte[]> keys, final List<byte[]> args) {
return redisCluster.withBinaryCluster(connection ->
executeAsync(connection, keys.toArray(BYTE_ARRAY_ARRAY), args.toArray(BYTE_ARRAY_ARRAY)));
}
public Flux<Object> executeBinaryReactive(final List<byte[]> keys, final List<byte[]> args) {
return redisCluster.withBinaryCluster(connection ->
executeReactive(connection, keys.toArray(BYTE_ARRAY_ARRAY), args.toArray(BYTE_ARRAY_ARRAY)));
}
private <T> Object execute(final StatefulRedisClusterConnection<T, T> connection, final T[] keys, final T[] args) {
try {
try {
@@ -90,4 +114,32 @@ public class ClusterLuaScript {
throw e;
}
}
private <T> CompletableFuture<Object> executeAsync(final StatefulRedisClusterConnection<T, T> connection,
final T[] keys, final T[] args) {
return connection.async().evalsha(sha, scriptOutputType, keys, args)
.exceptionallyCompose(throwable -> {
if (throwable instanceof RedisNoScriptException) {
return connection.async().eval(script, scriptOutputType, keys, args);
}
log.warn("Failed to execute script", throwable);
throw new RedisException(throwable);
}).toCompletableFuture();
}
private <T> Flux<Object> executeReactive(final StatefulRedisClusterConnection<T, T> connection,
final T[] keys, final T[] args) {
return connection.reactive().evalsha(sha, scriptOutputType, keys, args)
.onErrorResume(e -> {
if (e instanceof RedisNoScriptException) {
return connection.reactive().eval(script, scriptOutputType, keys, args);
}
log.warn("Failed to execute script", e);
return Mono.error(e);
});
}
}

View File

@@ -8,6 +8,8 @@ package org.whispersystems.textsecuregcm.redis;
import com.codahale.metrics.SharedMetricRegistries;
import com.google.common.annotations.VisibleForTesting;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.reactor.circuitbreaker.operator.CircuitBreakerOperator;
import io.github.resilience4j.reactor.retry.RetryOperator;
import io.github.resilience4j.retry.Retry;
import io.lettuce.core.ClientOptions.DisconnectedBehavior;
import io.lettuce.core.RedisCommandTimeoutException;
@@ -24,11 +26,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import org.reactivestreams.Publisher;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
import org.whispersystems.textsecuregcm.util.Constants;
import reactor.core.publisher.Flux;
/**
* A fault-tolerant access manager for a Redis cluster. A fault-tolerant Redis cluster provides managed,
@@ -49,96 +53,115 @@ public class FaultTolerantRedisCluster {
private final Retry retry;
public FaultTolerantRedisCluster(final String name, final RedisClusterConfiguration clusterConfiguration, final ClientResources clientResources) {
this(name,
RedisClusterClient.create(clientResources, clusterConfiguration.getConfigurationUri()),
clusterConfiguration.getTimeout(),
clusterConfiguration.getCircuitBreakerConfiguration(),
clusterConfiguration.getRetryConfiguration());
this(name,
RedisClusterClient.create(clientResources, clusterConfiguration.getConfigurationUri()),
clusterConfiguration.getTimeout(),
clusterConfiguration.getCircuitBreakerConfiguration(),
clusterConfiguration.getRetryConfiguration());
}
@VisibleForTesting
FaultTolerantRedisCluster(final String name, final RedisClusterClient clusterClient, final Duration commandTimeout, final CircuitBreakerConfiguration circuitBreakerConfiguration, final RetryConfiguration retryConfiguration) {
this.name = name;
this.name = name;
this.clusterClient = clusterClient;
this.clusterClient.setDefaultTimeout(commandTimeout);
this.clusterClient.setOptions(ClusterClientOptions.builder()
.disconnectedBehavior(DisconnectedBehavior.REJECT_COMMANDS)
.validateClusterNodeMembership(false)
.topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
.enableAllAdaptiveRefreshTriggers()
.build())
.build());
this.clusterClient = clusterClient;
this.clusterClient.setDefaultTimeout(commandTimeout);
this.clusterClient.setOptions(ClusterClientOptions.builder()
.disconnectedBehavior(DisconnectedBehavior.REJECT_COMMANDS)
.validateClusterNodeMembership(false)
.topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
.enableAllAdaptiveRefreshTriggers()
.build())
.publishOnScheduler(true)
.build());
this.stringConnection = clusterClient.connect();
this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
this.stringConnection = clusterClient.connect();
this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
this.circuitBreaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder().retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
this.circuitBreaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder()
.retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
CircuitBreakerUtil.registerMetrics(SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME), circuitBreaker, FaultTolerantRedisCluster.class);
CircuitBreakerUtil.registerMetrics(SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME), retry, FaultTolerantRedisCluster.class);
CircuitBreakerUtil.registerMetrics(SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME), circuitBreaker,
FaultTolerantRedisCluster.class);
CircuitBreakerUtil.registerMetrics(SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME), retry,
FaultTolerantRedisCluster.class);
}
void shutdown() {
stringConnection.close();
binaryConnection.close();
stringConnection.close();
binaryConnection.close();
for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
pubSubConnection.close();
}
for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
pubSubConnection.close();
}
clusterClient.shutdown();
clusterClient.shutdown();
}
public String getName() {
return name;
}
public String getName() {
return name;
}
public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
useConnection(stringConnection, consumer);
}
public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
useConnection(stringConnection, consumer);
}
public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
return withConnection(stringConnection, function);
}
public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
return withConnection(stringConnection, function);
}
public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
useConnection(binaryConnection, consumer);
}
public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
useConnection(binaryConnection, consumer);
}
public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
return withConnection(binaryConnection, function);
}
public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
return withConnection(binaryConnection, function);
}
private <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection, final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
try {
circuitBreaker.executeCheckedRunnable(() -> retry.executeRunnable(() -> consumer.accept(connection)));
} catch (final Throwable t) {
if (t instanceof RedisException) {
throw (RedisException) t;
} else {
throw new RedisException(t);
}
}
}
public <T> Publisher<T> withBinaryClusterReactive(
final Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function) {
return withConnectionReactive(binaryConnection, function);
}
private <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection, final Function<StatefulRedisClusterConnection<K, V>, T> function) {
try {
return circuitBreaker.executeCheckedSupplier(() -> retry.executeCallable(() -> function.apply(connection)));
} catch (final Throwable t) {
if (t instanceof RedisException) {
throw (RedisException) t;
} else {
throw new RedisException(t);
}
}
private <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection,
final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
try {
circuitBreaker.executeCheckedRunnable(() -> retry.executeRunnable(() -> consumer.accept(connection)));
} catch (final Throwable t) {
if (t instanceof RedisException) {
throw (RedisException) t;
} else {
throw new RedisException(t);
}
}
}
public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
pubSubConnections.add(pubSubConnection);
return new FaultTolerantPubSubConnection<>(name, pubSubConnection, circuitBreaker, retry);
private <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection,
final Function<StatefulRedisClusterConnection<K, V>, T> function) {
try {
return circuitBreaker.executeCheckedSupplier(() -> retry.executeCallable(() -> function.apply(connection)));
} catch (final Throwable t) {
if (t instanceof RedisException) {
throw (RedisException) t;
} else {
throw new RedisException(t);
}
}
}
private <T, K, V> Publisher<T> withConnectionReactive(final StatefulRedisClusterConnection<K, V> connection,
final Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function) {
return Flux.from(function.apply(connection))
.transformDeferred(RetryOperator.of(retry))
.transformDeferred(CircuitBreakerOperator.of(circuitBreaker));
}
public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
pubSubConnections.add(pubSubConnection);
return new FaultTolerantPubSubConnection<>(name, pubSubConnection, circuitBreaker, retry);
}
}

View File

@@ -46,7 +46,7 @@ public class SecureBackupClient {
.withExecutor(executor)
.withName("secure-backup")
.withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_2)
.withTrustedServerCertificate(configuration.getBackupCaCertificate())
.withTrustedServerCertificates(configuration.getBackupCaCertificates().toArray(new String[0]))
.build();
}

View File

@@ -47,7 +47,7 @@ public class SecureStorageClient {
.withExecutor(executor)
.withName("secure-storage")
.withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_3)
.withTrustedServerCertificate(configuration.getStorageCaCertificate())
.withTrustedServerCertificates(configuration.getStorageCaCertificates().toArray(new String[0]))
.build();
}

View File

@@ -1,57 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.sms;
import java.util.List;
import java.util.Locale.LanguageRange;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import javax.annotation.Nullable;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
public class SmsSender {
private final TwilioSmsSender twilioSender;
public SmsSender(TwilioSmsSender twilioSender) {
this.twilioSender = twilioSender;
}
public void deliverSmsVerification(String destination, Optional<String> clientType, String verificationCode) {
// Fix up mexico numbers to 'mobile' format just for SMS delivery.
if (destination.startsWith("+52") && !destination.startsWith("+521")) {
destination = "+521" + destination.substring("+52".length());
}
twilioSender.deliverSmsVerification(destination, clientType, verificationCode);
}
public void deliverVoxVerification(String destination, String verificationCode, List<LanguageRange> languageRanges) {
twilioSender.deliverVoxVerification(destination, verificationCode, languageRanges);
}
public CompletableFuture<Optional<String>> deliverSmsVerificationWithTwilioVerify(String destination,
Optional<String> clientType,
String verificationCode, List<LanguageRange> languageRanges) {
// Fix up mexico numbers to 'mobile' format just for SMS delivery.
if (destination.startsWith("+52") && !destination.startsWith("+521")) {
destination = "+521" + destination.substring(3);
}
return twilioSender.deliverSmsVerificationWithVerify(destination, clientType, verificationCode, languageRanges);
}
public CompletableFuture<Optional<String>> deliverVoxVerificationWithTwilioVerify(String destination,
String verificationCode,
List<LanguageRange> languageRanges) {
return twilioSender.deliverVoxVerificationWithVerify(destination, verificationCode, languageRanges);
}
public void reportVerificationSucceeded(String verificationSid, @Nullable String userAgent, String context) {
twilioSender.reportVerificationSucceeded(verificationSid, userAgent, context);
}
}

View File

@@ -1,345 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.sms;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Locale.LanguageRange;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.TwilioConfiguration;
import org.whispersystems.textsecuregcm.configuration.TwilioVerificationTextConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
import org.whispersystems.textsecuregcm.http.FormDataBodyPublisher;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.ExecutorUtils;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.Util;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
public class TwilioSmsSender {
private static final Logger logger = LoggerFactory.getLogger(TwilioSmsSender.class);
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Meter smsMeter = metricRegistry.meter(name(getClass(), "sms", "delivered"));
private final Meter voxMeter = metricRegistry.meter(name(getClass(), "vox", "delivered"));
private final Meter priceMeter = metricRegistry.meter(name(getClass(), "price"));
static final String FAILED_REQUEST_COUNTER_NAME = name(TwilioSmsSender.class, "failedRequest");
static final String SERVICE_NAME_TAG = "service";
static final String STATUS_CODE_TAG_NAME = "statusCode";
static final String ERROR_CODE_TAG_NAME = "errorCode";
static final String COUNTRY_CODE_TAG_NAME = "countryCode";
/**
* @deprecated "region" conflicts with cloud provider region tags; prefer "regionCode" instead
*/
@Deprecated
static final String REGION_TAG_NAME = "region";
static final String REGION_CODE_TAG_NAME = "regionCode";
private final String accountId;
private final String accountToken;
private final String messagingServiceSid;
private final String nanpaMessagingServiceSid;
private final String localDomain;
private final Random random;
private final TwilioVerificationTextConfiguration defaultClientVerificationTexts;
private final Map<String,TwilioVerificationTextConfiguration> regionalClientVerificationTexts;
private final FaultTolerantHttpClient httpClient;
private final URI smsUri;
private final URI voxUri;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
private final TwilioVerifySender twilioVerifySender;
@VisibleForTesting
public TwilioSmsSender(String baseUri,
String baseVerifyUri,
TwilioConfiguration twilioConfiguration,
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
Executor executor = ExecutorUtils.newFixedThreadBoundedQueueExecutor(10, 100);
this.accountId = twilioConfiguration.getAccountId();
this.accountToken = twilioConfiguration.getAccountToken();
this.localDomain = twilioConfiguration.getLocalDomain();
this.messagingServiceSid = twilioConfiguration.getMessagingServiceSid();
this.nanpaMessagingServiceSid = twilioConfiguration.getNanpaMessagingServiceSid();
this.random = new Random(System.currentTimeMillis());
this.smsUri = URI.create(baseUri + "/2010-04-01/Accounts/" + accountId + "/Messages.json");
this.voxUri = URI.create(baseUri + "/2010-04-01/Accounts/" + accountId + "/Calls.json" );
this.httpClient = FaultTolerantHttpClient.newBuilder()
.withCircuitBreaker(twilioConfiguration.getCircuitBreaker())
.withRetry(twilioConfiguration.getRetry())
.withVersion(HttpClient.Version.HTTP_2)
.withConnectTimeout(Duration.ofSeconds(10))
.withRedirect(HttpClient.Redirect.NEVER)
.withExecutor(executor)
.withName("twilio")
.build();
this.defaultClientVerificationTexts = twilioConfiguration.getDefaultClientVerificationTexts();
this.regionalClientVerificationTexts = twilioConfiguration.getRegionalClientVerificationTexts();
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.twilioVerifySender = new TwilioVerifySender(baseVerifyUri, httpClient, twilioConfiguration);
}
public TwilioSmsSender(TwilioConfiguration twilioConfiguration, DynamicConfigurationManager dynamicConfigurationManager) {
this("https://api.twilio.com", "https://verify.twilio.com", twilioConfiguration, dynamicConfigurationManager);
}
public CompletableFuture<Boolean> deliverSmsVerification(String destination, Optional<String> clientType, String verificationCode) {
Map<String, String> requestParameters = new HashMap<>();
requestParameters.put("To", destination);
requestParameters.put("MessagingServiceSid", "1".equals(Util.getCountryCode(destination)) ? nanpaMessagingServiceSid : messagingServiceSid);
requestParameters.put("Body", String.format(Locale.US, getBodyFormatString(destination, clientType.orElse(null)), verificationCode));
HttpRequest request = HttpRequest.newBuilder()
.uri(smsUri)
.POST(FormDataBodyPublisher.of(requestParameters))
.header("Content-Type", "application/x-www-form-urlencoded")
.header("Authorization", "Basic " + Base64.getEncoder().encodeToString((accountId + ":" + accountToken).getBytes(StandardCharsets.UTF_8)))
.build();
smsMeter.mark();
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(this::parseResponse)
.handle((response, throwable) -> processResponse(response, throwable, destination));
}
private String getBodyFormatString(@Nonnull String destination, @Nullable String clientType) {
final String countryCode = Util.getCountryCode(destination);
final TwilioVerificationTextConfiguration verificationTexts = regionalClientVerificationTexts
.getOrDefault(countryCode, defaultClientVerificationTexts);
final String result;
if ("ios".equals(clientType)) {
result = verificationTexts.getIosText();
} else if ("android-ng".equals(clientType)) {
result = verificationTexts.getAndroidNgText();
} else if ("android-2020-01".equals(clientType)) {
result = verificationTexts.getAndroid202001Text();
} else if ("android-2021-03".equals(clientType)) {
result = verificationTexts.getAndroid202103Text();
} else {
result = verificationTexts.getGenericText();
}
if ("86".equals(countryCode)) { // is China
return result + "\u2008";
// Twilio recommends adding this character to the end of strings delivered to China because some carriers in
// China are blocking GSM-7 encoding and this will force Twilio to send using UCS-2 instead.
} else {
return result;
}
}
public CompletableFuture<Boolean> deliverVoxVerification(String destination, String verificationCode, List<LanguageRange> languageRanges) {
String url = "https://" + localDomain + "/v1/voice/description/" + verificationCode;
final String languageQueryParams = languageRanges.stream()
.map(range -> Locale.forLanguageTag(range.getRange()))
.map(locale -> {
if (StringUtils.isNotBlank(locale.getCountry())) {
return locale.getLanguage().toLowerCase() + "-" + locale.getCountry().toUpperCase();
} else {
return locale.getLanguage().toLowerCase();
}
})
.map(languageTag -> "l=" + languageTag)
.collect(Collectors.joining("&"));
if (StringUtils.isNotBlank(languageQueryParams)) {
url += "?" + languageQueryParams;
}
Map<String, String> requestParameters = new HashMap<>();
requestParameters.put("Url", url);
requestParameters.put("To", destination);
requestParameters.put("From", getRandom(random, dynamicConfigurationManager.getConfiguration().getTwilioConfiguration().getNumbers()));
HttpRequest request = HttpRequest.newBuilder()
.uri(voxUri)
.POST(FormDataBodyPublisher.of(requestParameters))
.header("Content-Type", "application/x-www-form-urlencoded")
.header("Authorization", "Basic " + Base64.getEncoder().encodeToString((accountId + ":" + accountToken).getBytes()))
.build();
voxMeter.mark();
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(this::parseResponse)
.handle((response, throwable) -> processResponse(response, throwable, destination));
}
private String getRandom(Random random, List<String> elements) {
return elements.get(random.nextInt(elements.size()));
}
private boolean processResponse(TwilioResponse response, Throwable throwable, String destination) {
if (response != null && response.isSuccess()) {
priceMeter.mark((long) (response.successResponse.price * 1000));
return true;
} else if (response != null && response.isFailure()) {
String countryCode = Util.getCountryCode(destination);
String region = Util.getRegion(destination);
Metrics.counter(FAILED_REQUEST_COUNTER_NAME,
SERVICE_NAME_TAG, "classic",
STATUS_CODE_TAG_NAME, String.valueOf(response.failureResponse.status),
ERROR_CODE_TAG_NAME, String.valueOf(response.failureResponse.code),
COUNTRY_CODE_TAG_NAME, countryCode,
REGION_TAG_NAME, region,
REGION_CODE_TAG_NAME, region).increment();
logger.info("Failed with code={}, country={}",
response.failureResponse.code,
countryCode);
return false;
} else if (throwable != null) {
logger.info("Twilio request failed", throwable);
return false;
} else {
logger.warn("No response or throwable!");
return false;
}
}
private TwilioResponse parseResponse(HttpResponse<String> response) {
ObjectMapper mapper = SystemMapper.getMapper();
if (response.statusCode() >= 200 && response.statusCode() < 300) {
if ("application/json".equals(response.headers().firstValue("Content-Type").orElse(null))) {
return new TwilioResponse(TwilioResponse.TwilioSuccessResponse.fromBody(mapper, response.body()));
} else {
return new TwilioResponse(new TwilioResponse.TwilioSuccessResponse());
}
}
if ("application/json".equals(response.headers().firstValue("Content-Type").orElse(null))) {
return new TwilioResponse(TwilioResponse.TwilioFailureResponse.fromBody(mapper, response.body()));
} else {
return new TwilioResponse(new TwilioResponse.TwilioFailureResponse());
}
}
public CompletableFuture<Optional<String>> deliverSmsVerificationWithVerify(String destination,
Optional<String> clientType, String verificationCode, List<LanguageRange> languageRanges) {
smsMeter.mark();
return twilioVerifySender.deliverSmsVerificationWithVerify(destination, clientType, verificationCode,
languageRanges);
}
public CompletableFuture<Optional<String>> deliverVoxVerificationWithVerify(String destination,
String verificationCode, List<LanguageRange> languageRanges) {
voxMeter.mark();
return twilioVerifySender.deliverVoxVerificationWithVerify(destination, verificationCode, languageRanges);
}
public CompletableFuture<Boolean> reportVerificationSucceeded(String verificationSid, @Nullable String userAgent,
String context) {
return twilioVerifySender.reportVerificationSucceeded(verificationSid, userAgent, context);
}
public static class TwilioResponse {
private TwilioSuccessResponse successResponse;
private TwilioFailureResponse failureResponse;
TwilioResponse(TwilioSuccessResponse successResponse) {
this.successResponse = successResponse;
}
TwilioResponse(TwilioFailureResponse failureResponse) {
this.failureResponse = failureResponse;
}
boolean isSuccess() {
return successResponse != null;
}
boolean isFailure() {
return failureResponse != null;
}
private static class TwilioSuccessResponse {
@JsonProperty
private double price;
static TwilioSuccessResponse fromBody(ObjectMapper mapper, String body) {
try {
return mapper.readValue(body, TwilioSuccessResponse.class);
} catch (IOException e) {
logger.warn("Error parsing twilio success response: " + e);
return new TwilioSuccessResponse();
}
}
}
private static class TwilioFailureResponse {
@JsonProperty
private int status;
@JsonProperty
private String message;
@JsonProperty
private int code;
static TwilioFailureResponse fromBody(ObjectMapper mapper, String body) {
try {
return mapper.readValue(body, TwilioFailureResponse.class);
} catch (IOException e) {
logger.warn("Error parsing twilio success response: " + e);
return new TwilioFailureResponse();
}
}
}
}
}

View File

@@ -1,74 +0,0 @@
package org.whispersystems.textsecuregcm.sms;
import com.google.common.annotations.VisibleForTesting;
import java.util.List;
import java.util.Locale.LanguageRange;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.whispersystems.textsecuregcm.configuration.VoiceVerificationConfiguration;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
public class TwilioVerifyExperimentEnrollmentManager {
@VisibleForTesting
static final String EXPERIMENT_NAME = "twilio_verify_v1";
private final ExperimentEnrollmentManager experimentEnrollmentManager;
private static final Set<String> INELIGIBLE_CLIENTS = Set.of("android-ng", "android-2020-01");
private final Set<String> signalExclusiveVoiceVerificationLanguages;
public TwilioVerifyExperimentEnrollmentManager(final VoiceVerificationConfiguration voiceVerificationConfiguration,
final ExperimentEnrollmentManager experimentEnrollmentManager) {
this.experimentEnrollmentManager = experimentEnrollmentManager;
// Signal voice verification supports several languages that Verify does not. We want to honor
// clients that prioritize these languages, even if they would normally be enrolled in the experiment
signalExclusiveVoiceVerificationLanguages = voiceVerificationConfiguration.getLocales().stream()
.map(loc -> loc.split("-")[0])
.filter(language -> !TwilioVerifySender.TWILIO_VERIFY_LANGUAGES.contains(language))
.collect(Collectors.toSet());
}
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
public boolean isEnrolled(Optional<String> clientType, String number, List<LanguageRange> languageRanges,
String transport) {
final boolean clientEligible = clientType.map(client -> !INELIGIBLE_CLIENTS.contains(client))
.orElse(true);
final boolean languageEligible;
if ("sms".equals(transport)) {
// Signal only sends SMS in en, while Verify supports en + many other languages
languageEligible = true;
} else {
boolean clientPreferredLanguageOnlySupportedBySignal = false;
for (LanguageRange languageRange : languageRanges) {
final String language = languageRange.getRange().split("-")[0];
if (signalExclusiveVoiceVerificationLanguages.contains(language)) {
// Support is exclusive to Signal.
// Since this is the first match in the priority list, so let's break and honor it
clientPreferredLanguageOnlySupportedBySignal = true;
break;
}
if (TwilioVerifySender.TWILIO_VERIFY_LANGUAGES.contains(language)) {
// Twilio supports it, so we can stop looping
break;
}
// the language is supported by neither, so let's loop again
}
languageEligible = !clientPreferredLanguageOnlySupportedBySignal;
}
final boolean enrolled = experimentEnrollmentManager.isEnrolled(number, EXPERIMENT_NAME);
return clientEligible && languageEligible && enrolled;
}
}

View File

@@ -1,324 +0,0 @@
package org.whispersystems.textsecuregcm.sms;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Locale.LanguageRange;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import javax.annotation.Nullable;
import javax.validation.constraints.NotEmpty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.TwilioConfiguration;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
import org.whispersystems.textsecuregcm.http.FormDataBodyPublisher;
import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.Util;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
class TwilioVerifySender {
private static final Logger logger = LoggerFactory.getLogger(TwilioVerifySender.class);
private static final String VERIFICATION_SUCCEEDED_RESPONSE_COUNTER_NAME = name(TwilioVerifySender.class,
"verificationSucceeded");
private static final String CONTEXT_TAG_NAME = "context";
private static final String STATUS_CODE_TAG_NAME = "statusCode";
private static final String ERROR_CODE_TAG_NAME = "errorCode";
static final Set<String> TWILIO_VERIFY_LANGUAGES = Set.of(
"af",
"ar",
"ca",
"zh",
"zh-CN",
"zh-HK",
"hr",
"cs",
"da",
"nl",
"en",
"en-GB",
"fi",
"fr",
"de",
"el",
"he",
"hi",
"hu",
"id",
"it",
"ja",
"ko",
"ms",
"nb",
"pl",
"pt",
"pt-BR",
"ro",
"ru",
"es",
"sv",
"tl",
"th",
"tr",
"vi");
private final String accountId;
private final String accountToken;
private final URI verifyServiceUri;
private final URI verifyApprovalBaseUri;
private final String androidAppHash;
private final String verifyServiceFriendlyName;
private final FaultTolerantHttpClient httpClient;
TwilioVerifySender(String baseUri, FaultTolerantHttpClient httpClient, TwilioConfiguration twilioConfiguration) {
this.accountId = twilioConfiguration.getAccountId();
this.accountToken = twilioConfiguration.getAccountToken();
this.verifyServiceUri = URI
.create(baseUri + "/v2/Services/" + twilioConfiguration.getVerifyServiceSid() + "/Verifications");
this.verifyApprovalBaseUri = URI
.create(baseUri + "/v2/Services/" + twilioConfiguration.getVerifyServiceSid() + "/Verifications/");
this.androidAppHash = twilioConfiguration.getAndroidAppHash();
this.verifyServiceFriendlyName = twilioConfiguration.getVerifyServiceFriendlyName();
this.httpClient = httpClient;
}
CompletableFuture<Optional<String>> deliverSmsVerificationWithVerify(String destination, Optional<String> clientType,
String verificationCode, List<LanguageRange> languageRanges) {
HttpRequest request = buildVerifyRequest("sms", destination, verificationCode, findBestLocale(languageRanges),
clientType);
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(this::parseResponse)
.handle((response, throwable) -> extractVerifySid(response, throwable, destination));
}
private Optional<String> findBestLocale(List<LanguageRange> priorityList) {
return Util.findBestLocale(priorityList, TwilioVerifySender.TWILIO_VERIFY_LANGUAGES);
}
private TwilioVerifyResponse parseResponse(HttpResponse<String> response) {
ObjectMapper mapper = SystemMapper.getMapper();
if (response.statusCode() >= 200 && response.statusCode() < 300) {
if ("application/json".equals(response.headers().firstValue("Content-Type").orElse(null))) {
return new TwilioVerifyResponse(TwilioVerifyResponse.SuccessResponse.fromBody(mapper, response.body()));
} else {
return new TwilioVerifyResponse(new TwilioVerifyResponse.SuccessResponse());
}
}
if ("application/json".equals(response.headers().firstValue("Content-Type").orElse(null))) {
return new TwilioVerifyResponse(TwilioVerifyResponse.FailureResponse.fromBody(mapper, response.body()));
} else {
return new TwilioVerifyResponse(new TwilioVerifyResponse.FailureResponse());
}
}
CompletableFuture<Optional<String>> deliverVoxVerificationWithVerify(String destination,
String verificationCode, List<LanguageRange> languageRanges) {
HttpRequest request = buildVerifyRequest("call", destination, verificationCode, findBestLocale(languageRanges),
Optional.empty());
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(this::parseResponse)
.handle((response, throwable) -> extractVerifySid(response, throwable, destination));
}
private Optional<String> extractVerifySid(TwilioVerifyResponse twilioVerifyResponse, Throwable throwable,
String destination) {
if (throwable != null) {
logger.warn("Failed to send Twilio request", throwable);
return Optional.empty();
}
if (twilioVerifyResponse.isFailure()) {
String countryCode = Util.getCountryCode(destination);
String region = Util.getRegion(destination);
Metrics.counter(TwilioSmsSender.FAILED_REQUEST_COUNTER_NAME,
TwilioSmsSender.SERVICE_NAME_TAG, "verify",
TwilioSmsSender.STATUS_CODE_TAG_NAME, String.valueOf(twilioVerifyResponse.failureResponse.status),
TwilioSmsSender.ERROR_CODE_TAG_NAME, String.valueOf(twilioVerifyResponse.failureResponse.code),
TwilioSmsSender.COUNTRY_CODE_TAG_NAME, countryCode,
TwilioSmsSender.REGION_TAG_NAME, region,
TwilioSmsSender.REGION_CODE_TAG_NAME, region).increment();
logger.info("Failed with code={}, country={}",
twilioVerifyResponse.failureResponse.code,
countryCode);
return Optional.empty();
}
return Optional.ofNullable(twilioVerifyResponse.successResponse.getSid());
}
private HttpRequest buildVerifyRequest(String channel, String destination, String verificationCode,
Optional<String> locale, Optional<String> clientType) {
final Map<String, String> requestParameters = new HashMap<>();
requestParameters.put("To", destination);
requestParameters.put("CustomCode", verificationCode);
requestParameters.put("Channel", channel);
requestParameters.put("CustomFriendlyName", verifyServiceFriendlyName);
locale.ifPresent(loc -> requestParameters.put("Locale", loc));
clientType.filter(client -> client.startsWith("android"))
.ifPresent(ignored -> requestParameters.put("AppHash", androidAppHash));
return HttpRequest.newBuilder()
.uri(verifyServiceUri)
.POST(FormDataBodyPublisher.of(requestParameters))
.header("Content-Type", "application/x-www-form-urlencoded")
.header("Authorization",
"Basic " + Base64.getEncoder().encodeToString((accountId + ":" + accountToken).getBytes()))
.build();
}
public CompletableFuture<Boolean> reportVerificationSucceeded(String verificationSid, @Nullable String userAgent,
String context) {
final Map<String, String> requestParameters = new HashMap<>();
requestParameters.put("Status", "approved");
HttpRequest request = HttpRequest.newBuilder()
.uri(verifyApprovalBaseUri.resolve(verificationSid))
.POST(FormDataBodyPublisher.of(requestParameters))
.header("Content-Type", "application/x-www-form-urlencoded")
.header("Authorization",
"Basic " + Base64.getEncoder().encodeToString((accountId + ":" + accountToken).getBytes()))
.build();
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(this::parseResponse)
.handle((response, throwable) -> processVerificationSucceededResponse(response, throwable, userAgent, context));
}
private boolean processVerificationSucceededResponse(@Nullable final TwilioVerifyResponse response,
@Nullable final Throwable throwable,
final String userAgent,
final String context) {
if (throwable == null) {
assert response != null;
final Tags tags = Tags.of(Tag.of(CONTEXT_TAG_NAME, context), UserAgentTagUtil.getPlatformTag(userAgent));
if (response.isSuccess() && "approved".equals(response.successResponse.getStatus())) {
// the other possible values of `status` are `pending` or `canceled`, but these can never happen in a response
// to this POST, so we dont consider them
Metrics.counter(VERIFICATION_SUCCEEDED_RESPONSE_COUNTER_NAME, tags)
.increment();
return true;
}
// at this point, response.isFailure() == true
Metrics.counter(
VERIFICATION_SUCCEEDED_RESPONSE_COUNTER_NAME,
Tags.of(ERROR_CODE_TAG_NAME, String.valueOf(response.failureResponse.code),
STATUS_CODE_TAG_NAME, String.valueOf(response.failureResponse.status))
.and(tags))
.increment();
} else {
logger.warn("Failed to send verification succeeded", throwable);
}
return false;
}
public static class TwilioVerifyResponse {
private SuccessResponse successResponse;
private FailureResponse failureResponse;
TwilioVerifyResponse(SuccessResponse successResponse) {
this.successResponse = successResponse;
}
TwilioVerifyResponse(FailureResponse failureResponse) {
this.failureResponse = failureResponse;
}
boolean isSuccess() {
return successResponse != null;
}
boolean isFailure() {
return failureResponse != null;
}
private static class SuccessResponse {
@NotEmpty
public String sid;
@NotEmpty
public String status;
static SuccessResponse fromBody(ObjectMapper mapper, String body) {
try {
return mapper.readValue(body, SuccessResponse.class);
} catch (IOException e) {
logger.warn("Error parsing twilio success response: " + e);
return new SuccessResponse();
}
}
public String getSid() {
return sid;
}
public String getStatus() {
return status;
}
}
private static class FailureResponse {
@JsonProperty
private int status;
@JsonProperty
private String message;
@JsonProperty
private int code;
static FailureResponse fromBody(ObjectMapper mapper, String body) {
try {
return mapper.readValue(body, FailureResponse.class);
} catch (IOException e) {
logger.warn("Error parsing twilio response: " + e);
return new FailureResponse();
}
}
}
}
}

View File

@@ -16,7 +16,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
@@ -25,20 +24,26 @@ import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
import javax.annotation.Nonnull;
public class AbstractDynamoDbStore {
public abstract class AbstractDynamoDbStore {
private final DynamoDbClient dynamoDbClient;
private static final int MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE = 25; // This was arbitrarily chosen and may be entirely too high.
private final Timer batchWriteItemsFirstPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "true");
private final Timer batchWriteItemsRetryPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "false");
private final Counter batchWriteItemsUnprocessed = counter(name(getClass(), "batchWriteItemsUnprocessed"));
public static final int DYNAMO_DB_MAX_BATCH_SIZE = 25; // This limit comes from Amazon Dynamo DB itself. It will reject batch writes larger than this.
public static final int RESULT_SET_CHUNK_SIZE = 100;
private final Logger logger = LoggerFactory.getLogger(getClass());
private static final int MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE = 25; // This was arbitrarily chosen and may be entirely too high.
public static final int DYNAMO_DB_MAX_BATCH_SIZE = 25; // This limit comes from Amazon Dynamo DB itself. It will reject batch writes larger than this.
public static final int RESULT_SET_CHUNK_SIZE = 100;
private final Timer batchWriteItemsFirstPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "true");
private final Timer batchWriteItemsRetryPass = timer(name(getClass(), "batchWriteItems"), "firstAttempt", "false");
private final Counter batchWriteItemsUnprocessed = counter(name(getClass(), "batchWriteItemsUnprocessed"));
private final DynamoDbClient dynamoDbClient;
public AbstractDynamoDbStore(final DynamoDbClient dynamoDbClient) {
this.dynamoDbClient = dynamoDbClient;
@@ -49,18 +54,15 @@ public class AbstractDynamoDbStore {
}
protected void executeTableWriteItemsUntilComplete(final Map<String, List<WriteRequest>> items) {
AtomicReference<BatchWriteItemResponse> outcome = new AtomicReference<>();
batchWriteItemsFirstPass.record(
() -> outcome.set(dynamoDbClient.batchWriteItem(BatchWriteItemRequest.builder().requestItems(items).build())));
final AtomicReference<BatchWriteItemResponse> outcome = new AtomicReference<>();
writeAndStoreOutcome(items, batchWriteItemsFirstPass, outcome);
int attemptCount = 0;
while (!outcome.get().unprocessedItems().isEmpty() && attemptCount < MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE) {
batchWriteItemsRetryPass.record(() -> outcome.set(dynamoDbClient.batchWriteItem(BatchWriteItemRequest.builder()
.requestItems(outcome.get().unprocessedItems())
.build())));
writeAndStoreOutcome(outcome.get().unprocessedItems(), batchWriteItemsRetryPass, outcome);
++attemptCount;
}
if (!outcome.get().unprocessedItems().isEmpty()) {
int totalItems = outcome.get().unprocessedItems().values().stream().mapToInt(List::size).sum();
final int totalItems = outcome.get().unprocessedItems().values().stream().mapToInt(List::size).sum();
logger.error(
"Attempt count ({}) reached max ({}}) before applying all batch writes to dynamo. {} unprocessed items remain.",
attemptCount, MAX_ATTEMPTS_TO_SAVE_BATCH_WRITE, totalItems);
@@ -68,19 +70,28 @@ public class AbstractDynamoDbStore {
}
}
protected List<Map<String, AttributeValue>> scan(ScanRequest scanRequest, int max) {
@Nonnull
protected List<Map<String, AttributeValue>> scan(final ScanRequest scanRequest, final int max) {
return db().scanPaginator(scanRequest)
.items()
.stream()
.limit(max)
.collect(Collectors.toList());
.toList();
}
private void writeAndStoreOutcome(
final Map<String, List<WriteRequest>> items,
final Timer timer,
final AtomicReference<BatchWriteItemResponse> outcome) {
timer.record(
() -> outcome.set(dynamoDbClient.batchWriteItem(BatchWriteItemRequest.builder().requestItems(items).build()))
);
}
static <T> void writeInBatches(final Iterable<T> items, final Consumer<List<T>> action) {
final List<T> batch = new ArrayList<>(DYNAMO_DB_MAX_BATCH_SIZE);
for (T item : items) {
for (final T item : items) {
batch.add(item);
if (batch.size() == DYNAMO_DB_MAX_BATCH_SIZE) {

View File

@@ -1,52 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import java.time.Duration;
import com.google.common.annotations.VisibleForTesting;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.Constants;
public class AbusiveHostRules {
private static final String KEY_PREFIX = "abusive_hosts::";
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Timer getTimer = metricRegistry.timer(name(AbusiveHostRules.class, "get"));
private final Timer insertTimer = metricRegistry.timer(name(AbusiveHostRules.class, "setBlockedHost"));
private final FaultTolerantRedisCluster redisCluster;
private final DynamicConfigurationManager<DynamicConfiguration> configurationManager;
public AbusiveHostRules(FaultTolerantRedisCluster redisCluster, final DynamicConfigurationManager<DynamicConfiguration> configurationManager) {
this.redisCluster = redisCluster;
this.configurationManager = configurationManager;
}
public boolean isBlocked(String host) {
try (Timer.Context timer = getTimer.time()) {
return this.redisCluster.withCluster(connection -> connection.sync().exists(prefix(host))) > 0;
}
}
public void setBlockedHost(String host) {
Duration expireTime = configurationManager.getConfiguration().getAbusiveHostRules().getExpirationTime();
try (Timer.Context timer = insertTimer.time()) {
this.redisCluster.useCluster(connection -> connection.sync().setex(prefix(host), expireTime.toSeconds(), "1"));
}
}
@VisibleForTesting
public static String prefix(String keyName) {
return KEY_PREFIX + keyName;
}
}

View File

@@ -85,6 +85,7 @@ public class Account {
@JsonIgnore
private boolean canonicallyDiscoverable;
public UUID getUuid() {
// this is the one method that may be called on a stale account
return uuid;
@@ -180,14 +181,6 @@ public class Account {
return devices.stream().filter(device -> device.getId() == deviceId).findFirst();
}
public boolean isGroupsV2Supported() {
requireNotStale();
return devices.stream()
.filter(Device::isEnabled)
.allMatch(Device::isGroupsV2Supported);
}
public boolean isStorageSupported() {
requireNotStale();
@@ -200,10 +193,6 @@ public class Account {
return getMasterDevice().map(Device::getCapabilities).map(Device.DeviceCapabilities::isTransfer).orElse(false);
}
public boolean isGv1MigrationSupported() {
return allEnabledDevicesHaveCapability(DeviceCapabilities::isGv1Migration);
}
public boolean isSenderKeySupported() {
return allEnabledDevicesHaveCapability(DeviceCapabilities::isSenderKey);
}
@@ -225,15 +214,17 @@ public class Account {
return devices.stream()
.filter(Device::isEnabled)
// TODO stories capability
// .allMatch(device -> device.getCapabilities() != null && device.getCapabilities().isStories());
.anyMatch(device -> device.getCapabilities() != null && device.getCapabilities().isStories());
.allMatch(device -> device.getCapabilities() != null && device.getCapabilities().isStories());
}
public boolean isGiftBadgesSupported() {
return allEnabledDevicesHaveCapability(DeviceCapabilities::isGiftBadges);
}
public boolean isPaymentActivationSupported() {
return allEnabledDevicesHaveCapability(DeviceCapabilities::isPaymentActivation);
}
private boolean allEnabledDevicesHaveCapability(Predicate<DeviceCapabilities> predicate) {
requireNotStale();
@@ -306,16 +297,10 @@ public class Account {
public long getLastSeen() {
requireNotStale();
long lastSeen = 0;
for (Device device : devices) {
if (device.getLastSeen() > lastSeen) {
lastSeen = device.getLastSeen();
}
}
return lastSeen;
return devices.stream()
.map(Device::getLastSeen)
.max(Long::compare)
.orElse(0L);
}
public Optional<String> getCurrentProfileVersion() {
@@ -346,7 +331,6 @@ public class Account {
public void addBadge(Clock clock, AccountBadge badge) {
requireNotStale();
boolean added = false;
for (int i = 0; i < badges.size(); i++) {
AccountBadge badgeInList = badges.get(i);
@@ -480,6 +464,31 @@ public class Account {
this.version = version;
}
/**
* Have all this account's devices been manually locked?
*
* @see Device#hasLockedCredentials
*
* @return true if all the account's devices were locked, false otherwise.
*/
public boolean hasLockedCredentials() {
return devices.stream().allMatch(Device::hasLockedCredentials);
}
/**
* Lock account by invalidating authentication tokens.
*
* We only want to do this in cases where there is a potential conflict between the
* phone number holder and the registration lock holder. In that case, locking the
* account will ensure that either the registration lock holder proves ownership
* of the phone number, or after 7 days the phone number holder can register a new
* account.
*/
public void lockAuthenticationCredentials() {
devices.forEach(Device::lockAuthenticationCredentials);
}
boolean isStale() {
return stale;
}

View File

@@ -4,34 +4,32 @@
*/
package org.whispersystems.textsecuregcm.storage;
import com.google.common.annotations.VisibleForTesting;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.Util;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
public class AccountCleaner extends AccountDatabaseCrawlerListener {
private static final Logger log = LoggerFactory.getLogger(AccountCleaner.class);
private static final String DELETED_ACCOUNT_COUNTER_NAME = name(AccountCleaner.class, "deletedAccounts");
private static final String DELETION_REASON_TAG_NAME = "reason";
@VisibleForTesting
static final int MAX_ACCOUNT_DELETIONS_PER_CHUNK = 256;
private static final Counter DELETED_ACCOUNT_COUNTER = Metrics.counter(name(AccountCleaner.class, "deletedAccounts"));
private final AccountsManager accountsManager;
private final Executor deletionExecutor;
public AccountCleaner(AccountsManager accountsManager) {
public AccountCleaner(final AccountsManager accountsManager, final Executor deletionExecutor) {
this.accountsManager = accountsManager;
this.deletionExecutor = deletionExecutor;
}
@Override
@@ -44,43 +42,32 @@ public class AccountCleaner extends AccountDatabaseCrawlerListener {
@Override
protected void onCrawlChunk(Optional<UUID> fromUuid, List<Account> chunkAccounts) {
int accountUpdateCount = 0;
final List<CompletableFuture<Void>> deletionFutures = chunkAccounts.stream()
.filter(AccountCleaner::isExpired)
.map(account -> CompletableFuture.runAsync(() -> {
try {
accountsManager.delete(account, AccountsManager.DeletionReason.EXPIRED);
} catch (final InterruptedException e) {
throw new CompletionException(e);
}
}, deletionExecutor)
.whenComplete((ignored, throwable) -> {
if (throwable != null) {
log.warn("Failed to delete account {}", account.getUuid(), throwable);
} else {
DELETED_ACCOUNT_COUNTER.increment();
}
}))
.toList();
for (Account account : chunkAccounts) {
if (isExpired(account) || needsExplicitRemoval(account)) {
final Tag deletionReason;
if (needsExplicitRemoval(account)) {
deletionReason = Tag.of(DELETION_REASON_TAG_NAME, "newlyExpired");
} else {
deletionReason = Tag.of(DELETION_REASON_TAG_NAME, "previouslyExpired");
}
if (accountUpdateCount < MAX_ACCOUNT_DELETIONS_PER_CHUNK) {
try {
accountsManager.delete(account, AccountsManager.DeletionReason.EXPIRED);
accountUpdateCount++;
Metrics.counter(DELETED_ACCOUNT_COUNTER_NAME, Tags.of(deletionReason)).increment();
} catch (final Exception e) {
log.warn("Failed to delete account {}", account.getUuid(), e);
}
}
}
try {
CompletableFuture.allOf(deletionFutures.toArray(new CompletableFuture[0])).join();
} catch (final Exception e) {
log.debug("Failed to delete one or more accounts in chunk", e);
}
}
private boolean needsExplicitRemoval(Account account) {
return account.getMasterDevice().isPresent() &&
hasPushToken(account.getMasterDevice().get()) &&
isExpired(account);
}
private boolean hasPushToken(Device device) {
return !Util.isEmpty(device.getGcmId()) || !Util.isEmpty(device.getApnId()) || !Util.isEmpty(device.getVoipApnId()) || device.getFetchesMessages();
}
private boolean isExpired(Account account) {
private static boolean isExpired(Account account) {
return account.getLastSeen() + TimeUnit.DAYS.toMillis(365) < System.currentTimeMillis();
}

View File

@@ -5,6 +5,7 @@
package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import static java.util.Objects.requireNonNull;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.VisibleForTesting;
@@ -18,7 +19,6 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -29,14 +29,17 @@ import java.util.UUID;
import java.util.concurrent.CompletionException;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import org.whispersystems.textsecuregcm.util.UsernameNormalizer;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
@@ -56,8 +59,31 @@ import software.amazon.awssdk.services.dynamodb.model.Update;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
import software.amazon.awssdk.utils.CompletableFutureUtils;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
public class Accounts extends AbstractDynamoDbStore {
private static final Logger log = LoggerFactory.getLogger(Accounts.class);
private static final byte RESERVED_USERNAME_HASH_VERSION = 1;
private static final Timer CREATE_TIMER = Metrics.timer(name(Accounts.class, "create"));
private static final Timer CHANGE_NUMBER_TIMER = Metrics.timer(name(Accounts.class, "changeNumber"));
private static final Timer SET_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "setUsername"));
private static final Timer RESERVE_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "reserveUsername"));
private static final Timer CLEAR_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "clearUsername"));
private static final Timer UPDATE_TIMER = Metrics.timer(name(Accounts.class, "update"));
private static final Timer GET_BY_NUMBER_TIMER = Metrics.timer(name(Accounts.class, "getByNumber"));
private static final Timer GET_BY_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "getByUsername"));
private static final Timer GET_BY_PNI_TIMER = Metrics.timer(name(Accounts.class, "getByPni"));
private static final Timer GET_BY_UUID_TIMER = Metrics.timer(name(Accounts.class, "getByUuid"));
private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(Accounts.class, "getAllFrom"));
private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(Accounts.class, "getAllFromOffset"));
private static final Timer DELETE_TIMER = Metrics.timer(name(Accounts.class, "delete"));
private static final String CONDITIONAL_CHECK_FAILED = "ConditionalCheckFailed";
private static final String TRANSACTION_CONFLICT = "TransactionConflict";
// uuid, primary key
static final String KEY_ACCOUNT_UUID = "U";
// uuid, attribute on account table, primary key for PNI table
@@ -78,45 +104,32 @@ public class Accounts extends AbstractDynamoDbStore {
static final String ATTR_TTL = "TTL";
private final Clock clock;
private final DynamoDbClient client;
private final DynamoDbAsyncClient asyncClient;
private final String phoneNumberConstraintTableName;
private final String phoneNumberIdentifierConstraintTableName;
private final String usernamesConstraintTableName;
private final String accountsTableName;
private final int scanPageSize;
private static final byte RESERVED_USERNAME_HASH_VERSION = 1;
private static final Timer CREATE_TIMER = Metrics.timer(name(Accounts.class, "create"));
private static final Timer CHANGE_NUMBER_TIMER = Metrics.timer(name(Accounts.class, "changeNumber"));
private static final Timer SET_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "setUsername"));
private static final Timer RESERVE_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "reserveUsername"));
private static final Timer CLEAR_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "clearUsername"));
private static final Timer UPDATE_TIMER = Metrics.timer(name(Accounts.class, "update"));
private static final Timer GET_BY_NUMBER_TIMER = Metrics.timer(name(Accounts.class, "getByNumber"));
private static final Timer GET_BY_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "getByUsername"));
private static final Timer GET_BY_PNI_TIMER = Metrics.timer(name(Accounts.class, "getByPni"));
private static final Timer GET_BY_UUID_TIMER = Metrics.timer(name(Accounts.class, "getByUuid"));
private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(Accounts.class, "getAllFrom"));
private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(Accounts.class, "getAllFromOffset"));
private static final Timer DELETE_TIMER = Metrics.timer(name(Accounts.class, "delete"));
private static final Logger log = LoggerFactory.getLogger(Accounts.class);
@VisibleForTesting
public Accounts(
final Clock clock,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
DynamoDbClient client, DynamoDbAsyncClient asyncClient,
String accountsTableName, String phoneNumberConstraintTableName,
String phoneNumberIdentifierConstraintTableName, final String usernamesConstraintTableName,
final DynamoDbClient client,
final DynamoDbAsyncClient asyncClient,
final String accountsTableName,
final String phoneNumberConstraintTableName,
final String phoneNumberIdentifierConstraintTableName,
final String usernamesConstraintTableName,
final int scanPageSize) {
super(client);
this.clock = clock;
this.client = client;
this.asyncClient = asyncClient;
this.phoneNumberConstraintTableName = phoneNumberConstraintTableName;
this.phoneNumberIdentifierConstraintTableName = phoneNumberIdentifierConstraintTableName;
@@ -125,105 +138,61 @@ public class Accounts extends AbstractDynamoDbStore {
this.scanPageSize = scanPageSize;
}
public Accounts(final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
DynamoDbClient client, DynamoDbAsyncClient asyncClient,
String accountsTableName, String phoneNumberConstraintTableName,
String phoneNumberIdentifierConstraintTableName, final String usernamesConstraintTableName,
public Accounts(
final DynamoDbClient client,
final DynamoDbAsyncClient asyncClient,
final String accountsTableName,
final String phoneNumberConstraintTableName,
final String phoneNumberIdentifierConstraintTableName,
final String usernamesConstraintTableName,
final int scanPageSize) {
this(Clock.systemUTC(), dynamicConfigurationManager, client, asyncClient, accountsTableName,
this(Clock.systemUTC(), client, asyncClient, accountsTableName,
phoneNumberConstraintTableName, phoneNumberIdentifierConstraintTableName, usernamesConstraintTableName,
scanPageSize);
}
public boolean create(Account account) {
public boolean create(final Account account) {
return CREATE_TIMER.record(() -> {
try {
TransactWriteItem phoneNumberConstraintPut = TransactWriteItem.builder()
.put(
Put.builder()
.tableName(phoneNumberConstraintTableName)
.item(Map.of(
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.conditionExpression(
"attribute_not_exists(#number) OR (attribute_exists(#number) AND #uuid = :uuid)")
.expressionAttributeNames(
Map.of("#uuid", KEY_ACCOUNT_UUID,
"#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(
Map.of(":uuid", AttributeValues.fromUUID(account.getUuid())))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
final AttributeValue uuidAttr = AttributeValues.fromUUID(account.getUuid());
final AttributeValue numberAttr = AttributeValues.fromString(account.getNumber());
final AttributeValue pniUuidAttr = AttributeValues.fromUUID(account.getPhoneNumberIdentifier());
TransactWriteItem phoneNumberIdentifierConstraintPut = TransactWriteItem.builder()
.put(
Put.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.item(Map.of(
ATTR_PNI_UUID, AttributeValues.fromUUID(account.getPhoneNumberIdentifier()),
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.conditionExpression(
"attribute_not_exists(#pni) OR (attribute_exists(#pni) AND #uuid = :uuid)")
.expressionAttributeNames(
Map.of("#uuid", KEY_ACCOUNT_UUID,
"#pni", ATTR_PNI_UUID))
.expressionAttributeValues(
Map.of(":uuid", AttributeValues.fromUUID(account.getUuid())))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
final TransactWriteItem phoneNumberConstraintPut = buildConstraintTablePutIfAbsent(
phoneNumberConstraintTableName, uuidAttr, ATTR_ACCOUNT_E164, numberAttr);
final Map<String, AttributeValue> item = new HashMap<>(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
ATTR_PNI_UUID, AttributeValues.fromUUID(account.getPhoneNumberIdentifier()),
ATTR_ACCOUNT_DATA, AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
ATTR_VERSION, AttributeValues.fromInt(account.getVersion()),
ATTR_CANONICALLY_DISCOVERABLE, AttributeValues.fromBool(account.shouldBeVisibleInDirectory())));
final TransactWriteItem phoneNumberIdentifierConstraintPut = buildConstraintTablePutIfAbsent(
phoneNumberIdentifierConstraintTableName, uuidAttr, ATTR_PNI_UUID, pniUuidAttr);
// Add the UAK if it's in the account
account.getUnidentifiedAccessKey()
.map(AttributeValues::fromByteArray)
.ifPresent(uak -> item.put(ATTR_UAK, uak));
TransactWriteItem accountPut = TransactWriteItem.builder()
.put(Put.builder()
.conditionExpression("attribute_not_exists(#number) OR #number = :number")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(Map.of(":number", AttributeValues.fromString(account.getNumber())))
.tableName(accountsTableName)
.item(item)
.build())
.build();
final TransactWriteItem accountPut = buildAccountPut(account, uuidAttr, numberAttr, pniUuidAttr);
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(phoneNumberConstraintPut, phoneNumberIdentifierConstraintPut, accountPut)
.build();
try {
client.transactWriteItems(request);
} catch (TransactionCanceledException e) {
db().transactWriteItems(request);
} catch (final TransactionCanceledException e) {
final CancellationReason accountCancellationReason = e.cancellationReasons().get(2);
if ("ConditionalCheckFailed".equals(accountCancellationReason.code())) {
if (conditionalCheckFailed(accountCancellationReason)) {
throw new IllegalArgumentException("account identifier present with different phone number");
}
final CancellationReason phoneNumberConstraintCancellationReason = e.cancellationReasons().get(0);
final CancellationReason phoneNumberIdentifierConstraintCancellationReason = e.cancellationReasons().get(1);
if ("ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.code()) ||
"ConditionalCheckFailed".equals(phoneNumberIdentifierConstraintCancellationReason.code())) {
if (conditionalCheckFailed(phoneNumberConstraintCancellationReason)
|| conditionalCheckFailed(phoneNumberIdentifierConstraintCancellationReason)) {
// In theory, both reasons should trip in tandem and either should give us the information we need. Even so,
// we'll be cautious here and make sure we're choosing a condition check that really failed.
final CancellationReason reason = "ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.code()) ?
phoneNumberConstraintCancellationReason : phoneNumberIdentifierConstraintCancellationReason;
final CancellationReason reason = conditionalCheckFailed(phoneNumberConstraintCancellationReason)
? phoneNumberConstraintCancellationReason
: phoneNumberIdentifierConstraintCancellationReason;
ByteBuffer actualAccountUuid = reason.item().get(KEY_ACCOUNT_UUID).b().asByteBuffer();
final ByteBuffer actualAccountUuid = reason.item().get(KEY_ACCOUNT_UUID).b().asByteBuffer();
account.setUuid(UUIDUtil.fromByteBuffer(actualAccountUuid));
final Account existingAccount = getByAccountIdentifier(account.getUuid()).orElseThrow();
@@ -235,7 +204,7 @@ public class Accounts extends AbstractDynamoDbStore {
return false;
}
if ("TransactionConflict".equals(accountCancellationReason.code())) {
if (TRANSACTION_CONFLICT.equals(accountCancellationReason.code())) {
// this should only happen if two clients manage to make concurrent create() calls
throw new ContestedOptimisticLockException();
}
@@ -243,7 +212,7 @@ public class Accounts extends AbstractDynamoDbStore {
// this shouldn't happen
throw new RuntimeException("could not create account: " + extractCancellationReasonCodes(e));
}
} catch (JsonProcessingException e) {
} catch (final JsonProcessingException e) {
throw new IllegalArgumentException(e);
}
@@ -275,62 +244,34 @@ public class Accounts extends AbstractDynamoDbStore {
try {
final List<TransactWriteItem> writeItems = new ArrayList<>();
final AttributeValue uuidAttr = AttributeValues.fromUUID(account.getUuid());
final AttributeValue numberAttr = AttributeValues.fromString(number);
final AttributeValue pniAttr = AttributeValues.fromUUID(phoneNumberIdentifier);
writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberConstraintTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(originalNumber)))
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.put(Put.builder()
.tableName(phoneNumberConstraintTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_ACCOUNT_E164, AttributeValues.fromString(number)))
.conditionExpression("attribute_not_exists(#number)")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.key(Map.of(ATTR_PNI_UUID, AttributeValues.fromUUID(originalPni)))
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.put(Put.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_PNI_UUID, AttributeValues.fromUUID(phoneNumberIdentifier)))
.conditionExpression("attribute_not_exists(#pni)")
.expressionAttributeNames(Map.of("#pni", ATTR_PNI_UUID))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build());
writeItems.add(buildDelete(phoneNumberConstraintTableName, ATTR_ACCOUNT_E164, originalNumber));
writeItems.add(buildConstraintTablePut(phoneNumberConstraintTableName, uuidAttr, ATTR_ACCOUNT_E164, numberAttr));
writeItems.add(buildDelete(phoneNumberIdentifierConstraintTableName, ATTR_PNI_UUID, originalPni));
writeItems.add(buildConstraintTablePut(phoneNumberIdentifierConstraintTableName, uuidAttr, ATTR_PNI_UUID, pniAttr));
writeItems.add(
TransactWriteItem.builder()
.update(Update.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.updateExpression("SET #data = :data, #number = :number, #pni = :pni, #cds = :cds ADD #version :version_increment")
.conditionExpression("attribute_exists(#number) AND #version = :version")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
.key(Map.of(KEY_ACCOUNT_UUID, uuidAttr))
.updateExpression(
"SET #data = :data, #number = :number, #pni = :pni, #cds = :cds ADD #version :version_increment")
.conditionExpression(
"attribute_exists(#number) AND #version = :version")
.expressionAttributeNames(Map.of(
"#number", ATTR_ACCOUNT_E164,
"#data", ATTR_ACCOUNT_DATA,
"#cds", ATTR_CANONICALLY_DISCOVERABLE,
"#pni", ATTR_PNI_UUID,
"#version", ATTR_VERSION))
.expressionAttributeValues(Map.of(
":number", numberAttr,
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":number", AttributeValues.fromString(number),
":pni", AttributeValues.fromUUID(phoneNumberIdentifier),
":cds", AttributeValues.fromBool(account.shouldBeVisibleInDirectory()),
":pni", pniAttr,
":version", AttributeValues.fromInt(account.getVersion()),
":version_increment", AttributeValues.fromInt(1)))
.build())
@@ -340,7 +281,7 @@ public class Accounts extends AbstractDynamoDbStore {
.transactItems(writeItems)
.build();
client.transactWriteItems(request);
db().transactWriteItems(request);
account.setVersion(account.getVersion() + 1);
succeeded = true;
@@ -354,21 +295,6 @@ public class Accounts extends AbstractDynamoDbStore {
});
}
public static byte[] reservedUsernameHash(final UUID accountId, final String reservedUsername) {
final MessageDigest sha256;
try {
sha256 = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new AssertionError(e);
}
final ByteBuffer byteBuffer = ByteBuffer.allocate(32 + 1);
sha256.update(reservedUsername.getBytes(StandardCharsets.UTF_8));
sha256.update(UUIDUtil.toBytes(accountId));
byteBuffer.put(RESERVED_USERNAME_HASH_VERSION);
byteBuffer.put(sha256.digest());
return byteBuffer.array();
}
/**
* Reserve a username under a token
*
@@ -379,14 +305,13 @@ public class Accounts extends AbstractDynamoDbStore {
final String reservedUsername,
final Duration ttl) {
final long startNanos = System.nanoTime();
// if there is an existing old reservation it will be cleaned up via ttl
final Optional<byte[]> maybeOriginalReservation = account.getReservedUsernameHash();
account.setReservedUsernameHash(reservedUsernameHash(account.getUuid(), reservedUsername));
boolean succeeded = false;
long expirationTime = clock.instant().plus(ttl).getEpochSecond();
final long expirationTime = clock.instant().plus(ttl).getEpochSecond();
final UUID reservationToken = UUID.randomUUID();
try {
@@ -397,7 +322,7 @@ public class Accounts extends AbstractDynamoDbStore {
.tableName(usernamesConstraintTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(reservationToken),
ATTR_USERNAME, AttributeValues.fromString(reservedUsername),
ATTR_USERNAME, AttributeValues.fromString(UsernameNormalizer.normalize(reservedUsername)),
ATTR_TTL, AttributeValues.fromLong(expirationTime)))
.conditionExpression("attribute_not_exists(#username) OR (#ttl < :now)")
.expressionAttributeNames(Map.of("#username", ATTR_USERNAME, "#ttl", ATTR_TTL))
@@ -425,14 +350,14 @@ public class Accounts extends AbstractDynamoDbStore {
.transactItems(writeItems)
.build();
client.transactWriteItems(request);
db().transactWriteItems(request);
account.setVersion(account.getVersion() + 1);
succeeded = true;
} catch (final JsonProcessingException e) {
throw new IllegalArgumentException(e);
} catch (final TransactionCanceledException e) {
if (e.cancellationReasons().stream().map(CancellationReason::code).anyMatch("ConditionalCheckFailed"::equals)) {
if (e.cancellationReasons().stream().map(CancellationReason::code).anyMatch(CONDITIONAL_CHECK_FAILED::equals)) {
throw new ContestedOptimisticLockException();
}
throw e;
@@ -486,12 +411,13 @@ public class Accounts extends AbstractDynamoDbStore {
final List<TransactWriteItem> writeItems = new ArrayList<>();
// add the username to the constraint table, wiping out the ttl if we had already reserved the name
// Persist the normalized username in the usernamesConstraint table and the original username in the accounts table
writeItems.add(TransactWriteItem.builder()
.put(Put.builder()
.tableName(usernamesConstraintTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_USERNAME, AttributeValues.fromString(username)))
ATTR_USERNAME, AttributeValues.fromString(UsernameNormalizer.normalize(username))))
// it's not in the constraint table OR it's expired OR it was reserved by us
.conditionExpression("attribute_not_exists(#username) OR #ttl < :now OR #aci = :reservation ")
.expressionAttributeNames(Map.of("#username", ATTR_USERNAME, "#ttl", ATTR_TTL, "#aci", KEY_ACCOUNT_UUID))
@@ -520,25 +446,21 @@ public class Accounts extends AbstractDynamoDbStore {
.build())
.build());
maybeOriginalUsername.ifPresent(originalUsername -> writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(originalUsername)))
.build())
.build()));
maybeOriginalUsername.ifPresent(originalUsername -> writeItems.add(
buildDelete(usernamesConstraintTableName, ATTR_USERNAME, UsernameNormalizer.normalize(originalUsername))));
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(writeItems)
.build();
client.transactWriteItems(request);
db().transactWriteItems(request);
account.setVersion(account.getVersion() + 1);
succeeded = true;
} catch (final JsonProcessingException e) {
throw new IllegalArgumentException(e);
} catch (final TransactionCanceledException e) {
if (e.cancellationReasons().stream().map(CancellationReason::code).anyMatch("ConditionalCheckFailed"::equals)) {
if (e.cancellationReasons().stream().map(CancellationReason::code).anyMatch(CONDITIONAL_CHECK_FAILED::equals)) {
throw new ContestedOptimisticLockException();
}
throw e;
@@ -551,7 +473,7 @@ public class Accounts extends AbstractDynamoDbStore {
}
}
public void clearUsername(Account account) {
public void clearUsername(final Account account) {
account.getUsername().ifPresent(username -> {
CLEAR_USERNAME_TIMER.record(() -> {
account.setUsername(null);
@@ -578,25 +500,20 @@ public class Accounts extends AbstractDynamoDbStore {
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(username)))
.build())
.build());
writeItems.add(buildDelete(usernamesConstraintTableName, ATTR_USERNAME, UsernameNormalizer.normalize(username)));
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(writeItems)
.build();
client.transactWriteItems(request);
db().transactWriteItems(request);
account.setVersion(account.getVersion() + 1);
succeeded = true;
} catch (final JsonProcessingException e) {
throw new IllegalArgumentException(e);
} catch (final TransactionCanceledException e) {
if ("ConditionalCheckFailed".equals(e.cancellationReasons().get(0).code())) {
if (conditionalCheckFailed(e.cancellationReasons().get(0))) {
throw new ContestedOptimisticLockException();
}
@@ -610,27 +527,18 @@ public class Accounts extends AbstractDynamoDbStore {
});
}
/**
* Extract the cause from a CompletionException
*/
private static Throwable unwrap(Throwable throwable) {
while (throwable instanceof CompletionException e && throwable.getCause() != null) {
throwable = e.getCause();
}
return throwable;
}
public CompletionStage<Void> updateAsync(Account account) {
@Nonnull
public CompletionStage<Void> updateAsync(final Account account) {
return record(UPDATE_TIMER, () -> {
final UpdateItemRequest updateItemRequest;
try {
// username, e164, and pni cannot be modified through this method
Map<String, String> attrNames = new HashMap<>(Map.of(
final Map<String, String> attrNames = new HashMap<>(Map.of(
"#number", ATTR_ACCOUNT_E164,
"#data", ATTR_ACCOUNT_DATA,
"#cds", ATTR_CANONICALLY_DISCOVERABLE,
"#version", ATTR_VERSION));
Map<String, AttributeValue> attrValues = new HashMap<>(Map.of(
final Map<String, AttributeValue> attrValues = new HashMap<>(Map.of(
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":cds", AttributeValues.fromBool(account.shouldBeVisibleInDirectory()),
":version", AttributeValues.fromInt(account.getVersion()),
@@ -654,7 +562,7 @@ public class Accounts extends AbstractDynamoDbStore {
.expressionAttributeNames(attrNames)
.expressionAttributeValues(attrValues)
.build();
} catch (JsonProcessingException e) {
} catch (final JsonProcessingException e) {
throw new IllegalArgumentException(e);
}
@@ -664,7 +572,7 @@ public class Accounts extends AbstractDynamoDbStore {
return (Void) null;
})
.exceptionally(throwable -> {
final Throwable unwrapped = unwrap(throwable);
final Throwable unwrapped = ExceptionUtils.unwrap(throwable);
if (unwrapped instanceof TransactionConflictException) {
throw new ContestedOptimisticLockException();
} else if (unwrapped instanceof ConditionalCheckFailedException e) {
@@ -679,12 +587,12 @@ public class Accounts extends AbstractDynamoDbStore {
});
}
public void update(Account account) throws ContestedOptimisticLockException {
public void update(final Account account) throws ContestedOptimisticLockException {
try {
this.updateAsync(account).toCompletableFuture().join();
} catch (CompletionException e) {
updateAsync(account).toCompletableFuture().join();
} catch (final CompletionException e) {
// unwrap CompletionExceptions, throw as long is it's unchecked
Throwables.throwIfUnchecked(unwrap(e));
Throwables.throwIfUnchecked(ExceptionUtils.unwrap(e));
// if we otherwise somehow got a wrapped checked exception,
// rethrow the checked exception wrapped by the original CompletionException
@@ -698,15 +606,14 @@ public class Accounts extends AbstractDynamoDbStore {
}
public boolean usernameAvailable(final Optional<UUID> reservationToken, final String username) {
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(username)))
.build());
if (!response.hasItem()) {
final Optional<Map<String, AttributeValue>> usernameItem = itemByKey(
usernamesConstraintTableName, ATTR_USERNAME, AttributeValues.fromString(UsernameNormalizer.normalize(username)));
if (usernameItem.isEmpty()) {
// username is free
return true;
}
final Map<String, AttributeValue> item = response.item();
final Map<String, AttributeValue> item = usernameItem.get();
if (AttributeValues.getLong(item, ATTR_TTL, Long.MAX_VALUE) < clock.instant().getEpochSecond()) {
// username was reserved, but has expired
@@ -719,112 +626,55 @@ public class Accounts extends AbstractDynamoDbStore {
.orElse(false);
}
public Optional<Account> getByE164(String number) {
return GET_BY_NUMBER_TIMER.record(() -> {
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(phoneNumberConstraintTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(number)))
.build());
return Optional.ofNullable(response.item())
.map(item -> item.get(KEY_ACCOUNT_UUID))
.map(this::accountByUuid)
.map(Accounts::fromItem);
});
}
public Optional<Account> getByUsername(final String username) {
return GET_BY_USERNAME_TIMER.record(() -> {
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(username)))
.build());
return Optional.ofNullable(response.item())
// ignore items with a ttl (reservations)
.filter(item -> !item.containsKey(ATTR_TTL))
.map(item -> item.get(KEY_ACCOUNT_UUID))
.map(this::accountByUuid)
.map(Accounts::fromItem);
});
@Nonnull
public Optional<Account> getByE164(final String number) {
return getByIndirectLookup(
GET_BY_NUMBER_TIMER, phoneNumberConstraintTableName, ATTR_ACCOUNT_E164, AttributeValues.fromString(number));
}
@Nonnull
public Optional<Account> getByPhoneNumberIdentifier(final UUID phoneNumberIdentifier) {
return GET_BY_PNI_TIMER.record(() -> {
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.key(Map.of(ATTR_PNI_UUID, AttributeValues.fromUUID(phoneNumberIdentifier)))
.build());
return Optional.ofNullable(response.item())
.map(item -> item.get(KEY_ACCOUNT_UUID))
.map(this::accountByUuid)
.map(Accounts::fromItem);
});
return getByIndirectLookup(
GET_BY_PNI_TIMER, phoneNumberIdentifierConstraintTableName, ATTR_PNI_UUID, AttributeValues.fromUUID(phoneNumberIdentifier));
}
private Map<String, AttributeValue> accountByUuid(AttributeValue uuid) {
GetItemResponse r = client.getItem(GetItemRequest.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, uuid))
.consistentRead(true)
.build());
return r.item().isEmpty() ? null : r.item();
@Nonnull
public Optional<Account> getByUsername(final String username) {
return getByIndirectLookup(
GET_BY_USERNAME_TIMER,
usernamesConstraintTableName,
ATTR_USERNAME,
AttributeValues.fromString(UsernameNormalizer.normalize(username)),
item -> !item.containsKey(ATTR_TTL) // ignore items with a ttl (reservations)
);
}
public Optional<Account> getByAccountIdentifier(UUID uuid) {
return GET_BY_UUID_TIMER.record(() ->
Optional.ofNullable(accountByUuid(AttributeValues.fromUUID(uuid)))
.map(Accounts::fromItem));
@Nonnull
public Optional<Account> getByAccountIdentifier(final UUID uuid) {
return requireNonNull(GET_BY_UUID_TIMER.record(() ->
itemByKey(accountsTableName, KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid))
.map(Accounts::fromItem)));
}
public void delete(UUID uuid) {
DELETE_TIMER.record(() -> {
public void delete(final UUID uuid) {
DELETE_TIMER.record(() -> getByAccountIdentifier(uuid).ifPresent(account -> {
getByAccountIdentifier(uuid).ifPresent(account -> {
final List<TransactWriteItem> transactWriteItems = new ArrayList<>(List.of(
buildDelete(phoneNumberConstraintTableName, ATTR_ACCOUNT_E164, account.getNumber()),
buildDelete(accountsTableName, KEY_ACCOUNT_UUID, uuid),
buildDelete(phoneNumberIdentifierConstraintTableName, ATTR_PNI_UUID, account.getPhoneNumberIdentifier())
));
TransactWriteItem phoneNumberDelete = TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberConstraintTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber())))
.build())
.build();
account.getUsername().ifPresent(username -> transactWriteItems.add(
buildDelete(usernamesConstraintTableName, ATTR_USERNAME, UsernameNormalizer.normalize(username))));
TransactWriteItem accountDelete = TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
.build())
.build();
final List<TransactWriteItem> transactWriteItems = new ArrayList<>(List.of(phoneNumberDelete, accountDelete));
transactWriteItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.key(Map.of(ATTR_PNI_UUID, AttributeValues.fromUUID(account.getPhoneNumberIdentifier())))
.build())
.build());
account.getUsername().ifPresent(username -> transactWriteItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(username)))
.build())
.build()));
TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(transactWriteItems).build();
client.transactWriteItems(request);
});
});
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(transactWriteItems).build();
db().transactWriteItems(request);
}));
}
@Nonnull
public AccountCrawlChunk getAllFrom(final UUID from, final int maxCount) {
final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder()
.limit(scanPageSize)
@@ -833,6 +683,7 @@ public class Accounts extends AbstractDynamoDbStore {
return scanForChunk(scanRequestBuilder, maxCount, GET_ALL_FROM_OFFSET_TIMER);
}
@Nonnull
public AccountCrawlChunk getAllFromStart(final int maxCount) {
final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder()
.limit(scanPageSize);
@@ -840,34 +691,185 @@ public class Accounts extends AbstractDynamoDbStore {
return scanForChunk(scanRequestBuilder, maxCount, GET_ALL_FROM_START_TIMER);
}
private static <T> CompletionStage<T> record(final Timer timer, Supplier<CompletionStage<T>> toRecord) {
final Instant start = Instant.now();
return toRecord.get().whenComplete((ignoreT, ignoreE) -> timer.record(Duration.between(start, Instant.now())));
@Nonnull
private Optional<Account> getByIndirectLookup(
final Timer timer,
final String tableName,
final String keyName,
final AttributeValue keyValue) {
return getByIndirectLookup(timer, tableName, keyName, keyValue, i -> true);
}
@Nonnull
private Optional<Account> getByIndirectLookup(
final Timer timer,
final String tableName,
final String keyName,
final AttributeValue keyValue,
final Predicate<? super Map<String, AttributeValue>> predicate) {
return requireNonNull(timer.record(() -> itemByKey(tableName, keyName, keyValue)
.filter(predicate)
.map(item -> item.get(KEY_ACCOUNT_UUID))
.flatMap(uuid -> itemByKey(accountsTableName, KEY_ACCOUNT_UUID, uuid))
.map(Accounts::fromItem)));
}
@Nonnull
private Optional<Map<String, AttributeValue>> itemByKey(final String table, final String keyName, final AttributeValue keyValue) {
final GetItemResponse response = db().getItem(GetItemRequest.builder()
.tableName(table)
.key(Map.of(keyName, keyValue))
.consistentRead(true)
.build());
return Optional.ofNullable(response.item()).filter(m -> !m.isEmpty());
}
@Nonnull
private TransactWriteItem buildAccountPut(
final Account account,
final AttributeValue uuidAttr,
final AttributeValue numberAttr,
final AttributeValue pniUuidAttr) throws JsonProcessingException {
final Map<String, AttributeValue> item = new HashMap<>(Map.of(
KEY_ACCOUNT_UUID, uuidAttr,
ATTR_ACCOUNT_E164, numberAttr,
ATTR_PNI_UUID, pniUuidAttr,
ATTR_ACCOUNT_DATA, AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
ATTR_VERSION, AttributeValues.fromInt(account.getVersion()),
ATTR_CANONICALLY_DISCOVERABLE, AttributeValues.fromBool(account.shouldBeVisibleInDirectory())));
// Add the UAK if it's in the account
account.getUnidentifiedAccessKey()
.map(AttributeValues::fromByteArray)
.ifPresent(uak -> item.put(ATTR_UAK, uak));
return TransactWriteItem.builder()
.put(Put.builder()
.conditionExpression("attribute_not_exists(#number) OR #number = :number")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(Map.of(":number", numberAttr))
.tableName(accountsTableName)
.item(item)
.build())
.build();
}
@Nonnull
private static TransactWriteItem buildConstraintTablePutIfAbsent(
final String tableName,
final AttributeValue uuidAttr,
final String keyName,
final AttributeValue keyValue
) {
return TransactWriteItem.builder()
.put(Put.builder()
.tableName(tableName)
.item(Map.of(
keyName, keyValue,
KEY_ACCOUNT_UUID, uuidAttr))
.conditionExpression(
"attribute_not_exists(#key) OR #uuid = :uuid")
.expressionAttributeNames(Map.of(
"#key", keyName,
"#uuid", KEY_ACCOUNT_UUID))
.expressionAttributeValues(Map.of(
":uuid", uuidAttr))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
}
@Nonnull
private static TransactWriteItem buildConstraintTablePut(
final String tableName,
final AttributeValue uuidAttr,
final String keyName,
final AttributeValue keyValue) {
return TransactWriteItem.builder()
.put(Put.builder()
.tableName(tableName)
.item(Map.of(
keyName, keyValue,
KEY_ACCOUNT_UUID, uuidAttr))
.conditionExpression(
"attribute_not_exists(#key)")
.expressionAttributeNames(Map.of(
"#key", keyName))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
}
@Nonnull
private static TransactWriteItem buildDelete(final String tableName, final String keyName, final String keyValue) {
return buildDelete(tableName, keyName, AttributeValues.fromString(keyValue));
}
@Nonnull
private static TransactWriteItem buildDelete(final String tableName, final String keyName, final UUID keyValue) {
return buildDelete(tableName, keyName, AttributeValues.fromUUID(keyValue));
}
@Nonnull
private static TransactWriteItem buildDelete(final String tableName, final String keyName, final AttributeValue keyValue) {
return TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(tableName)
.key(Map.of(keyName, keyValue))
.build())
.build();
}
@Nonnull
private static <T> CompletionStage<T> record(final Timer timer, final Supplier<CompletionStage<T>> toRecord) {
final Timer.Sample sample = Timer.start();
return toRecord.get().whenComplete((ignoreT, ignoreE) -> sample.stop(timer));
}
@Nonnull
private AccountCrawlChunk scanForChunk(final ScanRequest.Builder scanRequestBuilder, final int maxCount, final Timer timer) {
scanRequestBuilder.tableName(accountsTableName);
final List<Map<String, AttributeValue>> items = timer.record(() -> scan(scanRequestBuilder.build(), maxCount));
final List<Map<String, AttributeValue>> items = requireNonNull(timer.record(() -> scan(scanRequestBuilder.build(), maxCount)));
final List<Account> accounts = items.stream().map(Accounts::fromItem).toList();
return new AccountCrawlChunk(accounts, accounts.size() > 0 ? accounts.get(accounts.size() - 1).getUuid() : null);
}
@Nonnull
private static String extractCancellationReasonCodes(final TransactionCanceledException exception) {
return exception.cancellationReasons().stream()
.map(CancellationReason::code)
.collect(Collectors.joining(", "));
}
@Nonnull
public static byte[] reservedUsernameHash(final UUID accountId, final String reservedUsername) {
final MessageDigest sha256;
try {
sha256 = MessageDigest.getInstance("SHA-256");
} catch (final NoSuchAlgorithmException e) {
throw new AssertionError(e);
}
final ByteBuffer byteBuffer = ByteBuffer.allocate(32 + 1);
sha256.update(UsernameNormalizer.normalize(reservedUsername).getBytes(StandardCharsets.UTF_8));
sha256.update(UUIDUtil.toBytes(accountId));
byteBuffer.put(RESERVED_USERNAME_HASH_VERSION);
byteBuffer.put(sha256.digest());
return byteBuffer.array();
}
@VisibleForTesting
static Account fromItem(Map<String, AttributeValue> item) {
if (!item.containsKey(ATTR_ACCOUNT_DATA) ||
!item.containsKey(ATTR_ACCOUNT_E164) ||
// TODO: eventually require ATTR_CANONICALLY_DISCOVERABLE
!item.containsKey(KEY_ACCOUNT_UUID)) {
@Nonnull
static Account fromItem(final Map<String, AttributeValue> item) {
// TODO: eventually require ATTR_CANONICALLY_DISCOVERABLE
if (!item.containsKey(ATTR_ACCOUNT_DATA)
|| !item.containsKey(ATTR_ACCOUNT_E164)
|| !item.containsKey(KEY_ACCOUNT_UUID)) {
throw new RuntimeException("item missing values");
}
try {
Account account = SystemMapper.getMapper().readValue(item.get(ATTR_ACCOUNT_DATA).b().asByteArray(), Account.class);
final Account account = SystemMapper.getMapper().readValue(item.get(ATTR_ACCOUNT_DATA).b().asByteArray(), Account.class);
final UUID accountIdentifier = UUIDUtil.fromByteBuffer(item.get(KEY_ACCOUNT_UUID).b().asByteBuffer());
final UUID phoneNumberIdentifierFromAttribute = AttributeValues.getUUID(item, ATTR_PNI_UUID, null);
@@ -883,12 +885,18 @@ public class Accounts extends AbstractDynamoDbStore {
account.setUuid(accountIdentifier);
account.setUsername(AttributeValues.getString(item, ATTR_USERNAME, null));
account.setVersion(Integer.parseInt(item.get(ATTR_VERSION).n()));
account.setCanonicallyDiscoverable(Optional.ofNullable(item.get(ATTR_CANONICALLY_DISCOVERABLE)).map(av -> av.bool()).orElse(false));
account.setCanonicallyDiscoverable(Optional.ofNullable(item.get(ATTR_CANONICALLY_DISCOVERABLE))
.map(AttributeValue::bool)
.orElse(false));
return account;
} catch (IOException e) {
} catch (final IOException e) {
throw new RuntimeException("Could not read stored account data", e);
}
}
private static boolean conditionalCheckFailed(final CancellationReason reason) {
return CONDITIONAL_CHECK_FAILED.equals(reason.code());
}
}

View File

@@ -51,6 +51,7 @@ import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.DestinationDeviceValidator;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.UsernameGenerator;
import org.whispersystems.textsecuregcm.util.UsernameNormalizer;
import org.whispersystems.textsecuregcm.util.Util;
public class AccountsManager {
@@ -391,7 +392,7 @@ public class AccountsManager {
return account;
}
final byte[] newHash = Accounts.reservedUsernameHash(account.getUuid(), reservedUsername);
final byte[] newHash = Accounts.reservedUsernameHash(account.getUuid(), UsernameNormalizer.normalize(reservedUsername));
if (!account.getReservedUsernameHash().map(oldHash -> Arrays.equals(oldHash, newHash)).orElse(false)) {
// no such reservation existed, either there was no previous call to reserveUsername
// or the reservation changed
@@ -720,8 +721,8 @@ public class AccountsManager {
clientPresenceManager.disconnectPresence(account.getUuid(), device.getId())));
}
private String getUsernameAccountMapKey(String key) {
return "UAccountMap::" + key;
private String getUsernameAccountMapKey(String username) {
return "UAccountMap::" + UsernameNormalizer.normalize(username);
}
private String getAccountMapKey(String key) {

View File

@@ -149,6 +149,33 @@ public class Device {
this.salt = credentials.getSalt();
}
/**
* Has this device been manually locked?
*
* We lock a device by prepending "!" to its token.
* This character cannot normally appear in valid tokens.
*
* @return true if the credential was locked, false otherwise.
*/
public boolean hasLockedCredentials() {
AuthenticationCredentials auth = getAuthenticationCredentials();
return auth.getHashedAuthenticationToken().startsWith("!");
}
/**
* Lock device by invalidating authentication tokens.
*
* This should only be used from Account::lockAuthenticationCredentials.
*
* See that method for more information.
*/
public void lockAuthenticationCredentials() {
AuthenticationCredentials oldAuth = getAuthenticationCredentials();
String token = "!" + oldAuth.getHashedAuthenticationToken();
String salt = oldAuth.getSalt();
setAuthenticationCredentials(new AuthenticationCredentials(token, salt));
}
public AuthenticationCredentials getAuthenticationCredentials() {
return new AuthenticationCredentials(authToken, salt);
}
@@ -225,39 +252,13 @@ public class Device {
return this.userAgent;
}
public boolean isGroupsV2Supported() {
final boolean groupsV2Supported;
if (this.capabilities != null) {
final boolean ios = this.apnId != null || this.voipApnId != null;
groupsV2Supported = this.capabilities.isGv2_3() || (ios && this.capabilities.isGv2_2());
} else {
groupsV2Supported = false;
}
return groupsV2Supported;
}
public static class DeviceCapabilities {
@JsonProperty
private boolean gv2;
@JsonProperty("gv2-2")
private boolean gv2_2;
@JsonProperty("gv2-3")
private boolean gv2_3;
@JsonProperty
private boolean storage;
@JsonProperty
private boolean transfer;
@JsonProperty("gv1-migration")
private boolean gv1Migration;
@JsonProperty
private boolean senderKey;
@@ -276,36 +277,24 @@ public class Device {
@JsonProperty
private boolean giftBadges;
@JsonProperty
private boolean paymentActivation;
public DeviceCapabilities() {
}
public DeviceCapabilities(boolean gv2, final boolean gv2_2, final boolean gv2_3, boolean storage, boolean transfer,
boolean gv1Migration, final boolean senderKey, final boolean announcementGroup, final boolean changeNumber,
final boolean pni, final boolean stories, final boolean giftBadges) {
this.gv2 = gv2;
this.gv2_2 = gv2_2;
this.gv2_3 = gv2_3;
public DeviceCapabilities(boolean storage, boolean transfer,
final boolean senderKey, final boolean announcementGroup, final boolean changeNumber,
final boolean pni, final boolean stories, final boolean giftBadges, final boolean paymentActivation) {
this.storage = storage;
this.transfer = transfer;
this.gv1Migration = gv1Migration;
this.senderKey = senderKey;
this.announcementGroup = announcementGroup;
this.changeNumber = changeNumber;
this.pni = pni;
this.stories = stories;
this.giftBadges = giftBadges;
}
public boolean isGv2() {
return gv2;
}
public boolean isGv2_2() {
return gv2_2;
}
public boolean isGv2_3() {
return gv2_3;
this.paymentActivation = paymentActivation;
}
public boolean isStorage() {
@@ -316,10 +305,6 @@ public class Device {
return transfer;
}
public boolean isGv1Migration() {
return gv1Migration;
}
public boolean isSenderKey() {
return senderKey;
}
@@ -343,5 +328,9 @@ public class Device {
public boolean isGiftBadges() {
return giftBadges;
}
public boolean isPaymentActivation() {
return paymentActivation;
}
}
}

View File

@@ -4,9 +4,6 @@
*/
package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.SharedMetricRegistries;
import java.security.KeyStore;
import java.security.cert.CertificateException;
import javax.net.ssl.SSLContext;
@@ -19,9 +16,7 @@ import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration;
import org.whispersystems.textsecuregcm.entities.DirectoryReconciliationRequest;
import org.whispersystems.textsecuregcm.entities.DirectoryReconciliationResponse;
import org.whispersystems.textsecuregcm.util.CertificateExpirationGauge;
import org.whispersystems.textsecuregcm.util.CertificateUtil;
import org.whispersystems.textsecuregcm.util.Constants;
public class DirectoryReconciliationClient {
@@ -33,10 +28,6 @@ public class DirectoryReconciliationClient {
{
this.replicationUrl = directoryServerConfiguration.getReplicationUrl();
this.client = initializeClient(directoryServerConfiguration);
SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME)
.register(name(getClass(), directoryServerConfiguration.getReplicationName(), "days_until_certificate_expiration"),
new CertificateExpirationGauge(CertificateUtil.getCertificate(directoryServerConfiguration.getReplicationCaCertificate())));
}
public DirectoryReconciliationResponse add(DirectoryReconciliationRequest request) {
@@ -63,7 +54,7 @@ public class DirectoryReconciliationClient {
private static Client initializeClient(DirectoryServerConfiguration directoryServerConfiguration)
throws CertificateException {
KeyStore trustStore = CertificateUtil.buildKeyStoreForPem(
directoryServerConfiguration.getReplicationCaCertificate());
directoryServerConfiguration.getReplicationCaCertificates().toArray(new String[0]));
SSLContext sslContext = SslConfigurator.newInstance()
.securityProtocol("TLSv1.2")
.trustStore(trustStore)

View File

@@ -26,14 +26,16 @@ import javax.crypto.spec.SecretKeySpec;
import javax.ws.rs.ClientErrorException;
import javax.ws.rs.core.Response.Status;
import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialRequest;
import org.whispersystems.textsecuregcm.subscriptions.SubscriptionProcessor;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
public class IssuedReceiptsManager {
public static final String KEY_STRIPE_ID = "A"; // S (HashKey)
public static final String KEY_PROCESSOR_ITEM_ID = "A"; // S (HashKey)
public static final String KEY_ISSUED_RECEIPT_TAG = "B"; // B
public static final String KEY_EXPIRATION = "E"; // N
@@ -54,29 +56,39 @@ public class IssuedReceiptsManager {
}
/**
* Returns a future that completes normally if either this stripe item was never issued a receipt credential
* Returns a future that completes normally if either this processor item was never issued a receipt credential
* previously OR if it was issued a receipt credential previously for the exact same receipt credential request
* enabling clients to retry in case they missed the original response.
*
* If this stripe item has already been used to issue another receipt, throws a 409 conflict web application
* exception.
*
* Stripe item is expected to refer to an invoice line item (subscriptions) or a payment intent (one-time).
* <p>
* If this item has already been used to issue another receipt, throws a 409 conflict web application exception.
* <p>
* For {@link SubscriptionProcessor#STRIPE}, item is expected to refer to an invoice line item (subscriptions) or a
* payment intent (one-time).
*/
public CompletableFuture<Void> recordIssuance(
String stripeId,
String processorItemId,
SubscriptionProcessor processor,
ReceiptCredentialRequest request,
Instant now) {
final AttributeValue key;
if (processor == SubscriptionProcessor.STRIPE) {
// As the first processor, Stripes IDs were not prefixed. Its item IDs have documented prefixes (`il_`, `pi_`)
// that will not collide with `SubscriptionProcessor` names
key = s(processorItemId);
} else {
key = s(processor.name() + "_" + processorItemId);
}
UpdateItemRequest updateItemRequest = UpdateItemRequest.builder()
.tableName(table)
.key(Map.of(KEY_STRIPE_ID, s(stripeId)))
.key(Map.of(KEY_PROCESSOR_ITEM_ID, key))
.conditionExpression("attribute_not_exists(#key) OR #tag = :tag")
.returnValues(ReturnValue.NONE)
.updateExpression("SET "
+ "#tag = if_not_exists(#tag, :tag), "
+ "#exp = if_not_exists(#exp, :exp)")
.expressionAttributeNames(Map.of(
"#key", KEY_STRIPE_ID,
"#key", KEY_PROCESSOR_ITEM_ID,
"#tag", KEY_ISSUED_RECEIPT_TAG,
"#exp", KEY_EXPIRATION))
.expressionAttributeValues(Map.of(

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
@@ -22,6 +22,7 @@ import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
@@ -34,23 +35,36 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.reactivestreams.Publisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.redis.ClusterLuaScript;
import org.whispersystems.textsecuregcm.redis.FaultTolerantPubSubConnection;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.Pair;
import org.whispersystems.textsecuregcm.util.RedisClusterUtil;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
public class MessagesCache extends RedisClusterPubSubAdapter<String, String> implements Managed {
private final FaultTolerantRedisCluster readDeleteCluster;
private final FaultTolerantPubSubConnection<String, String> pubSubConnection;
private final Clock clock;
private final ExecutorService notificationExecutorService;
private final ExecutorService messageDeletionExecutorService;
// messageDeletionExecutorService wrapped into a reactor Scheduler
private final Scheduler messageDeletionScheduler;
private final ClusterLuaScript insertScript;
private final ClusterLuaScript removeByGuidScript;
@@ -79,22 +93,25 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
private static final String QUEUE_KEYSPACE_PREFIX = "__keyspace@0__:user_queue::";
private static final String PERSISTING_KEYSPACE_PREFIX = "__keyspace@0__:user_queue_persisting::";
private static final Duration MAX_EPHEMERAL_MESSAGE_DELAY = Duration.ofSeconds(10);
@VisibleForTesting
static final Duration MAX_EPHEMERAL_MESSAGE_DELAY = Duration.ofSeconds(10);
private static final String REMOVE_TIMER_NAME = name(MessagesCache.class, "remove");
private static final String REMOVE_METHOD_TAG = "method";
private static final String REMOVE_METHOD_UUID = "uuid";
private static final String GET_FLUX_NAME = MetricsUtil.name(MessagesCache.class, "get");
private static final int PAGE_SIZE = 100;
private static final Logger logger = LoggerFactory.getLogger(MessagesCache.class);
public MessagesCache(final FaultTolerantRedisCluster insertCluster, final FaultTolerantRedisCluster readDeleteCluster,
final ExecutorService notificationExecutorService) throws IOException {
final Clock clock, final ExecutorService notificationExecutorService,
final ExecutorService messageDeletionExecutorService) throws IOException {
this.readDeleteCluster = readDeleteCluster;
this.pubSubConnection = readDeleteCluster.createPubSubConnection();
this.clock = clock;
this.notificationExecutorService = notificationExecutorService;
this.messageDeletionExecutorService = messageDeletionExecutorService;
this.messageDeletionScheduler = Schedulers.fromExecutorService(messageDeletionExecutorService, "messageDeletion");
this.insertScript = ClusterLuaScript.fromResource(insertCluster, "lua/insert_item.lua", ScriptOutputType.INTEGER);
this.removeByGuidScript = ClusterLuaScript.fromResource(readDeleteCluster, "lua/remove_item_by_guid.lua",
@@ -147,33 +164,39 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
guid.toString().getBytes(StandardCharsets.UTF_8))));
}
public Optional<MessageProtos.Envelope> remove(final UUID destinationUuid, final long destinationDevice,
public CompletableFuture<Optional<MessageProtos.Envelope>> remove(final UUID destinationUuid,
final long destinationDevice,
final UUID messageGuid) {
return remove(destinationUuid, destinationDevice, List.of(messageGuid)).stream().findFirst();
return remove(destinationUuid, destinationDevice, List.of(messageGuid))
.thenApply(removed -> removed.isEmpty() ? Optional.empty() : Optional.of(removed.get(0)));
}
@SuppressWarnings("unchecked")
public List<MessageProtos.Envelope> remove(final UUID destinationUuid, final long destinationDevice,
public CompletableFuture<List<MessageProtos.Envelope>> remove(final UUID destinationUuid,
final long destinationDevice,
final List<UUID> messageGuids) {
final List<byte[]> serialized = (List<byte[]>) Metrics.timer(REMOVE_TIMER_NAME, REMOVE_METHOD_TAG,
REMOVE_METHOD_UUID).record(() ->
removeByGuidScript.executeBinary(List.of(getMessageQueueKey(destinationUuid, destinationDevice),
return removeByGuidScript.executeBinaryAsync(List.of(getMessageQueueKey(destinationUuid, destinationDevice),
getMessageQueueMetadataKey(destinationUuid, destinationDevice),
getQueueIndexKey(destinationUuid, destinationDevice)),
messageGuids.stream().map(guid -> guid.toString().getBytes(StandardCharsets.UTF_8))
.collect(Collectors.toList())));
.collect(Collectors.toList()))
.thenApplyAsync(result -> {
List<byte[]> serialized = (List<byte[]>) result;
final List<MessageProtos.Envelope> removedMessages = new ArrayList<>(serialized.size());
final List<MessageProtos.Envelope> removedMessages = new ArrayList<>(serialized.size());
for (final byte[] bytes : serialized) {
try {
removedMessages.add(MessageProtos.Envelope.parseFrom(bytes));
} catch (final InvalidProtocolBufferException e) {
logger.warn("Failed to parse envelope", e);
}
}
for (final byte[] bytes : serialized) {
try {
removedMessages.add(MessageProtos.Envelope.parseFrom(bytes));
} catch (final InvalidProtocolBufferException e) {
logger.warn("Failed to parse envelope", e);
}
}
return removedMessages;
return removedMessages;
}, messageDeletionExecutorService);
}
public boolean hasMessages(final UUID destinationUuid, final long destinationDevice) {
@@ -181,50 +204,111 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
connection -> connection.sync().zcard(getMessageQueueKey(destinationUuid, destinationDevice)) > 0);
}
@SuppressWarnings("unchecked")
public List<MessageProtos.Envelope> get(final UUID destinationUuid, final long destinationDevice, final int limit) {
return getMessagesTimer.record(() -> {
final List<byte[]> queueItems = (List<byte[]>) getItemsScript.executeBinary(
List.of(getMessageQueueKey(destinationUuid, destinationDevice),
getPersistInProgressKey(destinationUuid, destinationDevice)),
List.of(String.valueOf(limit).getBytes(StandardCharsets.UTF_8)));
public Publisher<MessageProtos.Envelope> get(final UUID destinationUuid, final long destinationDevice) {
final long earliestAllowableEphemeralTimestamp =
System.currentTimeMillis() - MAX_EPHEMERAL_MESSAGE_DELAY.toMillis();
final long earliestAllowableEphemeralTimestamp =
clock.millis() - MAX_EPHEMERAL_MESSAGE_DELAY.toMillis();
final List<MessageProtos.Envelope> messageEntities;
final List<UUID> staleEphemeralMessageGuids = new ArrayList<>();
final Flux<MessageProtos.Envelope> allMessages = getAllMessages(destinationUuid, destinationDevice)
.publish()
// We expect exactly two subscribers to this base flux:
// 1. the websocket that delivers messages to clients
// 2. an internal process to discard stale ephemeral messages
// The discard subscriber will subscribe immediately, but we dont want to do any work if the
// websocket never subscribes.
.autoConnect(2);
if (queueItems.size() % 2 == 0) {
messageEntities = new ArrayList<>(queueItems.size() / 2);
final Flux<MessageProtos.Envelope> messagesToPublish = allMessages
.filter(Predicate.not(envelope -> isStaleEphemeralMessage(envelope, earliestAllowableEphemeralTimestamp)));
for (int i = 0; i < queueItems.size() - 1; i += 2) {
try {
final MessageProtos.Envelope message = MessageProtos.Envelope.parseFrom(queueItems.get(i));
if (message.getEphemeral() && message.getTimestamp() < earliestAllowableEphemeralTimestamp) {
staleEphemeralMessageGuids.add(UUID.fromString(message.getServerGuid()));
continue;
}
final Flux<MessageProtos.Envelope> staleEphemeralMessages = allMessages
.filter(envelope -> isStaleEphemeralMessage(envelope, earliestAllowableEphemeralTimestamp));
messageEntities.add(message);
} catch (InvalidProtocolBufferException e) {
logger.warn("Failed to parse envelope", e);
discardStaleEphemeralMessages(destinationUuid, destinationDevice, staleEphemeralMessages);
return messagesToPublish.name(GET_FLUX_NAME)
.metrics();
}
private static boolean isStaleEphemeralMessage(final MessageProtos.Envelope message,
long earliestAllowableTimestamp) {
return message.hasEphemeral() && message.getEphemeral() && message.getTimestamp() < earliestAllowableTimestamp;
}
private void discardStaleEphemeralMessages(final UUID destinationUuid, final long destinationDevice,
Flux<MessageProtos.Envelope> staleEphemeralMessages) {
staleEphemeralMessages
.map(e -> UUID.fromString(e.getServerGuid()))
.buffer(PAGE_SIZE)
.subscribeOn(messageDeletionScheduler)
.subscribe(staleEphemeralMessageGuids ->
remove(destinationUuid, destinationDevice, staleEphemeralMessageGuids)
.thenAccept(removedMessages -> staleEphemeralMessagesCounter.increment(removedMessages.size())),
e -> logger.warn("Could not remove stale ephemeral messages from cache", e));
}
@VisibleForTesting
Flux<MessageProtos.Envelope> getAllMessages(final UUID destinationUuid, final long destinationDevice) {
// fetch messages by page
return getNextMessagePage(destinationUuid, destinationDevice, -1)
.expand(queueItemsAndLastMessageId -> {
// expand() is breadth-first, so each page will be published in order
if (queueItemsAndLastMessageId.first().isEmpty()) {
return Mono.empty();
}
}
} else {
logger.error("\"Get messages\" operation returned a list with a non-even number of elements.");
messageEntities = Collections.emptyList();
}
try {
remove(destinationUuid, destinationDevice, staleEphemeralMessageGuids);
staleEphemeralMessagesCounter.increment(staleEphemeralMessageGuids.size());
} catch (final Throwable e) {
logger.warn("Could not remove stale ephemeral messages from cache", e);
}
return getNextMessagePage(destinationUuid, destinationDevice, queueItemsAndLastMessageId.second());
})
.limitRate(1)
// we want to ensure we dont accidentally block the Lettuce/netty i/o executors
.publishOn(Schedulers.boundedElastic())
.map(Pair::first)
.flatMapIterable(queueItems -> {
final List<MessageProtos.Envelope> envelopes = new ArrayList<>(queueItems.size() / 2);
return messageEntities;
});
for (int i = 0; i < queueItems.size() - 1; i += 2) {
try {
final MessageProtos.Envelope message = MessageProtos.Envelope.parseFrom(queueItems.get(i));
envelopes.add(message);
} catch (InvalidProtocolBufferException e) {
logger.warn("Failed to parse envelope", e);
}
}
return envelopes;
});
}
private Flux<Pair<List<byte[]>, Long>> getNextMessagePage(final UUID destinationUuid, final long destinationDevice,
long messageId) {
return getItemsScript.executeBinaryReactive(
List.of(getMessageQueueKey(destinationUuid, destinationDevice),
getPersistInProgressKey(destinationUuid, destinationDevice)),
List.of(String.valueOf(PAGE_SIZE).getBytes(StandardCharsets.UTF_8),
String.valueOf(messageId).getBytes(StandardCharsets.UTF_8)))
.map(result -> {
logger.trace("Processing page: {}", messageId);
@SuppressWarnings("unchecked")
List<byte[]> queueItems = (List<byte[]>) result;
if (queueItems.isEmpty()) {
return new Pair<>(Collections.emptyList(), null);
}
if (queueItems.size() % 2 != 0) {
logger.error("\"Get messages\" operation returned a list with a non-even number of elements.");
return new Pair<>(Collections.emptyList(), null);
}
final long lastMessageId = Long.parseLong(
new String(queueItems.get(queueItems.size() - 1), StandardCharsets.UTF_8));
return new Pair<>(queueItems, lastMessageId);
});
}
@VisibleForTesting
@@ -307,12 +391,13 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
}
public void removeMessageAvailabilityListener(final MessageAvailabilityListener listener) {
@Nullable final String queueName = queueNamesByMessageListener.remove(listener);
@Nullable final String queueName = queueNamesByMessageListener.get(listener);
if (queueName != null) {
unsubscribeFromKeyspaceNotifications(queueName);
synchronized (messageListenersByQueueName) {
queueNamesByMessageListener.remove(listener);
messageListenersByQueueName.remove(queueName);
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2021 Signal Messenger, LLC
* Copyright 2021-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
@@ -17,19 +17,26 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Predicate;
import org.reactivestreams.Publisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse;
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
import software.amazon.awssdk.services.dynamodb.model.PutRequest;
import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
@@ -48,22 +55,27 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
private static final String KEY_ENVELOPE_BYTES = "EB";
private final Timer storeTimer = timer(name(getClass(), "store"));
private final Timer loadTimer = timer(name(getClass(), "load"));
private final Timer deleteByGuid = timer(name(getClass(), "delete", "guid"));
private final Timer deleteByKey = timer(name(getClass(), "delete", "key"));
private final Timer deleteByAccount = timer(name(getClass(), "delete", "account"));
private final Timer deleteByDevice = timer(name(getClass(), "delete", "device"));
private final DynamoDbAsyncClient dbAsyncClient;
private final String tableName;
private final Duration timeToLive;
private final ExecutorService messageDeletionExecutor;
private final Scheduler messageDeletionScheduler;
private static final Logger logger = LoggerFactory.getLogger(MessagesDynamoDb.class);
public MessagesDynamoDb(DynamoDbClient dynamoDb, String tableName, Duration timeToLive) {
public MessagesDynamoDb(DynamoDbClient dynamoDb, DynamoDbAsyncClient dynamoDbAsyncClient, String tableName,
Duration timeToLive, ExecutorService messageDeletionExecutor) {
super(dynamoDb);
this.dbAsyncClient = dynamoDbAsyncClient;
this.tableName = tableName;
this.timeToLive = timeToLive;
this.messageDeletionExecutor = messageDeletionExecutor;
this.messageDeletionScheduler = Schedulers.fromExecutor(messageDeletionExecutor);
}
public void store(final List<MessageProtos.Envelope> messages, final UUID destinationAccountUuid, final long destinationDeviceId) {
@@ -95,105 +107,106 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
executeTableWriteItemsUntilComplete(Map.of(tableName, writeItems));
}
public List<MessageProtos.Envelope> load(final UUID destinationAccountUuid, final long destinationDeviceId, final int requestedNumberOfMessagesToFetch) {
return loadTimer.record(() -> {
final int numberOfMessagesToFetch = Math.min(requestedNumberOfMessagesToFetch, RESULT_SET_CHUNK_SIZE);
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.consistentRead(true)
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#sort", KEY_SORT))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)))
.limit(numberOfMessagesToFetch)
.build();
List<MessageProtos.Envelope> messageEntities = new ArrayList<>(numberOfMessagesToFetch);
for (Map<String, AttributeValue> message : db().queryPaginator(queryRequest).items()) {
try {
messageEntities.add(convertItemToEnvelope(message));
} catch (final InvalidProtocolBufferException e) {
logger.error("Failed to parse envelope", e);
}
public Publisher<MessageProtos.Envelope> load(final UUID destinationAccountUuid, final long destinationDeviceId,
final Integer limit) {
if (messageEntities.size() == numberOfMessagesToFetch) {
// queryPaginator() uses limit() as the page size, not as an absolute limit
// …but a page might be smaller than limit, because a page is capped at 1 MB
break;
}
}
return messageEntities;
});
}
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest.Builder queryRequestBuilder = QueryRequest.builder()
.tableName(tableName)
.consistentRead(true)
.keyConditionExpression("#part = :part AND begins_with ( #sort , :sortprefix )")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#sort", KEY_SORT))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":sortprefix", convertDestinationDeviceIdToSortKeyPrefix(destinationDeviceId)));
public Optional<MessageProtos.Envelope> deleteMessageByDestinationAndGuid(final UUID destinationAccountUuid,
final UUID messageUuid) {
return deleteByGuid.record(() -> {
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.indexName(LOCAL_INDEX_MESSAGE_UUID_NAME)
.projectionExpression(KEY_SORT)
.consistentRead(true)
.keyConditionExpression("#part = :part AND #uuid = :uuid")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#uuid", LOCAL_INDEX_MESSAGE_UUID_KEY_SORT))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":uuid", convertLocalIndexMessageUuidSortKey(messageUuid)))
.build();
return deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(partitionKey, queryRequest);
});
}
public Optional<MessageProtos.Envelope> deleteMessage(final UUID destinationAccountUuid,
final long destinationDeviceId, final UUID messageUuid, final long serverTimestamp) {
return deleteByKey.record(() -> {
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final AttributeValue sortKey = convertSortKey(destinationDeviceId, serverTimestamp, messageUuid);
DeleteItemRequest.Builder deleteItemRequest = DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_PARTITION, partitionKey, KEY_SORT, sortKey))
.returnValues(ReturnValue.ALL_OLD);
final DeleteItemResponse deleteItemResponse = db().deleteItem(deleteItemRequest.build());
if (deleteItemResponse.attributes() != null && deleteItemResponse.attributes().containsKey(KEY_PARTITION)) {
try {
return Optional.of(convertItemToEnvelope(deleteItemResponse.attributes()));
} catch (final InvalidProtocolBufferException e) {
logger.error("Failed to parse envelope", e);
return Optional.empty();
}
}
return Optional.empty();
});
}
@Nonnull
private Optional<MessageProtos.Envelope> deleteItemsMatchingQueryAndReturnFirstOneActuallyDeleted(AttributeValue partitionKey, QueryRequest queryRequest) {
Optional<MessageProtos.Envelope> result = Optional.empty();
for (Map<String, AttributeValue> item : db().queryPaginator(queryRequest).items()) {
final byte[] rangeKeyValue = item.get(KEY_SORT).b().asByteArray();
DeleteItemRequest.Builder deleteItemRequest = DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_PARTITION, partitionKey, KEY_SORT, AttributeValues.fromByteArray(rangeKeyValue)));
if (result.isEmpty()) {
deleteItemRequest.returnValues(ReturnValue.ALL_OLD);
}
final DeleteItemResponse deleteItemResponse = db().deleteItem(deleteItemRequest.build());
if (deleteItemResponse.attributes() != null && deleteItemResponse.attributes().containsKey(KEY_PARTITION)) {
try {
result = Optional.of(convertItemToEnvelope(deleteItemResponse.attributes()));
} catch (final InvalidProtocolBufferException e) {
logger.error("Failed to parse envelope", e);
}
}
if (limit != null) {
// some callers dont take advantage of reactive streams, so we want to support limiting the fetch size. Otherwise,
// we could fetch up to 1 MB (likely >1,000 messages) and discard 90% of them
queryRequestBuilder.limit(Math.min(RESULT_SET_CHUNK_SIZE, limit));
}
return result;
final QueryRequest queryRequest = queryRequestBuilder.build();
return dbAsyncClient.queryPaginator(queryRequest).items()
.map(message -> {
try {
return convertItemToEnvelope(message);
} catch (final InvalidProtocolBufferException e) {
logger.error("Failed to parse envelope", e);
return null;
}
})
.filter(Predicate.not(Objects::isNull));
}
public CompletableFuture<Optional<MessageProtos.Envelope>> deleteMessageByDestinationAndGuid(
final UUID destinationAccountUuid, final UUID messageUuid) {
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.indexName(LOCAL_INDEX_MESSAGE_UUID_NAME)
.projectionExpression(KEY_SORT)
.consistentRead(true)
.keyConditionExpression("#part = :part AND #uuid = :uuid")
.expressionAttributeNames(Map.of(
"#part", KEY_PARTITION,
"#uuid", LOCAL_INDEX_MESSAGE_UUID_KEY_SORT))
.expressionAttributeValues(Map.of(
":part", partitionKey,
":uuid", convertLocalIndexMessageUuidSortKey(messageUuid)))
.build();
// because we are filtering on message UUID, this query should return at most one item,
// but its simpler to handle the full stream and return the “last” item
return Flux.from(dbAsyncClient.queryPaginator(queryRequest).items())
.flatMap(item -> Mono.fromCompletionStage(dbAsyncClient.deleteItem(DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_PARTITION, partitionKey, KEY_SORT,
AttributeValues.fromByteArray(item.get(KEY_SORT).b().asByteArray())))
.returnValues(ReturnValue.ALL_OLD)
.build())))
.mapNotNull(deleteItemResponse -> {
try {
if (deleteItemResponse.attributes() != null && deleteItemResponse.attributes().containsKey(KEY_PARTITION)) {
return convertItemToEnvelope(deleteItemResponse.attributes());
}
} catch (final InvalidProtocolBufferException e) {
logger.error("Failed to parse envelope", e);
}
return null;
})
.map(Optional::ofNullable)
.subscribeOn(messageDeletionScheduler)
.last(Optional.empty()) // if the flux is empty, last() will throw without a default
.toFuture();
}
public CompletableFuture<Optional<MessageProtos.Envelope>> deleteMessage(final UUID destinationAccountUuid,
final long destinationDeviceId, final UUID messageUuid, final long serverTimestamp) {
final AttributeValue partitionKey = convertPartitionKey(destinationAccountUuid);
final AttributeValue sortKey = convertSortKey(destinationDeviceId, serverTimestamp, messageUuid);
DeleteItemRequest.Builder deleteItemRequest = DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_PARTITION, partitionKey, KEY_SORT, sortKey))
.returnValues(ReturnValue.ALL_OLD);
return dbAsyncClient.deleteItem(deleteItemRequest.build())
.thenApplyAsync(deleteItemResponse -> {
if (deleteItemResponse.attributes() != null && deleteItemResponse.attributes().containsKey(KEY_PARTITION)) {
try {
return Optional.of(convertItemToEnvelope(deleteItemResponse.attributes()));
} catch (final InvalidProtocolBufferException e) {
logger.error("Failed to parse envelope", e);
}
}
return Optional.empty();
}, messageDeletionExecutor);
}
public void deleteAllMessagesForAccount(final UUID destinationAccountUuid) {
@@ -248,7 +261,7 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
KEY_PARTITION, partitionKey,
KEY_SORT, item.get(KEY_SORT))).build())
.build())
.collect(Collectors.toList());
.toList();
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
@@ -9,18 +9,32 @@ import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.reactivestreams.Publisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.entities.MessageProtos.Envelope;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.Pair;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class MessagesManager {
private static final int RESULT_SET_CHUNK_SIZE = 100;
final String GET_MESSAGES_FOR_DEVICE_FLUX_NAME = MetricsUtil.name(MessagesManager.class, "getMessagesForDevice");
private static final Logger logger = LoggerFactory.getLogger(MessagesManager.class);
private static final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private static final Meter cacheHitByGuidMeter = metricRegistry.meter(name(MessagesManager.class, "cacheHitByGuid"));
@@ -31,14 +45,17 @@ public class MessagesManager {
private final MessagesDynamoDb messagesDynamoDb;
private final MessagesCache messagesCache;
private final ReportMessageManager reportMessageManager;
private final ExecutorService messageDeletionExecutor;
public MessagesManager(
final MessagesDynamoDb messagesDynamoDb,
final MessagesCache messagesCache,
final ReportMessageManager reportMessageManager) {
final ReportMessageManager reportMessageManager,
final ExecutorService messageDeletionExecutor) {
this.messagesDynamoDb = messagesDynamoDb;
this.messagesCache = messagesCache;
this.reportMessageManager = reportMessageManager;
this.messageDeletionExecutor = messageDeletionExecutor;
}
public void insert(UUID destinationUuid, long destinationDevice, Envelope message) {
@@ -55,18 +72,32 @@ public class MessagesManager {
return messagesCache.hasMessages(destinationUuid, destinationDevice);
}
public Pair<List<Envelope>, Boolean> getMessagesForDevice(UUID destinationUuid, long destinationDevice, final boolean cachedMessagesOnly) {
List<Envelope> messageList = new ArrayList<>();
public Mono<Pair<List<Envelope>, Boolean>> getMessagesForDevice(UUID destinationUuid, long destinationDevice,
boolean cachedMessagesOnly) {
if (!cachedMessagesOnly) {
messageList.addAll(messagesDynamoDb.load(destinationUuid, destinationDevice, RESULT_SET_CHUNK_SIZE));
}
return Flux.from(
getMessagesForDevice(destinationUuid, destinationDevice, RESULT_SET_CHUNK_SIZE, cachedMessagesOnly))
.take(RESULT_SET_CHUNK_SIZE, true)
.collectList()
.map(envelopes -> new Pair<>(envelopes, envelopes.size() >= RESULT_SET_CHUNK_SIZE));
}
if (messageList.size() < RESULT_SET_CHUNK_SIZE) {
messageList.addAll(messagesCache.get(destinationUuid, destinationDevice, RESULT_SET_CHUNK_SIZE - messageList.size()));
}
public Publisher<Envelope> getMessagesForDeviceReactive(UUID destinationUuid, long destinationDevice,
final boolean cachedMessagesOnly) {
return new Pair<>(messageList, messageList.size() >= RESULT_SET_CHUNK_SIZE);
return getMessagesForDevice(destinationUuid, destinationDevice, null, cachedMessagesOnly);
}
private Publisher<Envelope> getMessagesForDevice(UUID destinationUuid, long destinationDevice,
@Nullable Integer limit, final boolean cachedMessagesOnly) {
final Publisher<Envelope> dynamoPublisher =
cachedMessagesOnly ? Flux.empty() : messagesDynamoDb.load(destinationUuid, destinationDevice, limit);
final Publisher<Envelope> cachePublisher = messagesCache.get(destinationUuid, destinationDevice);
return Flux.concat(dynamoPublisher, cachePublisher)
.name(GET_MESSAGES_FOR_DEVICE_FLUX_NAME)
.metrics();
}
public void clear(UUID destinationUuid) {
@@ -79,21 +110,25 @@ public class MessagesManager {
messagesDynamoDb.deleteAllMessagesForDevice(destinationUuid, deviceId);
}
public Optional<Envelope> delete(UUID destinationUuid, long destinationDeviceId, UUID guid, Long serverTimestamp) {
Optional<Envelope> removed = messagesCache.remove(destinationUuid, destinationDeviceId, guid);
public CompletableFuture<Optional<Envelope>> delete(UUID destinationUuid, long destinationDeviceId, UUID guid,
@Nullable Long serverTimestamp) {
return messagesCache.remove(destinationUuid, destinationDeviceId, guid)
.thenComposeAsync(removed -> {
if (removed.isEmpty()) {
if (serverTimestamp == null) {
removed = messagesDynamoDb.deleteMessageByDestinationAndGuid(destinationUuid, guid);
} else {
removed = messagesDynamoDb.deleteMessage(destinationUuid, destinationDeviceId, guid, serverTimestamp);
}
cacheMissByGuidMeter.mark();
} else {
cacheHitByGuidMeter.mark();
}
if (removed.isPresent()) {
cacheHitByGuidMeter.mark();
return CompletableFuture.completedFuture(removed);
}
return removed;
cacheMissByGuidMeter.mark();
if (serverTimestamp == null) {
return messagesDynamoDb.deleteMessageByDestinationAndGuid(destinationUuid, guid);
} else {
return messagesDynamoDb.deleteMessage(destinationUuid, destinationDeviceId, guid, serverTimestamp);
}
}, messageDeletionExecutor);
}
/**
@@ -112,10 +147,15 @@ public class MessagesManager {
final List<UUID> messageGuids = messages.stream().map(message -> UUID.fromString(message.getServerGuid()))
.collect(Collectors.toList());
int messagesRemovedFromCache = messagesCache.remove(destinationUuid, destinationDeviceId, messageGuids).size();
persistMessageMeter.mark(nonEphemeralMessages.size());
int messagesRemovedFromCache = 0;
try {
messagesRemovedFromCache = messagesCache.remove(destinationUuid, destinationDeviceId, messageGuids)
.get(30, TimeUnit.SECONDS).size();
persistMessageMeter.mark(nonEphemeralMessages.size());
} catch (InterruptedException | ExecutionException | TimeoutException e) {
logger.warn("Failed to remove messages from cache", e);
}
return messagesRemovedFromCache;
}
@@ -129,4 +169,5 @@ public class MessagesManager {
public void removeMessageAvailabilityListener(final MessageAvailabilityListener listener) {
messagesCache.removeMessageAvailabilityListener(listener);
}
}

View File

@@ -6,5 +6,6 @@
package org.whispersystems.textsecuregcm.storage;
public interface PubSubAddress {
public String serialize();
String serialize();
}

View File

@@ -10,18 +10,20 @@ import static org.whispersystems.textsecuregcm.util.AttributeValues.m;
import static org.whispersystems.textsecuregcm.util.AttributeValues.n;
import static org.whispersystems.textsecuregcm.util.AttributeValues.s;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.Instant;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.ws.rs.ClientErrorException;
import javax.ws.rs.core.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.subscriptions.ProcessorCustomer;
@@ -44,10 +46,9 @@ public class SubscriptionManager {
public static final String KEY_USER = "U"; // B (Hash Key)
public static final String KEY_PASSWORD = "P"; // B
@Deprecated
public static final String KEY_CUSTOMER_ID = "C"; // S (GSI Hash Key of `c_to_u` index)
public static final String KEY_PROCESSOR_ID_CUSTOMER_ID = "PC"; // B (GSI Hash Key of `pc_to_u` index)
public static final String KEY_CREATED_AT = "R"; // N
@Deprecated
public static final String KEY_PROCESSOR_CUSTOMER_IDS_MAP = "PCI"; // M
public static final String KEY_SUBSCRIPTION_ID = "S"; // S
public static final String KEY_SUBSCRIPTION_CREATED_AT = "T"; // N
@@ -57,16 +58,16 @@ public class SubscriptionManager {
public static final String KEY_CANCELED_AT = "B"; // N
public static final String KEY_CURRENT_PERIOD_ENDS_AT = "D"; // N
public static final String INDEX_NAME = "c_to_u"; // Hash Key "C"
public static final String INDEX_NAME = "pc_to_u"; // Hash Key "PC"
public static class Record {
public final byte[] user;
public final byte[] password;
public final Instant createdAt;
public @Nullable String customerId;
public @Nullable SubscriptionProcessor processor;
public Map<SubscriptionProcessor, String> processorsToCustomerIds;
@VisibleForTesting
@Nullable
ProcessorCustomer processorCustomer;
public String subscriptionId;
public Instant subscriptionCreatedAt;
public Long subscriptionLevel;
@@ -82,40 +83,27 @@ public class SubscriptionManager {
}
public static Record from(byte[] user, Map<String, AttributeValue> item) {
Record self = new Record(
Record record = new Record(
user,
item.get(KEY_PASSWORD).b().asByteArray(),
getInstant(item, KEY_CREATED_AT));
final Pair<SubscriptionProcessor, String> processorCustomerId = getProcessorAndCustomer(item);
if (processorCustomerId != null) {
self.customerId = processorCustomerId.second();
self.processor = processorCustomerId.first();
} else {
// Until all existing data is migrated to KEY_PROCESSOR_ID_CUSTOMER_ID, fall back to KEY_CUSTOMER_ID
self.customerId = getString(item, KEY_CUSTOMER_ID);
record.processorCustomer = new ProcessorCustomer(processorCustomerId.second(), processorCustomerId.first());
}
self.processorsToCustomerIds = getProcessorsToCustomerIds(item);
self.subscriptionId = getString(item, KEY_SUBSCRIPTION_ID);
self.subscriptionCreatedAt = getInstant(item, KEY_SUBSCRIPTION_CREATED_AT);
self.subscriptionLevel = getLong(item, KEY_SUBSCRIPTION_LEVEL);
self.subscriptionLevelChangedAt = getInstant(item, KEY_SUBSCRIPTION_LEVEL_CHANGED_AT);
self.accessedAt = getInstant(item, KEY_ACCESSED_AT);
self.canceledAt = getInstant(item, KEY_CANCELED_AT);
self.currentPeriodEndsAt = getInstant(item, KEY_CURRENT_PERIOD_ENDS_AT);
return self;
record.subscriptionId = getString(item, KEY_SUBSCRIPTION_ID);
record.subscriptionCreatedAt = getInstant(item, KEY_SUBSCRIPTION_CREATED_AT);
record.subscriptionLevel = getLong(item, KEY_SUBSCRIPTION_LEVEL);
record.subscriptionLevelChangedAt = getInstant(item, KEY_SUBSCRIPTION_LEVEL_CHANGED_AT);
record.accessedAt = getInstant(item, KEY_ACCESSED_AT);
record.canceledAt = getInstant(item, KEY_CANCELED_AT);
record.currentPeriodEndsAt = getInstant(item, KEY_CURRENT_PERIOD_ENDS_AT);
return record;
}
private static Map<SubscriptionProcessor, String> getProcessorsToCustomerIds(Map<String, AttributeValue> item) {
final AttributeValue attributeValue = item.get(KEY_PROCESSOR_CUSTOMER_IDS_MAP);
final Map<String, AttributeValue> attribute =
attributeValue == null ? Collections.emptyMap() : attributeValue.m();
final Map<SubscriptionProcessor, String> processorsToCustomerIds = new HashMap<>();
attribute.forEach((processorName, customerId) ->
processorsToCustomerIds.put(SubscriptionProcessor.valueOf(processorName), customerId.s()));
return processorsToCustomerIds;
public Optional<ProcessorCustomer> getProcessorCustomer() {
return Optional.ofNullable(processorCustomer);
}
/**
@@ -185,26 +173,26 @@ public class SubscriptionManager {
/**
* Looks in the GSI for a record with the given customer id and returns the user id.
*/
public CompletableFuture<byte[]> getSubscriberUserByStripeCustomerId(@Nonnull String customerId) {
public CompletableFuture<byte[]> getSubscriberUserByProcessorCustomer(ProcessorCustomer processorCustomer) {
QueryRequest query = QueryRequest.builder()
.tableName(table)
.indexName(INDEX_NAME)
.keyConditionExpression("#customer_id = :customer_id")
.keyConditionExpression("#processor_customer_id = :processor_customer_id")
.projectionExpression("#user")
.expressionAttributeNames(Map.of(
"#customer_id", KEY_CUSTOMER_ID,
"#processor_customer_id", KEY_PROCESSOR_ID_CUSTOMER_ID,
"#user", KEY_USER))
.expressionAttributeValues(Map.of(
":customer_id", s(Objects.requireNonNull(customerId))))
":processor_customer_id", b(processorCustomer.toDynamoBytes())))
.build();
return client.query(query).thenApply(queryResponse -> {
int count = queryResponse.count();
if (count == 0) {
return null;
} else if (count > 1) {
logger.error("expected invariant of 1-1 subscriber-customer violated for customer {}", customerId);
logger.error("expected invariant of 1-1 subscriber-customer violated for customer {}", processorCustomer);
throw new IllegalStateException(
"expected invariant of 1-1 subscriber-customer violated for customer " + customerId);
"expected invariant of 1-1 subscriber-customer violated for customer " + processorCustomer);
} else {
Map<String, AttributeValue> result = queryResponse.items().get(0);
return result.get(KEY_USER).b().asByteArray();
@@ -307,75 +295,42 @@ public class SubscriptionManager {
}
/**
* Updates the active processor and customer ID for the given user record.
* Sets the processor and customer ID for the given user record.
*
* @return the updated user record.
* @return the user record.
*/
public CompletableFuture<Record> updateProcessorAndCustomerId(Record userRecord,
public CompletableFuture<Record> setProcessorAndCustomerId(Record userRecord,
ProcessorCustomer activeProcessorCustomer, Instant updatedAt) {
// Dont attempt to modify the existing map, since it may be immutable, and we also dont want to have side effects
final Map<SubscriptionProcessor, String> allProcessorsAndCustomerIds = new HashMap<>(
userRecord.processorsToCustomerIds);
allProcessorsAndCustomerIds.put(activeProcessorCustomer.processor(), activeProcessorCustomer.customerId());
UpdateItemRequest request = UpdateItemRequest.builder()
.tableName(table)
.key(Map.of(KEY_USER, b(userRecord.user)))
.returnValues(ReturnValue.ALL_NEW)
.conditionExpression(
// there is no customer attribute yet
"attribute_not_exists(#customer_id) " +
// OR this record doesn't have the new processor+customer attributes yet
"OR (#customer_id = :customer_id " +
"AND attribute_not_exists(#processor_customer_id) " +
// TODO once all records are guaranteed to have the map, we can do a more targeted update
// "AND attribute_not_exists(#processors_to_customer_ids.#processor_name) " +
"AND attribute_not_exists(#processors_to_customer_ids))"
)
.conditionExpression("attribute_not_exists(#processor_customer_id)")
.updateExpression("SET "
+ "#customer_id = :customer_id, "
+ "#processor_customer_id = :processor_customer_id, "
// TODO once all records are guaranteed to have the map, we can do a more targeted update
// + "#processors_to_customer_ids.#processor_name = :customer_id, "
+ "#processors_to_customer_ids = :processors_and_customer_ids, "
+ "#accessed_at = :accessed_at"
)
.expressionAttributeNames(Map.of(
"#accessed_at", KEY_ACCESSED_AT,
"#customer_id", KEY_CUSTOMER_ID,
"#processor_customer_id", KEY_PROCESSOR_ID_CUSTOMER_ID,
// TODO "#processor_name", activeProcessor.name(),
"#processors_to_customer_ids", KEY_PROCESSOR_CUSTOMER_IDS_MAP
"#processor_customer_id", KEY_PROCESSOR_ID_CUSTOMER_ID
))
.expressionAttributeValues(Map.of(
":accessed_at", n(updatedAt.getEpochSecond()),
":customer_id", s(activeProcessorCustomer.customerId()),
":processor_customer_id", b(activeProcessorCustomer.toDynamoBytes()),
":processors_and_customer_ids", m(createProcessorsToCustomerIdsAttributeMap(allProcessorsAndCustomerIds))
":processor_customer_id", b(activeProcessorCustomer.toDynamoBytes())
)).build();
return client.updateItem(request)
.thenApply(updateItemResponse -> Record.from(userRecord.user, updateItemResponse.attributes()))
.exceptionallyCompose(throwable -> {
if (Throwables.getRootCause(throwable) instanceof ConditionalCheckFailedException) {
return getUser(userRecord.user).thenApply(getItemResponse ->
Record.from(userRecord.user, getItemResponse.item()));
throw new ClientErrorException(Response.Status.CONFLICT);
}
Throwables.throwIfUnchecked(throwable);
throw new CompletionException(throwable);
});
}
private Map<String, AttributeValue> createProcessorsToCustomerIdsAttributeMap(
Map<SubscriptionProcessor, String> allProcessorsAndCustomerIds) {
final Map<String, AttributeValue> result = new HashMap<>();
allProcessorsAndCustomerIds.forEach((processor, customerId) -> result.put(processor.name(), s(customerId)));
return result;
}
public CompletableFuture<Void> accessedAt(byte[] user, Instant accessedAt) {
checkUserLength(user);

View File

@@ -0,0 +1,233 @@
/*
* Copyright 2022 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.subscriptions;
import com.apollographql.apollo3.api.ApolloResponse;
import com.apollographql.apollo3.api.Operation;
import com.apollographql.apollo3.api.Operations;
import com.apollographql.apollo3.api.Optional;
import com.apollographql.apollo3.api.json.BufferedSinkJsonWriter;
import com.braintree.graphql.client.type.ChargePaymentMethodInput;
import com.braintree.graphql.client.type.CreatePayPalOneTimePaymentInput;
import com.braintree.graphql.client.type.CustomFieldInput;
import com.braintree.graphql.client.type.MonetaryAmountInput;
import com.braintree.graphql.client.type.PayPalExperienceProfileInput;
import com.braintree.graphql.client.type.PayPalIntent;
import com.braintree.graphql.client.type.PayPalLandingPageType;
import com.braintree.graphql.client.type.PayPalOneTimePaymentInput;
import com.braintree.graphql.client.type.PayPalUserAction;
import com.braintree.graphql.client.type.TokenizePayPalOneTimePaymentInput;
import com.braintree.graphql.client.type.TransactionInput;
import com.braintree.graphql.clientoperation.ChargePayPalOneTimePaymentMutation;
import com.braintree.graphql.clientoperation.CreatePayPalOneTimePaymentMutation;
import com.braintree.graphql.clientoperation.TokenizePayPalOneTimePaymentMutation;
import java.math.BigDecimal;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.Base64;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import javax.ws.rs.ServiceUnavailableException;
import okio.Buffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
class BraintreeGraphqlClient {
// required header value, recommended to be the date the integration began
// https://graphql.braintreepayments.com/guides/making_api_calls/#the-braintree-version-header
private static final String BRAINTREE_VERSION = "2022-10-01";
private static final Logger logger = LoggerFactory.getLogger(BraintreeGraphqlClient.class);
private final FaultTolerantHttpClient httpClient;
private final URI graphqlUri;
private final String authorizationHeader;
BraintreeGraphqlClient(final FaultTolerantHttpClient httpClient,
final String graphqlUri,
final String publicKey,
final String privateKey) {
this.httpClient = httpClient;
try {
this.graphqlUri = new URI(graphqlUri);
} catch (URISyntaxException e) {
throw new IllegalArgumentException("Invalid URI", e);
}
// “public”/“private” key is a bit of a misnomer, but we follow the upstream nomenclature
// they are used for Basic auth similar to “client key”/“client secret” credentials
this.authorizationHeader = "Basic " + Base64.getEncoder().encodeToString((publicKey + ":" + privateKey).getBytes());
}
CompletableFuture<CreatePayPalOneTimePaymentMutation.CreatePayPalOneTimePayment> createPayPalOneTimePayment(
final BigDecimal amount, final String currency, final String returnUrl,
final String cancelUrl, final String locale) {
final CreatePayPalOneTimePaymentInput input = buildCreatePayPalOneTimePaymentInput(amount, currency, returnUrl,
cancelUrl, locale);
final CreatePayPalOneTimePaymentMutation mutation = new CreatePayPalOneTimePaymentMutation(input);
final HttpRequest request = buildRequest(mutation);
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(httpResponse ->
{
// IntelliJ users: type parameters error “no instance of type variable exists so that Data conforms to Data”
// is not accurate; this might be fixed in Kotlin 1.8: https://youtrack.jetbrains.com/issue/KTIJ-21905/
final CreatePayPalOneTimePaymentMutation.Data data = assertSuccessAndExtractData(httpResponse, mutation);
return data.createPayPalOneTimePayment;
});
}
private static CreatePayPalOneTimePaymentInput buildCreatePayPalOneTimePaymentInput(BigDecimal amount,
String currency, String returnUrl, String cancelUrl, String locale) {
return new CreatePayPalOneTimePaymentInput(
Optional.absent(),
Optional.absent(), // merchant account ID will be specified when charging
new MonetaryAmountInput(amount.toString(), currency), // this could potentially use a CustomScalarAdapter
cancelUrl,
Optional.absent(),
PayPalIntent.SALE,
Optional.absent(),
Optional.present(false), // offerPayLater,
Optional.absent(),
Optional.present(
new PayPalExperienceProfileInput(Optional.present("Signal"),
Optional.present(false),
Optional.present(PayPalLandingPageType.LOGIN),
Optional.present(locale),
Optional.absent(),
Optional.present(PayPalUserAction.COMMIT))),
Optional.absent(),
Optional.absent(),
returnUrl,
Optional.absent(),
Optional.absent()
);
}
CompletableFuture<TokenizePayPalOneTimePaymentMutation.TokenizePayPalOneTimePayment> tokenizePayPalOneTimePayment(
final String payerId, final String paymentId, final String paymentToken) {
final TokenizePayPalOneTimePaymentInput input = new TokenizePayPalOneTimePaymentInput(
Optional.absent(),
Optional.absent(), // merchant account ID will be specified when charging
new PayPalOneTimePaymentInput(payerId, paymentId, paymentToken)
);
final TokenizePayPalOneTimePaymentMutation mutation = new TokenizePayPalOneTimePaymentMutation(input);
final HttpRequest request = buildRequest(mutation);
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(httpResponse -> {
// IntelliJ users: type parameters error “no instance of type variable exists so that Data conforms to Data”
// is not accurate; this might be fixed in Kotlin 1.8: https://youtrack.jetbrains.com/issue/KTIJ-21905/
final TokenizePayPalOneTimePaymentMutation.Data data = assertSuccessAndExtractData(httpResponse, mutation);
return data.tokenizePayPalOneTimePayment;
});
}
CompletableFuture<ChargePayPalOneTimePaymentMutation.ChargePaymentMethod> chargeOneTimePayment(
final String paymentMethodId, final BigDecimal amount, final String merchantAccount, final long level) {
final List<CustomFieldInput> customFields = List.of(
new CustomFieldInput("level", Optional.present(Long.toString(level))));
final ChargePaymentMethodInput input = buildChargePaymentMethodInput(paymentMethodId, amount, merchantAccount,
customFields);
final ChargePayPalOneTimePaymentMutation mutation = new ChargePayPalOneTimePaymentMutation(input);
final HttpRequest request = buildRequest(mutation);
return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(httpResponse -> {
// IntelliJ users: type parameters error “no instance of type variable exists so that Data conforms to Data”
// is not accurate; this might be fixed in Kotlin 1.8: https://youtrack.jetbrains.com/issue/KTIJ-21905/
final ChargePayPalOneTimePaymentMutation.Data data = assertSuccessAndExtractData(httpResponse,
mutation);
return data.chargePaymentMethod;
});
}
private static ChargePaymentMethodInput buildChargePaymentMethodInput(String paymentMethodId, BigDecimal amount,
String merchantAccount, List<CustomFieldInput> customFields) {
return new ChargePaymentMethodInput(
Optional.absent(),
paymentMethodId,
new TransactionInput(
// documented as “amount: whole number, or exactly two or three decimal places”
amount.toString(), // this could potentially use a CustomScalarAdapter
Optional.present(merchantAccount),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.present(customFields),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent(),
Optional.absent()
)
);
}
/**
* Verifies that the HTTP response has a {@code 200} status code and the GraphQL response has no errors, otherwise
* throws a {@link ServiceUnavailableException}.
*/
private <T extends Operation<U>, U extends Operation.Data> U assertSuccessAndExtractData(
HttpResponse<String> httpResponse, T operation) {
if (httpResponse.statusCode() != 200) {
logger.warn("Received HTTP response status {} ({})", httpResponse.statusCode(),
httpResponse.headers().firstValue("paypal-debug-id").orElse("<debug id absent>"));
throw new ServiceUnavailableException();
}
ApolloResponse<U> response = Operations.parseJsonResponse(operation, httpResponse.body());
if (response.hasErrors() || response.data == null) {
//noinspection ConstantConditions
response.errors.forEach(
error -> {
final Object legacyCode = java.util.Optional.ofNullable(error.getExtensions())
.map(extensions -> extensions.get("legacyCode"))
.orElse("<none>");
logger.warn("Received GraphQL error for {}: \"{}\" (legacyCode: {})",
response.operation.name(), error.getMessage(), legacyCode);
});
throw new ServiceUnavailableException();
}
return response.data;
}
private HttpRequest buildRequest(final Operation<?> operation) {
final Buffer buffer = new Buffer();
Operations.composeJsonRequest(operation, new BufferedSinkJsonWriter(buffer));
return HttpRequest.newBuilder()
.uri(graphqlUri)
.method("POST", HttpRequest.BodyPublishers.ofString(buffer.readUtf8()))
.header("Content-Type", "application/json")
.header("Authorization", authorizationHeader)
.header("Braintree-Version", BRAINTREE_VERSION)
.build();
}
}

Some files were not shown because too many files have changed in this diff Show More