Bump version to 0.16

1) Use WebSockets for delivery if a client is connected.

2) If a client isn't connected, write to a redis queue and send
   an APN push.

.gitignore

@@ -7,3 +7,5 @@ run.sh
 local.yml
 config/production.yml
 config/federated.yml
+config/staging.yml
+.opsmanage

README.md

@@ -30,6 +30,10 @@ whispersystems@lists.riseup.net
 
 https://lists.riseup.net/www/info/whispersystems
 
+Current BitHub Payment Per Commit:
+=================
+![Current Price](https://bithub.herokuapp.com/v1/status/payment/commit)
+
 
 Cryptography Notice
 ------------

config/sample.yml

@@ -3,9 +3,12 @@ twilio:
   accountToken:
   number:
   localDomain: # The domain Twilio can call back to.
+  international: # Boolean specifying Twilio for international delivery
 
 # Optional. If specified, Nexmo will be used for non-US SMS and 
-# voice verification.
+# voice verification if twilio.international is false. Otherwise,
+# Nexmo, if specified, Nexmo will only be used as a fallback
+# for failed Twilio deliveries.
 nexmo:
   apiKey:
   apiSecret:
@@ -53,9 +56,6 @@ graphite:
   host:
   port:
 
-http:
-  shutdownGracePeriod: 0s
-
 database:
   # the name of your JDBC driver
   driverClass: org.postgresql.Driver
@@ -72,24 +72,3 @@ database:
   # any properties specific to your JDBC driver:
   properties:
     charSet: UTF-8
-
-  # the maximum amount of time to wait on an empty pool before throwing an exception
-  maxWaitForConnection: 1s
-
-  # the SQL query to run when validating a connection's liveness
-  validationQuery: "/* MyService Health Check */ SELECT 1"
-
-  # the minimum number of connections to keep open
-  minSize: 8
-
-  # the maximum number of connections to keep open
-  maxSize: 32
-
-  # whether or not idle connections should be validated
-  checkConnectionWhileIdle: false
-
-  # how long a connection must be held before it can be validated
-  checkConnectionHealthWhenIdleFor: 10s
-
-  # the maximum lifetime of an idle connection
-  closeConnectionIfIdleFor: 1 minute

pom.xml

@@ -9,24 +9,72 @@
 
     <groupId>org.whispersystems.textsecure</groupId>
     <artifactId>TextSecureServer</artifactId>
-    <version>0.2</version>
+    <version>0.16</version>
+
+    <properties>
+        <dropwizard.version>0.7.0</dropwizard.version>
+        <jackson.api.version>2.3.3</jackson.api.version>
+        <commons-codec.version>1.6</commons-codec.version>
+    </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-core</artifactId>
+            <version>${dropwizard.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-jdbi</artifactId>
+            <version>${dropwizard.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-auth</artifactId>
+            <version>${dropwizard.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-client</artifactId>
+            <version>${dropwizard.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-migrations</artifactId>
+            <version>${dropwizard.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-testing</artifactId>
+            <version>${dropwizard.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.dropwizard</groupId>
+            <artifactId>dropwizard-metrics-graphite</artifactId>
+            <version>${dropwizard.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.sun.jersey</groupId>
+            <artifactId>jersey-json</artifactId>
+            <version>1.18.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.codahale.metrics</groupId>
+            <artifactId>metrics-graphite</artifactId>
+            <version>3.0.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.jetty.websocket</groupId>
+            <artifactId>websocket-server</artifactId>
+            <version>9.0.7.v20131107</version>
+        </dependency>
+
         <dependency>
             <groupId>bouncycastle</groupId>
             <artifactId>bcprov-jdk16</artifactId>
             <version>140</version>
         </dependency>
-        <dependency>
-            <groupId>com.yammer.dropwizard</groupId>
-            <artifactId>dropwizard-core</artifactId>
-            <version>0.6.2</version>
-        </dependency>
-        <dependency>
-            <groupId>com.yammer.metrics</groupId>
-            <artifactId>metrics-graphite</artifactId>
-            <version>2.2.0</version>
-        </dependency>
         <dependency>
             <groupId>com.google.android.gcm</groupId>
             <artifactId>gcm-server</artifactId>
@@ -66,35 +114,10 @@
             <type>jar</type>
             <scope>compile</scope>
         </dependency>
-        <dependency>
-            <groupId>com.yammer.dropwizard</groupId>
-            <artifactId>dropwizard-jdbi</artifactId>
-            <version>0.6.2</version>
-        </dependency>
-        <dependency>
-            <groupId>com.yammer.dropwizard</groupId>
-            <artifactId>dropwizard-auth</artifactId>
-            <version>0.6.2</version>
-        </dependency>
-        <dependency>
-            <groupId>com.yammer.dropwizard</groupId>
-            <artifactId>dropwizard-client</artifactId>
-            <version>0.6.2</version>
-        </dependency>
-        <dependency>
-            <groupId>com.yammer.dropwizard</groupId>
-            <artifactId>dropwizard-migrations</artifactId>
-            <version>0.6.2</version>
-        </dependency>
-        <dependency>
-            <groupId>com.yammer.dropwizard</groupId>
-            <artifactId>dropwizard-testing</artifactId>
-            <version>0.6.2</version>
-        </dependency>
         <dependency>
             <groupId>com.twilio.sdk</groupId>
             <artifactId>twilio-java-sdk</artifactId>
-            <version>3.4.1</version>
+            <version>3.4.5</version>
         </dependency>
 
         <dependency>
@@ -103,14 +126,24 @@
             <version>9.1-901.jdbc4</version>
         </dependency>
 
-        <dependency>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-json</artifactId>
-            <version>1.17.1</version>
-        </dependency>
 
     </dependencies>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>${jackson.api.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-codec</groupId>
+                <artifactId>commons-codec</artifactId>
+                <version>${commons-codec.version}</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <build>
         <plugins>
             <plugin>

protobuf/OutgoingMessageSignal.proto

@@ -22,8 +22,9 @@ option java_outer_classname = "MessageProtos";
 message OutgoingMessageSignal {
   optional uint32 type = 1;
   optional string source = 2;
+  optional  uint32 sourceDevice = 7;
   optional string relay = 3;
-  repeated string destinations = 4;
+//  repeated string destinations = 4;
   optional uint64 timestamp = 5;
   optional bytes message = 6;
 }

src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java

@@ -17,22 +17,25 @@
 package org.whispersystems.textsecuregcm;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.yammer.dropwizard.config.Configuration;
-import com.yammer.dropwizard.db.DatabaseConfiguration;
 import org.whispersystems.textsecuregcm.configuration.ApnConfiguration;
 import org.whispersystems.textsecuregcm.configuration.FederationConfiguration;
 import org.whispersystems.textsecuregcm.configuration.GcmConfiguration;
 import org.whispersystems.textsecuregcm.configuration.GraphiteConfiguration;
 import org.whispersystems.textsecuregcm.configuration.MemcacheConfiguration;
+import org.whispersystems.textsecuregcm.configuration.MetricsConfiguration;
 import org.whispersystems.textsecuregcm.configuration.NexmoConfiguration;
 import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration;
 import org.whispersystems.textsecuregcm.configuration.RedisConfiguration;
 import org.whispersystems.textsecuregcm.configuration.S3Configuration;
 import org.whispersystems.textsecuregcm.configuration.TwilioConfiguration;
+import org.whispersystems.textsecuregcm.configuration.WebsocketConfiguration;
 
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 
+import io.dropwizard.Configuration;
+import io.dropwizard.db.DataSourceFactory;
+
 public class WhisperServerConfiguration extends Configuration {
 
   @NotNull
@@ -72,7 +75,7 @@ public class WhisperServerConfiguration extends Configuration {
   @Valid
   @NotNull
   @JsonProperty
-  private DatabaseConfiguration database = new DatabaseConfiguration();
+  private DataSourceFactory database = new DataSourceFactory();
 
   @Valid
   @NotNull
@@ -83,6 +86,18 @@ public class WhisperServerConfiguration extends Configuration {
   @JsonProperty
   private GraphiteConfiguration graphite = new GraphiteConfiguration();
 
+  @Valid
+  @JsonProperty
+  private MetricsConfiguration viz = new MetricsConfiguration();
+
+  @Valid
+  @JsonProperty
+  private WebsocketConfiguration websocket = new WebsocketConfiguration();
+
+  public WebsocketConfiguration getWebsocketConfiguration() {
+    return websocket;
+  }
+
   public TwilioConfiguration getTwilioConfiguration() {
     return twilio;
   }
@@ -111,7 +126,7 @@ public class WhisperServerConfiguration extends Configuration {
     return redis;
   }
 
-  public DatabaseConfiguration getDatabaseConfiguration() {
+  public DataSourceFactory getDataSourceFactory() {
     return database;
   }
 
@@ -126,4 +141,8 @@ public class WhisperServerConfiguration extends Configuration {
   public GraphiteConfiguration getGraphiteConfiguration() {
     return graphite;
   }
+
+  public MetricsConfiguration getMetricsConfiguration() {
+    return viz;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java

@@ -16,21 +16,21 @@
  */
 package org.whispersystems.textsecuregcm;
 
-import com.yammer.dropwizard.Service;
-import com.yammer.dropwizard.config.Bootstrap;
-import com.yammer.dropwizard.config.Environment;
-import com.yammer.dropwizard.db.DatabaseConfiguration;
-import com.yammer.dropwizard.jdbi.DBIFactory;
-import com.yammer.dropwizard.migrations.MigrationsBundle;
-import com.yammer.metrics.reporting.GraphiteReporter;
+import com.codahale.metrics.SharedMetricRegistries;
+import com.codahale.metrics.graphite.GraphiteReporter;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.google.common.base.Optional;
 import net.spy.memcached.MemcachedClient;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.eclipse.jetty.servlets.CrossOriginFilter;
 import org.skife.jdbi.v2.DBI;
 import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
 import org.whispersystems.textsecuregcm.auth.FederatedPeerAuthenticator;
 import org.whispersystems.textsecuregcm.auth.MultiBasicAuthProvider;
+import org.whispersystems.textsecuregcm.configuration.NexmoConfiguration;
 import org.whispersystems.textsecuregcm.controllers.AccountController;
 import org.whispersystems.textsecuregcm.controllers.AttachmentController;
+import org.whispersystems.textsecuregcm.controllers.DeviceController;
 import org.whispersystems.textsecuregcm.controllers.DirectoryController;
 import org.whispersystems.textsecuregcm.controllers.FederationController;
 import org.whispersystems.textsecuregcm.controllers.KeysController;
@@ -40,28 +40,53 @@ import org.whispersystems.textsecuregcm.federation.FederatedPeer;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
 import org.whispersystems.textsecuregcm.mappers.IOExceptionMapper;
 import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper;
+import org.whispersystems.textsecuregcm.metrics.CpuUsageGauge;
+import org.whispersystems.textsecuregcm.metrics.FreeMemoryGauge;
+import org.whispersystems.textsecuregcm.metrics.JsonMetricsReporter;
+import org.whispersystems.textsecuregcm.metrics.NetworkReceivedGauge;
+import org.whispersystems.textsecuregcm.metrics.NetworkSentGauge;
 import org.whispersystems.textsecuregcm.providers.MemcacheHealthCheck;
 import org.whispersystems.textsecuregcm.providers.MemcachedClientFactory;
 import org.whispersystems.textsecuregcm.providers.RedisClientFactory;
 import org.whispersystems.textsecuregcm.providers.RedisHealthCheck;
 import org.whispersystems.textsecuregcm.push.PushSender;
-import org.whispersystems.textsecuregcm.sms.SenderFactory;
-import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.sms.NexmoSmsSender;
+import org.whispersystems.textsecuregcm.sms.SmsSender;
+import org.whispersystems.textsecuregcm.sms.TwilioSmsSender;
 import org.whispersystems.textsecuregcm.storage.Accounts;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
 import org.whispersystems.textsecuregcm.storage.DirectoryManager;
 import org.whispersystems.textsecuregcm.storage.Keys;
 import org.whispersystems.textsecuregcm.storage.PendingAccounts;
 import org.whispersystems.textsecuregcm.storage.PendingAccountsManager;
+import org.whispersystems.textsecuregcm.storage.PendingDevices;
+import org.whispersystems.textsecuregcm.storage.PendingDevicesManager;
+import org.whispersystems.textsecuregcm.storage.PubSubManager;
+import org.whispersystems.textsecuregcm.storage.StoredMessages;
+import org.whispersystems.textsecuregcm.util.Constants;
 import org.whispersystems.textsecuregcm.util.UrlSigner;
+import org.whispersystems.textsecuregcm.websocket.WebsocketControllerFactory;
 import org.whispersystems.textsecuregcm.workers.DirectoryCommand;
 
+import javax.servlet.DispatcherType;
+import javax.servlet.FilterRegistration;
+import javax.servlet.ServletRegistration;
 import java.security.Security;
+import java.util.EnumSet;
 import java.util.concurrent.TimeUnit;
 
+import static com.codahale.metrics.MetricRegistry.name;
+import io.dropwizard.Application;
+import io.dropwizard.db.DataSourceFactory;
+import io.dropwizard.jdbi.DBIFactory;
+import io.dropwizard.metrics.graphite.GraphiteReporterFactory;
+import io.dropwizard.migrations.MigrationsBundle;
+import io.dropwizard.setup.Bootstrap;
+import io.dropwizard.setup.Environment;
 import redis.clients.jedis.JedisPool;
 
-public class WhisperServerService extends Service<WhisperServerConfiguration> {
+public class WhisperServerService extends Application<WhisperServerConfiguration> {
 
   static {
     Security.addProvider(new BouncyCastleProvider());
@@ -69,25 +94,33 @@ public class WhisperServerService extends Service<WhisperServerConfiguration> {
 
   @Override
   public void initialize(Bootstrap<WhisperServerConfiguration> bootstrap) {
-    bootstrap.setName("whisper-server");
     bootstrap.addCommand(new DirectoryCommand());
     bootstrap.addBundle(new MigrationsBundle<WhisperServerConfiguration>() {
       @Override
-      public DatabaseConfiguration getDatabaseConfiguration(WhisperServerConfiguration configuration) {
-        return configuration.getDatabaseConfiguration();
+      public DataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) {
+        return configuration.getDataSourceFactory();
       }
     });
   }
 
+  @Override
+  public String getName() {
+    return "whisper-server";
+  }
+
   @Override
   public void run(WhisperServerConfiguration config, Environment environment)
       throws Exception
   {
+    SharedMetricRegistries.add(Constants.METRICS_NAME, environment.metrics());
+    environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
     DBIFactory dbiFactory = new DBIFactory();
-    DBI        jdbi       = dbiFactory.build(environment, config.getDatabaseConfiguration(), "postgresql");
+    DBI        jdbi       = dbiFactory.build(environment, config.getDataSourceFactory(), "postgresql");
 
     Accounts        accounts        = jdbi.onDemand(Accounts.class);
     PendingAccounts pendingAccounts = jdbi.onDemand(PendingAccounts.class);
+    PendingDevices  pendingDevices  = jdbi.onDemand(PendingDevices.class);
     Keys            keys            = jdbi.onDemand(Keys.class);
 
     MemcachedClient memcachedClient = new MemcachedClientFactory(config.getMemcacheConfiguration()).getClient();
@@ -95,41 +128,93 @@ public class WhisperServerService extends Service<WhisperServerConfiguration> {
 
     DirectoryManager       directory              = new DirectoryManager(redisClient);
     PendingAccountsManager pendingAccountsManager = new PendingAccountsManager(pendingAccounts, memcachedClient);
+    PendingDevicesManager  pendingDevicesManager  = new PendingDevicesManager (pendingDevices, memcachedClient );
     AccountsManager        accountsManager        = new AccountsManager(accounts, directory, memcachedClient);
-    AccountAuthenticator   accountAuthenticator   = new AccountAuthenticator(accountsManager);
     FederatedClientManager federatedClientManager = new FederatedClientManager(config.getFederationConfiguration());
-    RateLimiters           rateLimiters           = new RateLimiters(config.getLimitsConfiguration(), memcachedClient);
-    SenderFactory          senderFactory          = new SenderFactory(config.getTwilioConfiguration(), config.getNexmoConfiguration());
-    UrlSigner              urlSigner              = new UrlSigner(config.getS3Configuration());
-    PushSender             pushSender             = new PushSender(config.getGcmConfiguration(),
-                                                                   config.getApnConfiguration(),
-                                                                   accountsManager, directory);
+    StoredMessages         storedMessages         = new StoredMessages(redisClient);
+    PubSubManager          pubSubManager          = new PubSubManager(redisClient);
 
-    environment.addProvider(new MultiBasicAuthProvider<>(new FederatedPeerAuthenticator(config.getFederationConfiguration()),
-                                                         FederatedPeer.class,
-                                                         accountAuthenticator,
-                                                         Account.class, "WhisperServer"));
+    AccountAuthenticator     deviceAuthenticator    = new AccountAuthenticator(accountsManager);
+    RateLimiters             rateLimiters           = new RateLimiters(config.getLimitsConfiguration(), memcachedClient);
 
-    environment.addResource(new AccountController(pendingAccountsManager, accountsManager, rateLimiters, senderFactory));
-    environment.addResource(new DirectoryController(rateLimiters, directory));
-    environment.addResource(new AttachmentController(rateLimiters, federatedClientManager, urlSigner));
-    environment.addResource(new KeysController(rateLimiters, keys, federatedClientManager));
-    environment.addResource(new FederationController(keys, accountsManager, pushSender, urlSigner));
+    TwilioSmsSender          twilioSmsSender        = new TwilioSmsSender(config.getTwilioConfiguration());
+    Optional<NexmoSmsSender> nexmoSmsSender         = initializeNexmoSmsSender(config.getNexmoConfiguration());
+    SmsSender                smsSender              = new SmsSender(twilioSmsSender, nexmoSmsSender, config.getTwilioConfiguration().isInternational());
+    UrlSigner                urlSigner              = new UrlSigner(config.getS3Configuration());
+    PushSender               pushSender             = new PushSender(config.getGcmConfiguration(),
+                                                                     config.getApnConfiguration(),
+                                                                     storedMessages, pubSubManager,
+                                                                     accountsManager);
 
-    environment.addServlet(new MessageController(rateLimiters, accountAuthenticator,
-                                                 pushSender, federatedClientManager),
-                           MessageController.PATH);
+    AttachmentController attachmentController = new AttachmentController(rateLimiters, federatedClientManager, urlSigner);
+    KeysController       keysController       = new KeysController(rateLimiters, keys, accountsManager, federatedClientManager);
+    MessageController    messageController    = new MessageController(rateLimiters, pushSender, accountsManager, federatedClientManager);
 
-    environment.addHealthCheck(new RedisHealthCheck(redisClient));
-    environment.addHealthCheck(new MemcacheHealthCheck(memcachedClient));
+    environment.jersey().register(new MultiBasicAuthProvider<>(new FederatedPeerAuthenticator(config.getFederationConfiguration()),
+                                                               FederatedPeer.class,
+                                                               deviceAuthenticator,
+                                                               Device.class, "WhisperServer"));
 
-    environment.addProvider(new IOExceptionMapper());
-    environment.addProvider(new RateLimitExceededExceptionMapper());
+    environment.jersey().register(new AccountController(pendingAccountsManager, accountsManager, rateLimiters, smsSender));
+    environment.jersey().register(new DeviceController(pendingDevicesManager, accountsManager, rateLimiters));
+    environment.jersey().register(new DirectoryController(rateLimiters, directory));
+    environment.jersey().register(new FederationController(accountsManager, attachmentController, keysController, messageController));
+    environment.jersey().register(attachmentController);
+    environment.jersey().register(keysController);
+    environment.jersey().register(messageController);
+
+    if (config.getWebsocketConfiguration().isEnabled()) {
+      WebsocketControllerFactory servlet = new WebsocketControllerFactory(deviceAuthenticator,
+                                                                          pushSender,
+                                                                          storedMessages,
+                                                                          pubSubManager);
+
+      ServletRegistration.Dynamic websocket = environment.servlets().addServlet("WebSocket", servlet);
+      websocket.addMapping("/v1/websocket/*");
+      websocket.setAsyncSupported(true);
+
+      FilterRegistration.Dynamic filter = environment.servlets().addFilter("CORS", CrossOriginFilter.class);
+      filter.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*");
+      filter.setInitParameter("allowedOrigins", "*");
+      filter.setInitParameter("allowedHeaders", "Content-Type,Authorization,X-Requested-With,Content-Length,Accept,Origin");
+      filter.setInitParameter("allowedMethods", "GET,PUT,POST,DELETE,OPTIONS");
+      filter.setInitParameter("preflightMaxAge", "5184000");
+      filter.setInitParameter("allowCredentials", "true");
+    }
+
+    environment.healthChecks().register("redis", new RedisHealthCheck(redisClient));
+    environment.healthChecks().register("memcache", new MemcacheHealthCheck(memcachedClient));
+
+    environment.jersey().register(new IOExceptionMapper());
+    environment.jersey().register(new RateLimitExceededExceptionMapper());
+
+    environment.metrics().register(name(CpuUsageGauge.class, "cpu"), new CpuUsageGauge());
+    environment.metrics().register(name(FreeMemoryGauge.class, "free_memory"), new FreeMemoryGauge());
+    environment.metrics().register(name(NetworkSentGauge.class, "bytes_sent"), new NetworkSentGauge());
+    environment.metrics().register(name(NetworkReceivedGauge.class, "bytes_received"), new NetworkReceivedGauge());
 
     if (config.getGraphiteConfiguration().isEnabled()) {
-      GraphiteReporter.enable(15, TimeUnit.SECONDS,
-                              config.getGraphiteConfiguration().getHost(),
-                              config.getGraphiteConfiguration().getPort());
+      GraphiteReporterFactory graphiteReporterFactory = new GraphiteReporterFactory();
+      graphiteReporterFactory.setHost(config.getGraphiteConfiguration().getHost());
+      graphiteReporterFactory.setPort(config.getGraphiteConfiguration().getPort());
+
+      GraphiteReporter graphiteReporter = (GraphiteReporter) graphiteReporterFactory.build(environment.metrics());
+      graphiteReporter.start(15, TimeUnit.SECONDS);
+    }
+
+    if (config.getMetricsConfiguration().isEnabled()) {
+      new JsonMetricsReporter(environment.metrics(),
+                              config.getMetricsConfiguration().getToken(),
+                              config.getMetricsConfiguration().getHost())
+          .start(60, TimeUnit.SECONDS);
+    }
+  }
+
+  private Optional<NexmoSmsSender> initializeNexmoSmsSender(NexmoConfiguration configuration) {
+    if (configuration == null) {
+      return Optional.absent();
+    } else {
+      return Optional.of(new NexmoSmsSender(configuration));
     }
   }
 

src/main/java/org/whispersystems/textsecuregcm/auth/AccountAuthenticator.java

@@ -16,28 +16,27 @@
  */
 package org.whispersystems.textsecuregcm.auth;
 
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import com.google.common.base.Optional;
-import com.yammer.dropwizard.auth.AuthenticationException;
-import com.yammer.dropwizard.auth.Authenticator;
-import com.yammer.dropwizard.auth.basic.BasicCredentials;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.util.Constants;
 
-import java.util.concurrent.TimeUnit;
+import static com.codahale.metrics.MetricRegistry.name;
+import io.dropwizard.auth.AuthenticationException;
+import io.dropwizard.auth.Authenticator;
+import io.dropwizard.auth.basic.BasicCredentials;
 
 public class AccountAuthenticator implements Authenticator<BasicCredentials, Account> {
 
-  private final Meter authenticationFailedMeter    = Metrics.newMeter(AccountAuthenticator.class,
-                                                                      "authentication", "failed",
-                                                                      TimeUnit.MINUTES);
-
-  private final Meter authenticationSucceededMeter = Metrics.newMeter(AccountAuthenticator.class,
-                                                                      "authentication", "succeeded",
-                                                                      TimeUnit.MINUTES);
+  private final MetricRegistry metricRegistry               = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+  private final Meter          authenticationFailedMeter    = metricRegistry.meter(name(getClass(), "authentication", "failed"   ));
+  private final Meter          authenticationSucceededMeter = metricRegistry.meter(name(getClass(), "authentication", "succeeded"));
 
   private final Logger logger = LoggerFactory.getLogger(AccountAuthenticator.class);
 
@@ -51,18 +50,30 @@ public class AccountAuthenticator implements Authenticator<BasicCredentials, Acc
   public Optional<Account> authenticate(BasicCredentials basicCredentials)
       throws AuthenticationException
   {
-    Optional<Account> account = accountsManager.get(basicCredentials.getUsername());
+    try {
+      AuthorizationHeader authorizationHeader = AuthorizationHeader.fromUserAndPassword(basicCredentials.getUsername(), basicCredentials.getPassword());
+      Optional<Account>   account             = accountsManager.get(authorizationHeader.getNumber());
 
-    if (!account.isPresent()) {
+      if (!account.isPresent()) {
+        return Optional.absent();
+      }
+
+      Optional<Device> device = account.get().getDevice(authorizationHeader.getDeviceId());
+
+      if (!device.isPresent()) {
+        return Optional.absent();
+      }
+
+      if (device.get().getAuthenticationCredentials().verify(basicCredentials.getPassword())) {
+        authenticationSucceededMeter.mark();
+        account.get().setAuthenticatedDevice(device.get());
+        return account;
+      }
+
+      authenticationFailedMeter.mark();
+      return Optional.absent();
+    } catch (InvalidAuthorizationHeaderException iahe) {
       return Optional.absent();
     }
-
-    if (account.get().getAuthenticationCredentials().verify(basicCredentials.getPassword())) {
-      authenticationSucceededMeter.mark();
-      return account;
-    }
-
-    authenticationFailedMeter.mark();
-    return Optional.absent();
   }
 }

src/main/java/org/whispersystems/textsecuregcm/auth/AuthorizationHeader.java

@@ -24,10 +24,28 @@ import java.io.IOException;
 
 public class AuthorizationHeader {
 
-  private final String user;
+  private final String number;
+  private final long   accountId;
   private final String password;
 
-  public AuthorizationHeader(String header) throws InvalidAuthorizationHeaderException {
+  private AuthorizationHeader(String number, long accountId, String password) {
+    this.number    = number;
+    this.accountId = accountId;
+    this.password  = password;
+  }
+
+  public static AuthorizationHeader fromUserAndPassword(String user, String password) throws InvalidAuthorizationHeaderException {
+    try {
+      String[] numberAndId = user.split("\\.");
+      return new AuthorizationHeader(numberAndId[0],
+                                     numberAndId.length > 1 ? Long.parseLong(numberAndId[1]) : 1,
+                                     password);
+    } catch (NumberFormatException nfe) {
+      throw new InvalidAuthorizationHeaderException(nfe);
+    }
+  }
+
+  public static AuthorizationHeader fromFullHeader(String header) throws InvalidAuthorizationHeaderException {
     try {
       if (header == null) {
         throw new InvalidAuthorizationHeaderException("Null header");
@@ -55,16 +73,18 @@ public class AuthorizationHeader {
         throw new InvalidAuthorizationHeaderException("Badly formated credentials: " + concatenatedValues);
       }
 
-      this.user     = credentialParts[0];
-      this.password = credentialParts[1];
-
+      return fromUserAndPassword(credentialParts[0], credentialParts[1]);
     } catch (IOException ioe) {
       throw new InvalidAuthorizationHeaderException(ioe);
     }
   }
 
-  public String getUserName() {
-    return user;
+  public String getNumber() {
+    return number;
+  }
+
+  public long getDeviceId() {
+    return accountId;
   }
 
   public String getPassword() {

src/main/java/org/whispersystems/textsecuregcm/auth/FederatedPeerAuthenticator.java

@@ -16,30 +16,35 @@
  */
 package org.whispersystems.textsecuregcm.auth;
 
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import com.google.common.base.Optional;
-import com.yammer.dropwizard.auth.AuthenticationException;
-import com.yammer.dropwizard.auth.Authenticator;
-import com.yammer.dropwizard.auth.basic.BasicCredentials;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.configuration.FederationConfiguration;
 import org.whispersystems.textsecuregcm.federation.FederatedPeer;
+import org.whispersystems.textsecuregcm.util.Constants;
 
 import java.util.List;
-import java.util.concurrent.TimeUnit;
+
+import static com.codahale.metrics.MetricRegistry.name;
+import io.dropwizard.auth.AuthenticationException;
+import io.dropwizard.auth.Authenticator;
+import io.dropwizard.auth.basic.BasicCredentials;
 
 
 public class FederatedPeerAuthenticator implements Authenticator<BasicCredentials, FederatedPeer> {
 
-  private final Meter authenticationFailedMeter    = Metrics.newMeter(FederatedPeerAuthenticator.class,
-                                                                      "authentication", "failed",
-                                                                      TimeUnit.MINUTES);
+  private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
 
-  private final Meter authenticationSucceededMeter = Metrics.newMeter(FederatedPeerAuthenticator.class,
-                                                                      "authentication", "succeeded",
-                                                                      TimeUnit.MINUTES);
+  private final Meter authenticationFailedMeter    = metricRegistry.meter(name(getClass(),
+                                                                               "authentication",
+                                                                               "failed"));
+
+  private final Meter authenticationSucceededMeter = metricRegistry.meter(name(getClass(),
+                                                                               "authentication",
+                                                                               "succeeded"));
 
   private final Logger logger = LoggerFactory.getLogger(FederatedPeerAuthenticator.class);
 

src/main/java/org/whispersystems/textsecuregcm/auth/MultiBasicAuthProvider.java

@@ -21,17 +21,14 @@ import com.sun.jersey.core.spi.component.ComponentContext;
 import com.sun.jersey.core.spi.component.ComponentScope;
 import com.sun.jersey.spi.inject.Injectable;
 import com.sun.jersey.spi.inject.InjectableProvider;
-import com.yammer.dropwizard.auth.Auth;
-import com.yammer.dropwizard.auth.Authenticator;
-import com.yammer.dropwizard.auth.basic.BasicAuthProvider;
-import com.yammer.dropwizard.auth.basic.BasicCredentials;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import io.dropwizard.auth.Auth;
+import io.dropwizard.auth.Authenticator;
+import io.dropwizard.auth.basic.BasicAuthProvider;
+import io.dropwizard.auth.basic.BasicCredentials;
 
 public class MultiBasicAuthProvider<T1,T2> implements InjectableProvider<Auth, Parameter> {
 
-  private final Logger logger = LoggerFactory.getLogger(MultiBasicAuthProvider.class);
-
   private final BasicAuthProvider<T1> provider1;
   private final BasicAuthProvider<T2> provider2;
 
@@ -44,8 +41,8 @@ public class MultiBasicAuthProvider<T1,T2> implements InjectableProvider<Auth, P
                                 Class<?> clazz2,
                                 String realm)
   {
-    this.provider1 = new BasicAuthProvider<T1>(authenticator1, realm);
-    this.provider2 = new BasicAuthProvider<T2>(authenticator2, realm);
+    this.provider1 = new BasicAuthProvider<>(authenticator1, realm);
+    this.provider2 = new BasicAuthProvider<>(authenticator2, realm);
     this.clazz1    = clazz1;
     this.clazz2    = clazz2;
   }

src/main/java/org/whispersystems/textsecuregcm/configuration/MetricsConfiguration.java

@@ -0,0 +1,27 @@
+package org.whispersystems.textsecuregcm.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class MetricsConfiguration {
+
+  @JsonProperty
+  private String token;
+
+  @JsonProperty
+  private String host;
+
+  @JsonProperty
+  private boolean enabled = false;
+
+  public String getHost() {
+    return host;
+  }
+
+  public String getToken() {
+    return token;
+  }
+
+  public boolean isEnabled() {
+    return enabled && token != null && host != null;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/configuration/RateLimitsConfiguration.java

@@ -41,6 +41,20 @@ public class RateLimitsConfiguration {
   @JsonProperty
   private RateLimitConfiguration messages = new RateLimitConfiguration(60, 60);
 
+  @JsonProperty
+  private RateLimitConfiguration allocateDevice = new RateLimitConfiguration(2, 1.0 / 2.0);
+
+  @JsonProperty
+  private RateLimitConfiguration verifyDevice = new RateLimitConfiguration(2, 2);
+
+  public RateLimitConfiguration getAllocateDevice() {
+    return allocateDevice;
+  }
+
+  public RateLimitConfiguration getVerifyDevice() {
+    return verifyDevice;
+  }
+
   public RateLimitConfiguration getMessages() {
     return messages;
   }

src/main/java/org/whispersystems/textsecuregcm/configuration/TwilioConfiguration.java

@@ -37,6 +37,9 @@ public class TwilioConfiguration {
   @JsonProperty
   private String localDomain;
 
+  @JsonProperty
+  private boolean international;
+
   public String getAccountId() {
     return accountId;
   }
@@ -52,4 +55,8 @@ public class TwilioConfiguration {
   public String getLocalDomain() {
     return localDomain;
   }
+
+  public boolean isInternational() {
+    return international;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/configuration/WebsocketConfiguration.java

@@ -0,0 +1,14 @@
+package org.whispersystems.textsecuregcm.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class WebsocketConfiguration {
+
+  @JsonProperty
+  private boolean enabled = false;
+
+  public boolean isEnabled() {
+    return enabled;
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/controllers/AccountController.java

@@ -16,9 +16,9 @@
  */
 package org.whispersystems.textsecuregcm.controllers;
 
+import com.codahale.metrics.annotation.Timed;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
-import com.yammer.dropwizard.auth.Auth;
-import com.yammer.metrics.annotation.Timed;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
@@ -28,10 +28,11 @@ import org.whispersystems.textsecuregcm.entities.AccountAttributes;
 import org.whispersystems.textsecuregcm.entities.ApnRegistrationId;
 import org.whispersystems.textsecuregcm.entities.GcmRegistrationId;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
-import org.whispersystems.textsecuregcm.sms.SenderFactory;
+import org.whispersystems.textsecuregcm.sms.SmsSender;
 import org.whispersystems.textsecuregcm.sms.TwilioSmsSender;
 import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
 import org.whispersystems.textsecuregcm.storage.PendingAccountsManager;
 import org.whispersystems.textsecuregcm.util.Util;
 import org.whispersystems.textsecuregcm.util.VerificationCode;
@@ -53,6 +54,8 @@ import java.io.IOException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 
+import io.dropwizard.auth.Auth;
+
 @Path("/v1/accounts")
 public class AccountController {
 
@@ -61,17 +64,17 @@ public class AccountController {
   private final PendingAccountsManager pendingAccounts;
   private final AccountsManager        accounts;
   private final RateLimiters           rateLimiters;
-  private final SenderFactory          senderFactory;
+  private final SmsSender              smsSender;
 
   public AccountController(PendingAccountsManager pendingAccounts,
                            AccountsManager accounts,
                            RateLimiters rateLimiters,
-                           SenderFactory smsSenderFactory)
+                           SmsSender smsSenderFactory)
   {
     this.pendingAccounts = pendingAccounts;
     this.accounts        = accounts;
     this.rateLimiters    = rateLimiters;
-    this.senderFactory   = smsSenderFactory;
+    this.smsSender       = smsSenderFactory;
   }
 
   @Timed
@@ -94,16 +97,16 @@ public class AccountController {
         rateLimiters.getVoiceDestinationLimiter().validate(number);
         break;
       default:
-        throw new WebApplicationException(Response.status(415).build());
+        throw new WebApplicationException(Response.status(422).build());
     }
 
     VerificationCode verificationCode = generateVerificationCode();
     pendingAccounts.store(number, verificationCode.getVerificationCode());
 
     if (transport.equals("sms")) {
-      senderFactory.getSmsSender(number).deliverSmsVerification(number, verificationCode.getVerificationCodeDisplay());
+      smsSender.deliverSmsVerification(number, verificationCode.getVerificationCodeDisplay());
     } else if (transport.equals("voice")) {
-      senderFactory.getVoxSender(number).deliverVoxVerification(number, verificationCode.getVerificationCodeSpeech());
+      smsSender.deliverVoxVerification(number, verificationCode.getVerificationCodeSpeech());
     }
 
     return Response.ok().build();
@@ -119,8 +122,8 @@ public class AccountController {
       throws RateLimitExceededException
   {
     try {
-      AuthorizationHeader header = new AuthorizationHeader(authorizationHeader);
-      String number              = header.getUserName();
+      AuthorizationHeader header = AuthorizationHeader.fromFullHeader(authorizationHeader);
+      String number              = header.getNumber();
       String password            = header.getPassword();
 
       rateLimiters.getVerifyLimiter().validate(number);
@@ -133,28 +136,43 @@ public class AccountController {
         throw new WebApplicationException(Response.status(403).build());
       }
 
+      if (accounts.isRelayListed(number)) {
+        throw new WebApplicationException(Response.status(417).build());
+      }
+
+      Device device = new Device();
+      device.setId(Device.MASTER_ID);
+      device.setAuthenticationCredentials(new AuthenticationCredentials(password));
+      device.setSignalingKey(accountAttributes.getSignalingKey());
+      device.setFetchesMessages(accountAttributes.getFetchesMessages());
+      device.setRegistrationId(accountAttributes.getRegistrationId());
+
       Account account = new Account();
       account.setNumber(number);
-      account.setAuthenticationCredentials(new AuthenticationCredentials(password));
-      account.setSignalingKey(accountAttributes.getSignalingKey());
       account.setSupportsSms(accountAttributes.getSupportsSms());
+      account.addDevice(device);
 
       accounts.create(account);
-      logger.debug("Stored account...");
 
+      pendingAccounts.remove(number);
+
+      logger.debug("Stored device...");
     } catch (InvalidAuthorizationHeaderException e) {
       logger.info("Bad Authorization Header", e);
       throw new WebApplicationException(Response.status(401).build());
     }
   }
 
+
+
   @Timed
   @PUT
   @Path("/gcm/")
   @Consumes(MediaType.APPLICATION_JSON)
   public void setGcmRegistrationId(@Auth Account account, @Valid GcmRegistrationId registrationId)  {
-    account.setApnRegistrationId(null);
-    account.setGcmRegistrationId(registrationId.getGcmRegistrationId());
+    Device device = account.getAuthenticatedDevice().get();
+    device.setApnId(null);
+    device.setGcmId(registrationId.getGcmRegistrationId());
     accounts.update(account);
   }
 
@@ -162,7 +180,8 @@ public class AccountController {
   @DELETE
   @Path("/gcm/")
   public void deleteGcmRegistrationId(@Auth Account account) {
-    account.setGcmRegistrationId(null);
+    Device device = account.getAuthenticatedDevice().get();
+    device.setGcmId(null);
     accounts.update(account);
   }
 
@@ -171,8 +190,9 @@ public class AccountController {
   @Path("/apn/")
   @Consumes(MediaType.APPLICATION_JSON)
   public void setApnRegistrationId(@Auth Account account, @Valid ApnRegistrationId registrationId) {
-    account.setApnRegistrationId(registrationId.getApnRegistrationId());
-    account.setGcmRegistrationId(null);
+    Device device = account.getAuthenticatedDevice().get();
+    device.setApnId(registrationId.getApnRegistrationId());
+    device.setGcmId(null);
     accounts.update(account);
   }
 
@@ -180,7 +200,8 @@ public class AccountController {
   @DELETE
   @Path("/apn/")
   public void deleteApnRegistrationId(@Auth Account account) {
-    account.setApnRegistrationId(null);
+    Device device = account.getAuthenticatedDevice().get();
+    device.setApnId(null);
     accounts.update(account);
   }
 
@@ -190,11 +211,10 @@ public class AccountController {
   @Produces(MediaType.APPLICATION_XML)
   public Response getTwiml(@PathParam("code") String encodedVerificationText) {
     return Response.ok().entity(String.format(TwilioSmsSender.SAY_TWIML,
-                                              SenderFactory.VoxSender.VERIFICATION_TEXT +
-                                                  encodedVerificationText)).build();
+        encodedVerificationText)).build();
   }
 
-  private VerificationCode generateVerificationCode() {
+  @VisibleForTesting protected VerificationCode generateVerificationCode() {
     try {
       SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
       int randomInt       = 100000 + random.nextInt(900000);
@@ -203,5 +223,4 @@ public class AccountController {
       throw new AssertionError(e);
     }
   }
-
 }

src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentController.java

@@ -17,8 +17,8 @@
 package org.whispersystems.textsecuregcm.controllers;
 
 import com.amazonaws.HttpMethod;
-import com.yammer.dropwizard.auth.Auth;
-import com.yammer.metrics.annotation.Timed;
+import com.codahale.metrics.annotation.Timed;
+import com.google.common.base.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.entities.AttachmentDescriptor;
@@ -35,14 +35,16 @@ import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.io.IOException;
-import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 
+import io.dropwizard.auth.Auth;
+
 
 @Path("/v1/attachments")
 public class AttachmentController {
@@ -65,37 +67,38 @@ public class AttachmentController {
   @Timed
   @GET
   @Produces(MediaType.APPLICATION_JSON)
-  public Response allocateAttachment(@Auth Account account) throws RateLimitExceededException {
-    rateLimiters.getAttachmentLimiter().validate(account.getNumber());
+  public AttachmentDescriptor allocateAttachment(@Auth Account account)
+      throws RateLimitExceededException
+  {
+    if (account.isRateLimited()) {
+      rateLimiters.getAttachmentLimiter().validate(account.getNumber());
+    }
 
-    long                 attachmentId = generateAttachmentId();
-    URL                  url          = urlSigner.getPreSignedUrl(attachmentId, HttpMethod.PUT);
-    AttachmentDescriptor descriptor   = new AttachmentDescriptor(attachmentId, url.toExternalForm());
+    long attachmentId = generateAttachmentId();
+    URL  url          = urlSigner.getPreSignedUrl(attachmentId, HttpMethod.PUT);
+
+    return new AttachmentDescriptor(attachmentId, url.toExternalForm());
 
-    return Response.ok().entity(descriptor).build();
   }
 
   @Timed
   @GET
   @Produces(MediaType.APPLICATION_JSON)
   @Path("/{attachmentId}")
-  public Response redirectToAttachment(@Auth                      Account account,
-                                       @PathParam("attachmentId") long attachmentId,
-                                       @QueryParam("relay")       String relay)
+  public AttachmentUri redirectToAttachment(@Auth                      Account account,
+                                            @PathParam("attachmentId") long    attachmentId,
+                                            @QueryParam("relay")       Optional<String> relay)
+      throws IOException
   {
     try {
-      URL url;
-
-      if (relay == null) url = urlSigner.getPreSignedUrl(attachmentId, HttpMethod.GET);
-      else               url = federatedClientManager.getClient(relay).getSignedAttachmentUri(attachmentId);
-
-      return Response.ok().entity(new AttachmentUri(url)).build();
-    } catch (IOException e) {
-      logger.warn("No conectivity", e);
-      return Response.status(500).build();
+      if (!relay.isPresent()) {
+        return new AttachmentUri(urlSigner.getPreSignedUrl(attachmentId, HttpMethod.GET));
+      } else {
+        return new AttachmentUri(federatedClientManager.getClient(relay.get()).getSignedAttachmentUri(attachmentId));
+      }
     } catch (NoSuchPeerException e) {
       logger.info("No such peer: " + relay);
-      return Response.status(404).build();
+      throw new WebApplicationException(Response.status(404).build());
     }
   }
 

src/main/java/org/whispersystems/textsecuregcm/controllers/DeviceController.java

@@ -0,0 +1,143 @@
+/**
+ * Copyright (C) 2013 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.controllers;
+
+import com.codahale.metrics.annotation.Timed;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
+import org.whispersystems.textsecuregcm.auth.AuthorizationHeader;
+import org.whispersystems.textsecuregcm.auth.InvalidAuthorizationHeaderException;
+import org.whispersystems.textsecuregcm.entities.AccountAttributes;
+import org.whispersystems.textsecuregcm.entities.DeviceResponse;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.storage.PendingDevicesManager;
+import org.whispersystems.textsecuregcm.util.VerificationCode;
+
+import javax.validation.Valid;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.HeaderParam;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+
+import io.dropwizard.auth.Auth;
+
+@Path("/v1/devices")
+public class DeviceController {
+
+  private final Logger logger = LoggerFactory.getLogger(DeviceController.class);
+
+  private final PendingDevicesManager pendingDevices;
+  private final AccountsManager       accounts;
+  private final RateLimiters          rateLimiters;
+
+  public DeviceController(PendingDevicesManager pendingDevices,
+                          AccountsManager accounts,
+                          RateLimiters rateLimiters)
+  {
+    this.pendingDevices  = pendingDevices;
+    this.accounts        = accounts;
+    this.rateLimiters    = rateLimiters;
+  }
+
+  @Timed
+  @GET
+  @Path("/provisioning_code")
+  @Produces(MediaType.APPLICATION_JSON)
+  public VerificationCode createDeviceToken(@Auth Account account)
+      throws RateLimitExceededException
+  {
+    rateLimiters.getAllocateDeviceLimiter().validate(account.getNumber());
+
+    VerificationCode verificationCode = generateVerificationCode();
+    pendingDevices.store(account.getNumber(), verificationCode.getVerificationCode());
+
+    return verificationCode;
+  }
+
+  @Timed
+  @PUT
+  @Produces(MediaType.APPLICATION_JSON)
+  @Consumes(MediaType.APPLICATION_JSON)
+  @Path("/{verification_code}")
+  public DeviceResponse verifyDeviceToken(@PathParam("verification_code") String verificationCode,
+                                          @HeaderParam("Authorization")   String authorizationHeader,
+                                          @Valid                          AccountAttributes accountAttributes)
+      throws RateLimitExceededException
+  {
+    try {
+      AuthorizationHeader header = AuthorizationHeader.fromFullHeader(authorizationHeader);
+      String number              = header.getNumber();
+      String password            = header.getPassword();
+
+      rateLimiters.getVerifyDeviceLimiter().validate(number);
+
+      Optional<String> storedVerificationCode = pendingDevices.getCodeForNumber(number);
+
+      if (!storedVerificationCode.isPresent() ||
+          !verificationCode.equals(storedVerificationCode.get()))
+      {
+        throw new WebApplicationException(Response.status(403).build());
+      }
+
+      Optional<Account> account = accounts.get(number);
+
+      if (!account.isPresent()) {
+        throw new WebApplicationException(Response.status(403).build());
+      }
+
+      Device device = new Device();
+      device.setAuthenticationCredentials(new AuthenticationCredentials(password));
+      device.setSignalingKey(accountAttributes.getSignalingKey());
+      device.setFetchesMessages(accountAttributes.getFetchesMessages());
+      device.setId(account.get().getNextDeviceId());
+
+      account.get().addDevice(device);
+      accounts.update(account.get());
+
+      pendingDevices.remove(number);
+
+      return new DeviceResponse(device.getId());
+    } catch (InvalidAuthorizationHeaderException e) {
+      logger.info("Bad Authorization Header", e);
+      throw new WebApplicationException(Response.status(401).build());
+    }
+  }
+
+  @VisibleForTesting protected VerificationCode generateVerificationCode() {
+    try {
+      SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
+      int randomInt       = 100000 + random.nextInt(900000);
+      return new VerificationCode(randomInt);
+    } catch (NoSuchAlgorithmException e) {
+      throw new AssertionError(e);
+    }
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/controllers/DirectoryController.java

@@ -16,19 +16,21 @@
  */
 package org.whispersystems.textsecuregcm.controllers;
 
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
+import com.codahale.metrics.annotation.Timed;
 import com.google.common.base.Optional;
-import com.yammer.dropwizard.auth.Auth;
-import com.yammer.metrics.annotation.Metered;
-import com.yammer.metrics.annotation.Timed;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.limits.RateLimiters;
-import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.entities.ClientContact;
 import org.whispersystems.textsecuregcm.entities.ClientContactTokens;
 import org.whispersystems.textsecuregcm.entities.ClientContacts;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.storage.DirectoryManager;
 import org.whispersystems.textsecuregcm.util.Base64;
+import org.whispersystems.textsecuregcm.util.Constants;
 
 import javax.validation.Valid;
 import javax.ws.rs.Consumes;
@@ -44,10 +46,15 @@ import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
 
+import static com.codahale.metrics.MetricRegistry.name;
+import io.dropwizard.auth.Auth;
+
 @Path("/v1/directory")
 public class DirectoryController {
 
-  private final Logger logger = LoggerFactory.getLogger(DirectoryController.class);
+  private final Logger         logger            = LoggerFactory.getLogger(DirectoryController.class);
+  private final MetricRegistry metricRegistry    = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+  private final Histogram      contactsHistogram = metricRegistry.histogram(name(getClass(), "contacts"));
 
   private final RateLimiters     rateLimiters;
   private final DirectoryManager directory;
@@ -57,7 +64,7 @@ public class DirectoryController {
     this.rateLimiters = rateLimiters;
   }
 
-  @Timed()
+  @Timed
   @GET
   @Path("/{token}")
   @Produces(MediaType.APPLICATION_JSON)
@@ -78,7 +85,7 @@ public class DirectoryController {
     }
   }
 
-  @Timed()
+  @Timed
   @PUT
   @Path("/tokens")
   @Produces(MediaType.APPLICATION_JSON)
@@ -87,6 +94,7 @@ public class DirectoryController {
       throws RateLimitExceededException
   {
     rateLimiters.getContactsLimiter().validate(account.getNumber(), contacts.getContacts().size());
+    contactsHistogram.update(contacts.getContacts().size());
 
     try {
       List<byte[]> tokens = new LinkedList<>();

src/main/java/org/whispersystems/textsecuregcm/controllers/FederationController.java

@@ -16,42 +16,36 @@
  */
 package org.whispersystems.textsecuregcm.controllers;
 
-import com.amazonaws.HttpMethod;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.yammer.dropwizard.auth.Auth;
-import com.yammer.metrics.annotation.Timed;
+import com.codahale.metrics.annotation.Timed;
+import com.google.common.base.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.entities.AccountCount;
 import org.whispersystems.textsecuregcm.entities.AttachmentUri;
 import org.whispersystems.textsecuregcm.entities.ClientContact;
 import org.whispersystems.textsecuregcm.entities.ClientContacts;
-import org.whispersystems.textsecuregcm.entities.MessageProtos.OutgoingMessageSignal;
+import org.whispersystems.textsecuregcm.entities.IncomingMessageList;
 import org.whispersystems.textsecuregcm.entities.PreKey;
-import org.whispersystems.textsecuregcm.entities.RelayMessage;
+import org.whispersystems.textsecuregcm.entities.UnstructuredPreKeyList;
 import org.whispersystems.textsecuregcm.federation.FederatedPeer;
-import org.whispersystems.textsecuregcm.push.PushSender;
+import org.whispersystems.textsecuregcm.federation.NonLimitedAccount;
 import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
-import org.whispersystems.textsecuregcm.storage.Keys;
-import org.whispersystems.textsecuregcm.util.UrlSigner;
 import org.whispersystems.textsecuregcm.util.Util;
 
 import javax.validation.Valid;
-import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
 import java.io.IOException;
-import java.net.URL;
 import java.util.LinkedList;
 import java.util.List;
 
+import io.dropwizard.auth.Auth;
+
 @Path("/v1/federation")
 public class FederationController {
 
@@ -59,16 +53,20 @@ public class FederationController {
 
   private static final int ACCOUNT_CHUNK_SIZE = 10000;
 
-  private final PushSender      pushSender;
-  private final Keys            keys;
-  private final AccountsManager accounts;
-  private final UrlSigner       urlSigner;
+  private final AccountsManager      accounts;
+  private final AttachmentController attachmentController;
+  private final KeysController       keysController;
+  private final MessageController    messageController;
 
-  public FederationController(Keys keys, AccountsManager accounts, PushSender pushSender, UrlSigner urlSigner) {
-    this.keys       = keys;
-    this.accounts   = accounts;
-    this.pushSender = pushSender;
-    this.urlSigner  = urlSigner;
+  public FederationController(AccountsManager      accounts,
+                              AttachmentController attachmentController,
+                              KeysController       keysController,
+                              MessageController     messageController)
+  {
+    this.accounts             = accounts;
+    this.attachmentController = attachmentController;
+    this.keysController       = keysController;
+    this.messageController    = messageController;
   }
 
   @Timed
@@ -77,9 +75,10 @@ public class FederationController {
   @Produces(MediaType.APPLICATION_JSON)
   public AttachmentUri getSignedAttachmentUri(@Auth                      FederatedPeer peer,
                                               @PathParam("attachmentId") long attachmentId)
+      throws IOException
   {
-    URL url = urlSigner.getPreSignedUrl(attachmentId, HttpMethod.GET);
-    return new AttachmentUri(url);
+    return attachmentController.redirectToAttachment(new NonLimitedAccount("Unknown", -1, peer.getName()),
+                                                     attachmentId, Optional.<String>absent());
   }
 
   @Timed
@@ -88,36 +87,50 @@ public class FederationController {
   @Produces(MediaType.APPLICATION_JSON)
   public PreKey getKey(@Auth                FederatedPeer peer,
                        @PathParam("number") String number)
+      throws IOException
   {
-    PreKey preKey = keys.get(number);
-
-    if (preKey == null) {
-      throw new WebApplicationException(Response.status(404).build());
+    try {
+      return keysController.get(new NonLimitedAccount("Unknown", -1, peer.getName()), number, Optional.<String>absent());
+    } catch (RateLimitExceededException e) {
+      logger.warn("Rate limiting on federated channel", e);
+      throw new IOException(e);
     }
+  }
 
-    return preKey;
+  @Timed
+  @GET
+  @Path("/key/{number}/{device}")
+  @Produces(MediaType.APPLICATION_JSON)
+  public UnstructuredPreKeyList getKeys(@Auth                FederatedPeer peer,
+                                        @PathParam("number") String number,
+                                        @PathParam("device") String device)
+      throws IOException
+  {
+    try {
+      return keysController.getDeviceKey(new NonLimitedAccount("Unknown", -1, peer.getName()),
+                                         number, device, Optional.<String>absent());
+    } catch (RateLimitExceededException e) {
+      logger.warn("Rate limiting on federated channel", e);
+      throw new IOException(e);
+    }
   }
 
   @Timed
   @PUT
-  @Path("/message")
-  @Consumes(MediaType.APPLICATION_JSON)
-  public void relayMessage(@Auth FederatedPeer peer, @Valid RelayMessage message)
+  @Path("/messages/{source}/{sourceDeviceId}/{destination}")
+  public void sendMessages(@Auth                        FederatedPeer peer,
+                           @PathParam("source")         String source,
+                           @PathParam("sourceDeviceId") long sourceDeviceId,
+                           @PathParam("destination")    String destination,
+                           @Valid                       IncomingMessageList messages)
       throws IOException
   {
     try {
-      OutgoingMessageSignal signal = OutgoingMessageSignal.parseFrom(message.getOutgoingMessageSignal())
-                                                          .toBuilder()
-                                                          .setRelay(peer.getName())
-                                                          .build();
-
-      pushSender.sendMessage(message.getDestination(), signal);
-    } catch (InvalidProtocolBufferException ipe) {
-      logger.warn("ProtoBuf", ipe);
-      throw new WebApplicationException(Response.status(400).build());
-    } catch (NoSuchUserException e) {
-      logger.debug("No User", e);
-      throw new WebApplicationException(Response.status(404).build());
+      messages.setRelay(null);
+      messageController.sendMessage(new NonLimitedAccount(source, sourceDeviceId, peer.getName()), destination, messages);
+    } catch (RateLimitExceededException e) {
+      logger.warn("Rate limiting on federated channel", e);
+      throw new IOException(e);
     }
   }
 
@@ -143,9 +156,7 @@ public class FederationController {
       byte[]        token         = Util.getContactToken(account.getNumber());
       ClientContact clientContact = new ClientContact(token, null, account.getSupportsSms());
 
-      if (Util.isEmpty(account.getApnRegistrationId()) &&
-          Util.isEmpty(account.getGcmRegistrationId()))
-      {
+      if (!account.isActive()) {
         clientContact.setInactive(true);
       }
 

src/main/java/org/whispersystems/textsecuregcm/controllers/KeysController.java

@@ -16,16 +16,20 @@
  */
 package org.whispersystems.textsecuregcm.controllers;
 
-import com.yammer.dropwizard.auth.Auth;
-import com.yammer.metrics.annotation.Timed;
+import com.codahale.metrics.annotation.Timed;
+import com.google.common.base.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.entities.PreKey;
 import org.whispersystems.textsecuregcm.entities.PreKeyList;
+import org.whispersystems.textsecuregcm.entities.PreKeyStatus;
+import org.whispersystems.textsecuregcm.entities.UnstructuredPreKeyList;
 import org.whispersystems.textsecuregcm.federation.FederatedClientManager;
 import org.whispersystems.textsecuregcm.federation.NoSuchPeerException;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
 import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
 import org.whispersystems.textsecuregcm.storage.Keys;
 
 import javax.validation.Valid;
@@ -39,21 +43,27 @@ import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.LinkedList;
+import java.util.List;
+
+import io.dropwizard.auth.Auth;
 
 @Path("/v1/keys")
 public class KeysController {
 
-  private final Logger logger = LoggerFactory.getLogger(AccountController.class);
+  private final Logger logger = LoggerFactory.getLogger(KeysController.class);
 
   private final RateLimiters           rateLimiters;
   private final Keys                   keys;
+  private final AccountsManager        accounts;
   private final FederatedClientManager federatedClientManager;
 
-  public KeysController(RateLimiters rateLimiters, Keys keys,
+  public KeysController(RateLimiters rateLimiters, Keys keys, AccountsManager accounts,
                         FederatedClientManager federatedClientManager)
   {
     this.rateLimiters           = rateLimiters;
     this.keys                   = keys;
+    this.accounts               = accounts;
     this.federatedClientManager = federatedClientManager;
   }
 
@@ -61,7 +71,55 @@ public class KeysController {
   @PUT
   @Consumes(MediaType.APPLICATION_JSON)
   public void setKeys(@Auth Account account, @Valid PreKeyList preKeys)  {
-    keys.store(account.getNumber(), preKeys.getLastResortKey(), preKeys.getKeys());
+    Device device      = account.getAuthenticatedDevice().get();
+    String identityKey = preKeys.getLastResortKey().getIdentityKey();
+
+    if (!identityKey.equals(account.getIdentityKey())) {
+      account.setIdentityKey(identityKey);
+      accounts.update(account);
+    }
+
+    keys.store(account.getNumber(), device.getId(), preKeys.getKeys(), preKeys.getLastResortKey());
+  }
+
+  @Timed
+  @GET
+  @Produces(MediaType.APPLICATION_JSON)
+  public PreKeyStatus getStatus(@Auth Account account) {
+    int count = keys.getCount(account.getNumber(), account.getAuthenticatedDevice().get().getId());
+
+    if (count > 0) {
+      count = count - 1;
+    }
+
+    return new PreKeyStatus(count);
+  }
+
+  @Timed
+  @GET
+  @Path("/{number}/{device_id}")
+  @Produces(MediaType.APPLICATION_JSON)
+  public UnstructuredPreKeyList getDeviceKey(@Auth                   Account account,
+                                             @PathParam("number")    String number,
+                                             @PathParam("device_id") String deviceId,
+                                             @QueryParam("relay")    Optional<String> relay)
+      throws RateLimitExceededException
+  {
+    try {
+      if (account.isRateLimited()) {
+        rateLimiters.getPreKeysLimiter().validate(account.getNumber() +  "__" + number + "." + deviceId);
+      }
+
+      Optional<UnstructuredPreKeyList> results;
+
+      if (!relay.isPresent()) results = getLocalKeys(number, deviceId);
+      else                    results = federatedClientManager.getClient(relay.get()).getKeys(number, deviceId);
+
+      if (results.isPresent()) return results.get();
+      else                     throw new WebApplicationException(Response.status(404).build());
+    } catch (NoSuchPeerException e) {
+      throw new WebApplicationException(Response.status(404).build());
+    }
   }
 
   @Timed
@@ -70,23 +128,58 @@ public class KeysController {
   @Produces(MediaType.APPLICATION_JSON)
   public PreKey get(@Auth                Account account,
                     @PathParam("number") String number,
-                    @QueryParam("relay") String relay)
+                    @QueryParam("relay") Optional<String> relay)
       throws RateLimitExceededException
   {
-    rateLimiters.getPreKeysLimiter().validate(account.getNumber() + "__" + number);
+    UnstructuredPreKeyList results = getDeviceKey(account, number, String.valueOf(Device.MASTER_ID), relay);
+    return results.getKeys().get(0);
+  }
+
+  private Optional<UnstructuredPreKeyList> getLocalKeys(String number, String deviceIdSelector) {
+    Optional<Account> destination = accounts.get(number);
+
+    if (!destination.isPresent() || !destination.get().isActive()) {
+      return Optional.absent();
+    }
 
     try {
-      PreKey key;
+      if (deviceIdSelector.equals("*")) {
+        Optional<UnstructuredPreKeyList> preKeys = keys.get(number);
+        return getActiveKeys(destination.get(), preKeys);
+      }
 
-      if (relay == null) key = keys.get(number);
-      else               key = federatedClientManager.getClient(relay).getKey(number);
+      long             deviceId     = Long.parseLong(deviceIdSelector);
+      Optional<Device> targetDevice = destination.get().getDevice(deviceId);
 
-      if (key == null) throw new WebApplicationException(Response.status(404).build());
-      else             return key;
-    } catch (NoSuchPeerException e) {
-      logger.info("No peer: " + relay);
-      throw new WebApplicationException(Response.status(404).build());
+      if (!targetDevice.isPresent() || !targetDevice.get().isActive()) {
+        return Optional.absent();
+      }
+
+      Optional<UnstructuredPreKeyList> preKeys = keys.get(number, deviceId);
+      return getActiveKeys(destination.get(), preKeys);
+    } catch (NumberFormatException e) {
+      throw new WebApplicationException(Response.status(422).build());
     }
   }
 
+  private Optional<UnstructuredPreKeyList> getActiveKeys(Account destination,
+                                                         Optional<UnstructuredPreKeyList> preKeys)
+  {
+    if (!preKeys.isPresent()) return Optional.absent();
+
+    List<PreKey> filteredKeys = new LinkedList<>();
+
+    for (PreKey preKey : preKeys.get().getKeys()) {
+      Optional<Device> device = destination.getDevice(preKey.getDeviceId());
+
+      if (device.isPresent() && device.get().isActive()) {
+        preKey.setRegistrationId(device.get().getRegistrationId());
+        preKey.setIdentityKey(destination.getIdentityKey());
+        filteredKeys.add(preKey);
+      }
+    }
+
+    if (filteredKeys.isEmpty()) return Optional.absent();
+    else                        return Optional.of(new UnstructuredPreKeyList(filteredKeys));
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/controllers/MessageController.java

@@ -16,298 +16,273 @@
  */
 package org.whispersystems.textsecuregcm.controllers;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import com.codahale.metrics.annotation.Timed;
 import com.google.common.base.Optional;
 import com.google.protobuf.ByteString;
-import com.yammer.dropwizard.auth.AuthenticationException;
-import com.yammer.dropwizard.auth.basic.BasicCredentials;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
-import com.yammer.metrics.core.Timer;
-import com.yammer.metrics.core.TimerContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
-import org.whispersystems.textsecuregcm.auth.AuthorizationHeader;
-import org.whispersystems.textsecuregcm.auth.InvalidAuthorizationHeaderException;
 import org.whispersystems.textsecuregcm.entities.IncomingMessage;
 import org.whispersystems.textsecuregcm.entities.IncomingMessageList;
 import org.whispersystems.textsecuregcm.entities.MessageProtos.OutgoingMessageSignal;
 import org.whispersystems.textsecuregcm.entities.MessageResponse;
+import org.whispersystems.textsecuregcm.entities.MismatchedDevices;
+import org.whispersystems.textsecuregcm.entities.StaleDevices;
 import org.whispersystems.textsecuregcm.federation.FederatedClient;
 import org.whispersystems.textsecuregcm.federation.FederatedClientManager;
 import org.whispersystems.textsecuregcm.federation.NoSuchPeerException;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.push.NotPushRegisteredException;
 import org.whispersystems.textsecuregcm.push.PushSender;
+import org.whispersystems.textsecuregcm.push.TransientPushFailureException;
 import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
 import org.whispersystems.textsecuregcm.util.Base64;
-import org.whispersystems.textsecuregcm.util.IterablePair;
-import org.whispersystems.textsecuregcm.util.IterablePair.Pair;
-import org.whispersystems.textsecuregcm.util.Util;
 
-import javax.servlet.AsyncContext;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedReader;
+import javax.validation.Valid;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
+import java.util.Set;
 
-public class MessageController extends HttpServlet {
+import io.dropwizard.auth.Auth;
 
-  public static final String PATH = "/v1/messages/";
+@Path("/v1/messages")
+public class MessageController {
 
-  private final Meter  successMeter = Metrics.newMeter(MessageController.class, "deliver_message", "success", TimeUnit.MINUTES);
-  private final Meter  failureMeter = Metrics.newMeter(MessageController.class, "deliver_message", "failure", TimeUnit.MINUTES);
-  private final Timer  timer        = Metrics.newTimer(MessageController.class, "deliver_message_time", TimeUnit.MILLISECONDS, TimeUnit.MINUTES);
-  private final Logger logger       = LoggerFactory.getLogger(MessageController.class);
+  private final Logger logger = LoggerFactory.getLogger(MessageController.class);
 
   private final RateLimiters           rateLimiters;
-  private final AccountAuthenticator   accountAuthenticator;
   private final PushSender             pushSender;
   private final FederatedClientManager federatedClientManager;
-  private final ObjectMapper           objectMapper;
-  private final ExecutorService        executor;
+  private final AccountsManager        accountsManager;
 
   public MessageController(RateLimiters rateLimiters,
-                           AccountAuthenticator accountAuthenticator,
                            PushSender pushSender,
+                           AccountsManager accountsManager,
                            FederatedClientManager federatedClientManager)
   {
     this.rateLimiters           = rateLimiters;
-    this.accountAuthenticator   = accountAuthenticator;
     this.pushSender             = pushSender;
+    this.accountsManager        = accountsManager;
     this.federatedClientManager = federatedClientManager;
-    this.objectMapper           = new ObjectMapper();
-    this.executor               = Executors.newFixedThreadPool(10);
   }
 
-  @Override
-  protected void doPost(HttpServletRequest req, HttpServletResponse resp) {
-    TimerContext timerContext = timer.time();
+  @Timed
+  @Path("/{destination}")
+  @PUT
+  @Consumes(MediaType.APPLICATION_JSON)
+  public void sendMessage(@Auth                     Account source,
+                          @PathParam("destination") String destinationName,
+                          @Valid                    IncomingMessageList messages)
+      throws IOException, RateLimitExceededException
+  {
+    rateLimiters.getMessagesLimiter().validate(source.getNumber());
 
     try {
-      Account             sender   = authenticate(req);
-      IncomingMessageList messages = parseIncomingMessages(req);
-
-      rateLimiters.getMessagesLimiter().validate(sender.getNumber());
-
-      List<IncomingMessage>       incomingMessages = messages.getMessages();
-      List<OutgoingMessageSignal> outgoingMessages = getOutgoingMessageSignals(sender.getNumber(),
-                                                                               incomingMessages);
-
-      IterablePair<IncomingMessage, OutgoingMessageSignal> listPair = new IterablePair<>(incomingMessages,
-                                                                                         outgoingMessages);
-
-      handleAsyncDelivery(timerContext, req.startAsync(), listPair);
-    } catch (AuthenticationException e) {
-      failureMeter.mark();
-      timerContext.stop();
-      resp.setStatus(401);
-    } catch (ValidationException e) {
-      failureMeter.mark();
-      timerContext.stop();
-      resp.setStatus(415);
-    } catch (IOException e) {
-      logger.warn("IOE", e);
-      failureMeter.mark();
-      timerContext.stop();
-      resp.setStatus(501);
-    } catch (RateLimitExceededException e) {
-      timerContext.stop();
-      failureMeter.mark();
-      resp.setStatus(413);
+      if (messages.getRelay() == null) sendLocalMessage(source, destinationName, messages);
+      else                             sendRelayMessage(source, destinationName, messages);
+    } catch (NoSuchUserException e) {
+      throw new WebApplicationException(Response.status(404).build());
+    } catch (MismatchedDevicesException e) {
+      throw new WebApplicationException(Response.status(409)
+                                                .type(MediaType.APPLICATION_JSON_TYPE)
+                                                .entity(new MismatchedDevices(e.getMissingDevices(),
+                                                                              e.getExtraDevices()))
+                                                .build());
+    } catch (StaleDevicesException e) {
+      throw new WebApplicationException(Response.status(410)
+                                                .type(MediaType.APPLICATION_JSON)
+                                                .entity(new StaleDevices(e.getStaleDevices()))
+                                                .build());
     }
   }
 
-  private void handleAsyncDelivery(final TimerContext timerContext,
-                                   final AsyncContext context,
-                                   final IterablePair<IncomingMessage, OutgoingMessageSignal> listPair)
+  @Timed
+  @POST
+  @Consumes(MediaType.APPLICATION_JSON)
+  @Produces(MediaType.APPLICATION_JSON)
+  public MessageResponse sendMessageLegacy(@Auth Account source, @Valid IncomingMessageList messages)
+      throws IOException, RateLimitExceededException
   {
-    executor.submit(new Runnable() {
-      @Override
-      public void run() {
-        List<String>        success  = new LinkedList<>();
-        List<String>        failure  = new LinkedList<>();
-        HttpServletResponse response = (HttpServletResponse) context.getResponse();
+    try {
+      List<IncomingMessage> incomingMessages = messages.getMessages();
+      validateLegacyDestinations(incomingMessages);
 
-        try {
-          for (Pair<IncomingMessage, OutgoingMessageSignal> messagePair : listPair) {
-            String destination = messagePair.first().getDestination();
-            String relay       = messagePair.first().getRelay();
+      messages.setRelay(incomingMessages.get(0).getRelay());
+      sendMessage(source, incomingMessages.get(0).getDestination(), messages);
 
-            try {
-              if (Util.isEmpty(relay)) sendLocalMessage(destination, messagePair.second());
-              else                     sendRelayMessage(relay, destination, messagePair.second());
-              success.add(destination);
-            } catch (NoSuchUserException e) {
-              logger.debug("No such user", e);
-              failure.add(destination);
-            }
-          }
+      return new MessageResponse(new LinkedList<String>(), new LinkedList<String>());
+    } catch (ValidationException e) {
+      throw new WebApplicationException(Response.status(422).build());
+    }
+  }
 
-          byte[] responseData = serializeResponse(new MessageResponse(success, failure));
-          response.setContentLength(responseData.length);
-          response.getOutputStream().write(responseData);
-          context.complete();
-          successMeter.mark();
-        } catch (IOException e) {
-          logger.warn("Async Handler", e);
-          failureMeter.mark();
-          response.setStatus(501);
-          context.complete();
-        }
+  private void sendLocalMessage(Account source,
+                                String destinationName,
+                                IncomingMessageList messages)
+      throws NoSuchUserException, MismatchedDevicesException, IOException, StaleDevicesException
+  {
+    Account destination = getDestinationAccount(destinationName);
 
-        timerContext.stop();
+    validateCompleteDeviceList(destination, messages.getMessages());
+    validateRegistrationIds(destination, messages.getMessages());
+
+    for (IncomingMessage incomingMessage : messages.getMessages()) {
+      Optional<Device> destinationDevice = destination.getDevice(incomingMessage.getDestinationDeviceId());
+
+      if (destinationDevice.isPresent()) {
+        sendLocalMessage(source, destination, destinationDevice.get(), incomingMessage);
       }
-    });
+    }
   }
 
-  private void sendLocalMessage(String destination, OutgoingMessageSignal outgoingMessage)
-      throws IOException, NoSuchUserException
+  private void sendLocalMessage(Account source,
+                                Account destinationAccount,
+                                Device destinationDevice,
+                                IncomingMessage incomingMessage)
+      throws NoSuchUserException, IOException
   {
-    pushSender.sendMessage(destination, outgoingMessage);
+    try {
+      Optional<byte[]>              messageBody    = getMessageBody(incomingMessage);
+      OutgoingMessageSignal.Builder messageBuilder = OutgoingMessageSignal.newBuilder();
+
+      messageBuilder.setType(incomingMessage.getType())
+                    .setSource(source.getNumber())
+                    .setTimestamp(System.currentTimeMillis())
+                    .setSourceDevice((int)source.getAuthenticatedDevice().get().getId());
+
+      if (messageBody.isPresent()) {
+        messageBuilder.setMessage(ByteString.copyFrom(messageBody.get()));
+      }
+
+      if (source.getRelay().isPresent()) {
+        messageBuilder.setRelay(source.getRelay().get());
+      }
+
+      pushSender.sendMessage(destinationAccount, destinationDevice, messageBuilder.build());
+    } catch (NotPushRegisteredException e) {
+      if (destinationDevice.isMaster()) throw new NoSuchUserException(e);
+      else                              logger.debug("Not registered", e);
+    } catch (TransientPushFailureException e) {
+      if (destinationDevice.isMaster()) throw new IOException(e);
+      else                              logger.debug("Transient failure", e);
+    }
   }
 
-  private void sendRelayMessage(String relay, String destination, OutgoingMessageSignal outgoingMessage)
+  private void sendRelayMessage(Account source,
+                                String destinationName,
+                                IncomingMessageList messages)
       throws IOException, NoSuchUserException
   {
     try {
-      FederatedClient client = federatedClientManager.getClient(relay);
-      client.sendMessage(destination, outgoingMessage);
+      FederatedClient client = federatedClientManager.getClient(messages.getRelay());
+      client.sendMessages(source.getNumber(), source.getAuthenticatedDevice().get().getId(),
+                          destinationName, messages);
     } catch (NoSuchPeerException e) {
-      logger.info("No such peer", e);
       throw new NoSuchUserException(e);
     }
   }
 
-  private List<OutgoingMessageSignal> getOutgoingMessageSignals(String number,
-                                                                List<IncomingMessage> incomingMessages)
+  private Account getDestinationAccount(String destination)
+      throws NoSuchUserException
   {
-    List<OutgoingMessageSignal> outgoingMessages = new LinkedList<>();
+    Optional<Account> account = accountsManager.get(destination);
 
-    for (IncomingMessage incoming : incomingMessages) {
-      OutgoingMessageSignal.Builder outgoingMessage = OutgoingMessageSignal.newBuilder();
-      outgoingMessage.setType(incoming.getType());
-      outgoingMessage.setSource(number);
+    if (!account.isPresent() || !account.get().isActive()) {
+      throw new NoSuchUserException(destination);
+    }
 
-      byte[] messageBody = getMessageBody(incoming);
+    return account.get();
+  }
 
-      if (messageBody != null) {
-        outgoingMessage.setMessage(ByteString.copyFrom(messageBody));
+  private void validateRegistrationIds(Account account, List<IncomingMessage> messages)
+      throws StaleDevicesException
+  {
+    List<Long> staleDevices = new LinkedList<>();
+
+    for (IncomingMessage message : messages) {
+      Optional<Device> device = account.getDevice(message.getDestinationDeviceId());
+
+      if (device.isPresent() &&
+          message.getDestinationRegistrationId() > 0 &&
+          message.getDestinationRegistrationId() != device.get().getRegistrationId())
+      {
+        staleDevices.add(device.get().getId());
       }
+    }
 
-      outgoingMessage.setTimestamp(System.currentTimeMillis());
+    if (!staleDevices.isEmpty()) {
+      throw new StaleDevicesException(staleDevices);
+    }
+  }
 
-      int index = 0;
+  private void validateCompleteDeviceList(Account account, List<IncomingMessage> messages)
+      throws MismatchedDevicesException
+  {
+    Set<Long> messageDeviceIds = new HashSet<>();
+    Set<Long> accountDeviceIds = new HashSet<>();
 
-      for (IncomingMessage sub : incomingMessages) {
-        if (sub != incoming) {
-          outgoingMessage.setDestinations(index++, sub.getDestination());
+    List<Long> missingDeviceIds = new LinkedList<>();
+    List<Long> extraDeviceIds   = new LinkedList<>();
+
+    for (IncomingMessage message : messages) {
+      messageDeviceIds.add(message.getDestinationDeviceId());
+    }
+
+    for (Device device : account.getDevices()) {
+      if (device.isActive()) {
+        accountDeviceIds.add(device.getId());
+
+        if (!messageDeviceIds.contains(device.getId())) {
+          missingDeviceIds.add(device.getId());
         }
       }
-
-      outgoingMessages.add(outgoingMessage.build());
     }
 
-    return outgoingMessages;
-  }
+    for (IncomingMessage message : messages) {
+      if (!accountDeviceIds.contains(message.getDestinationDeviceId())) {
+        extraDeviceIds.add(message.getDestinationDeviceId());
+      }
+    }
 
-  private byte[] getMessageBody(IncomingMessage message) {
-    try {
-      return Base64.decode(message.getBody());
-    } catch (IOException ioe) {
-      ioe.printStackTrace();
-      return null;
+    if (!missingDeviceIds.isEmpty() || !extraDeviceIds.isEmpty()) {
+      throw new MismatchedDevicesException(missingDeviceIds, extraDeviceIds);
     }
   }
 
-  private byte[] serializeResponse(MessageResponse response) throws IOException {
-    try {
-      return objectMapper.writeValueAsBytes(response);
-    } catch (JsonProcessingException e) {
-      throw new IOException(e);
-    }
-  }
-
-  private IncomingMessageList parseIncomingMessages(HttpServletRequest request)
-      throws IOException, ValidationException
+  private void validateLegacyDestinations(List<IncomingMessage> messages)
+      throws ValidationException
   {
-    BufferedReader reader  = request.getReader();
-    StringBuilder  content = new StringBuilder();
-    String         line;
+    String destination = null;
 
-    while ((line = reader.readLine()) != null) {
-      content.append(line);
+    for (IncomingMessage message : messages) {
+      if ((message.getDestination() == null) ||
+          (destination != null && !destination.equals(message.getDestination())))
+      {
+        throw new ValidationException("Multiple account destinations!");
+      }
+
+      destination = message.getDestination();
     }
-
-    IncomingMessageList messages = objectMapper.readValue(content.toString(),
-                                                          IncomingMessageList.class);
-
-    if (messages.getMessages() == null) {
-      throw new ValidationException();
-    }
-
-    for (IncomingMessage message : messages.getMessages()) {
-      if (message.getBody() == null)        throw new ValidationException();
-      if (message.getDestination() == null) throw new ValidationException();
-    }
-
-    return messages;
   }
 
-  private Account authenticate(HttpServletRequest request) throws AuthenticationException {
+  private Optional<byte[]> getMessageBody(IncomingMessage message) {
     try {
-      AuthorizationHeader authorizationHeader = new AuthorizationHeader(request.getHeader("Authorization"));
-      BasicCredentials    credentials         = new BasicCredentials(authorizationHeader.getUserName(),
-                                                                     authorizationHeader.getPassword()  );
-
-      Optional<Account> account = accountAuthenticator.authenticate(credentials);
-
-      if (account.isPresent()) return account.get();
-      else                     throw new AuthenticationException("Bad credentials");
-    } catch (InvalidAuthorizationHeaderException e) {
-      throw new AuthenticationException(e);
+      return Optional.of(Base64.decode(message.getBody()));
+    } catch (IOException ioe) {
+      logger.debug("Bad B64", ioe);
+      return Optional.absent();
     }
   }
-
-
-
-
-//  @Timed
-//  @POST
-//  @Consumes(MediaType.APPLICATION_JSON)
-//  @Produces(MediaType.APPLICATION_JSON)
-//  public MessageResponse sendMessage(@Auth Account sender, IncomingMessageList messages)
-//      throws IOException
-//  {
-//    List<String>                success          = new LinkedList<>();
-//    List<String>                failure          = new LinkedList<>();
-//    List<IncomingMessage>       incomingMessages = messages.getMessages();
-//    List<OutgoingMessageSignal> outgoingMessages = getOutgoingMessageSignals(sender.getNumber(), incomingMessages);
-//
-//    IterablePair<IncomingMessage, OutgoingMessageSignal> listPair = new IterablePair<>(incomingMessages, outgoingMessages);
-//
-//    for (Pair<IncomingMessage, OutgoingMessageSignal> messagePair : listPair) {
-//      String destination = messagePair.first().getDestination();
-//      String relay       = messagePair.first().getRelay();
-//
-//      try {
-//        if (Util.isEmpty(relay)) sendLocalMessage(destination, messagePair.second());
-//        else                     sendRelayMessage(relay, destination, messagePair.second());
-//        success.add(destination);
-//      } catch (NoSuchUserException e) {
-//        logger.debug("No such user", e);
-//        failure.add(destination);
-//      }
-//    }
-//
-//    return new MessageResponse(success, failure);
-//  }
-
 }

src/main/java/org/whispersystems/textsecuregcm/controllers/MismatchedDevicesException.java

@@ -0,0 +1,22 @@
+package org.whispersystems.textsecuregcm.controllers;
+
+import java.util.List;
+
+public class MismatchedDevicesException extends Exception {
+
+  private final List<Long> missingDevices;
+  private final List<Long> extraDevices;
+
+  public MismatchedDevicesException(List<Long> missingDevices, List<Long> extraDevices) {
+    this.missingDevices = missingDevices;
+    this.extraDevices   = extraDevices;
+  }
+
+  public List<Long> getMissingDevices() {
+    return missingDevices;
+  }
+
+  public List<Long> getExtraDevices() {
+    return extraDevices;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/controllers/NoSuchUserException.java

@@ -16,8 +16,6 @@
  */
 package org.whispersystems.textsecuregcm.controllers;
 
-import org.whispersystems.textsecuregcm.federation.NoSuchPeerException;
-
 import java.util.LinkedList;
 import java.util.List;
 
@@ -27,7 +25,7 @@ public class NoSuchUserException extends Exception {
 
   public NoSuchUserException(String user) {
     super(user);
-    missing = new LinkedList<String>();
+    missing = new LinkedList<>();
     missing.add(user);
   }
 

src/main/java/org/whispersystems/textsecuregcm/controllers/StaleDevicesException.java

@@ -0,0 +1,16 @@
+package org.whispersystems.textsecuregcm.controllers;
+
+import java.util.List;
+
+
+public class StaleDevicesException extends Throwable {
+  private final List<Long> staleDevices;
+
+  public StaleDevicesException(List<Long> staleDevices) {
+    this.staleDevices = staleDevices;
+  }
+
+  public List<Long> getStaleDevices() {
+    return staleDevices;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/controllers/ValidationException.java

@@ -18,4 +18,7 @@ package org.whispersystems.textsecuregcm.controllers;
 
 
 public class ValidationException extends Exception {
+  public ValidationException(String s) {
+    super(s);
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/controllers/WebsocketController.java

@@ -0,0 +1,223 @@
+package org.whispersystems.textsecuregcm.controllers;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Optional;
+import org.eclipse.jetty.websocket.api.CloseStatus;
+import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.UpgradeRequest;
+import org.eclipse.jetty.websocket.api.WebSocketListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
+import org.whispersystems.textsecuregcm.entities.AcknowledgeWebsocketMessage;
+import org.whispersystems.textsecuregcm.entities.EncryptedOutgoingMessage;
+import org.whispersystems.textsecuregcm.entities.IncomingWebsocketMessage;
+import org.whispersystems.textsecuregcm.push.NotPushRegisteredException;
+import org.whispersystems.textsecuregcm.push.PushSender;
+import org.whispersystems.textsecuregcm.push.TransientPushFailureException;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.storage.PubSubListener;
+import org.whispersystems.textsecuregcm.storage.PubSubManager;
+import org.whispersystems.textsecuregcm.storage.PubSubMessage;
+import org.whispersystems.textsecuregcm.storage.StoredMessages;
+import org.whispersystems.textsecuregcm.websocket.WebsocketAddress;
+import org.whispersystems.textsecuregcm.websocket.WebsocketMessage;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import io.dropwizard.auth.AuthenticationException;
+import io.dropwizard.auth.basic.BasicCredentials;
+
+public class WebsocketController implements WebSocketListener, PubSubListener {
+
+  private static final Logger            logger          = LoggerFactory.getLogger(WebsocketController.class);
+  private static final ObjectMapper      mapper          = new ObjectMapper();
+  private static final Map<Long, String> pendingMessages = new HashMap<>();
+
+  private final AccountAuthenticator accountAuthenticator;
+  private final PubSubManager        pubSubManager;
+  private final StoredMessages       storedMessages;
+  private final PushSender           pushSender;
+
+  private WebsocketAddress address;
+  private Account          account;
+  private Device           device;
+  private Session          session;
+
+  private long pendingMessageSequence;
+
+  public WebsocketController(AccountAuthenticator accountAuthenticator,
+                             PushSender           pushSender,
+                             PubSubManager        pubSubManager,
+                             StoredMessages       storedMessages)
+  {
+    this.accountAuthenticator = accountAuthenticator;
+    this.pushSender           = pushSender;
+    this.pubSubManager        = pubSubManager;
+    this.storedMessages       = storedMessages;
+  }
+
+  @Override
+  public void onWebSocketConnect(Session session) {
+    try {
+      UpgradeRequest        request    = session.getUpgradeRequest();
+      Map<String, String[]> parameters = request.getParameterMap();
+      String[]              usernames  = parameters.get("login"   );
+      String[]              passwords  = parameters.get("password");
+
+      if (usernames == null || usernames.length == 0 ||
+          passwords == null || passwords.length == 0)
+      {
+        session.close(new CloseStatus(4001, "Unauthorized"));
+        return;
+      }
+
+      BasicCredentials  credentials = new BasicCredentials(usernames[0], passwords[0]);
+      Optional<Account> account     = accountAuthenticator.authenticate(credentials);
+
+      if (!account.isPresent()) {
+        session.close(new CloseStatus(4001, "Unauthorized"));
+        return;
+      }
+
+      this.account = account.get();
+      this.device  = account.get().getAuthenticatedDevice().get();
+      this.address = new WebsocketAddress(this.account.getId(), this.device.getId());
+      this.session = session;
+
+      this.session.setIdleTimeout(10 * 60 * 1000);
+      this.pubSubManager.subscribe(this.address, this);
+
+      handleQueryDatabase();
+    } catch (AuthenticationException e) {
+      try { session.close(1011, "Server Error");} catch (IOException e1) {}
+    } catch (IOException ioe) {
+      logger.info("Abrupt session close.");
+    }
+  }
+
+  @Override
+  public void onWebSocketText(String body) {
+    try {
+      IncomingWebsocketMessage incomingMessage = mapper.readValue(body, IncomingWebsocketMessage.class);
+
+      switch (incomingMessage.getType()) {
+        case IncomingWebsocketMessage.TYPE_ACKNOWLEDGE_MESSAGE:
+          handleMessageAck(body);
+          break;
+        default:
+          close(new CloseStatus(1008, "Unknown Type"));
+      }
+    } catch (IOException e) {
+      logger.debug("Parse", e);
+      close(new CloseStatus(1008, "Badly Formatted"));
+    }
+  }
+
+  @Override
+  public void onWebSocketClose(int i, String s) {
+    pubSubManager.unsubscribe(this.address, this);
+
+    List<String> remainingMessages = new LinkedList<>();
+
+    synchronized (pendingMessages) {
+      Long[] pendingKeys = pendingMessages.keySet().toArray(new Long[0]);
+      Arrays.sort(pendingKeys);
+
+      for (long pendingKey : pendingKeys) {
+        remainingMessages.add(pendingMessages.get(pendingKey));
+      }
+
+      pendingMessages.clear();
+    }
+
+    for (String remainingMessage : remainingMessages) {
+      try {
+        pushSender.sendMessage(account, device, new EncryptedOutgoingMessage(remainingMessage));
+      } catch (NotPushRegisteredException | TransientPushFailureException e) {
+        logger.warn("onWebSocketClose", e);
+        storedMessages.insert(account.getId(), device.getId(), remainingMessage);
+      }
+    }
+  }
+
+
+  @Override
+  public void onPubSubMessage(PubSubMessage outgoingMessage) {
+    switch (outgoingMessage.getType()) {
+      case PubSubMessage.TYPE_DELIVER:
+        handleDeliverOutgoingMessage(outgoingMessage.getContents());
+        break;
+      case PubSubMessage.TYPE_QUERY_DB:
+        handleQueryDatabase();
+        break;
+      default:
+        logger.warn("Unknown pubsub message: " + outgoingMessage.getType());
+    }
+  }
+
+  private void handleDeliverOutgoingMessage(String message) {
+    try {
+      long messageSequence;
+
+      synchronized (pendingMessages) {
+        messageSequence = pendingMessageSequence++;
+        pendingMessages.put(messageSequence, message);
+      }
+
+      WebsocketMessage websocketMessage = new WebsocketMessage(messageSequence, message);
+      session.getRemote().sendStringByFuture(mapper.writeValueAsString(websocketMessage));
+    } catch (IOException e) {
+      logger.debug("Response failed", e);
+      close(null);
+    }
+  }
+
+  private void handleMessageAck(String message) {
+    try {
+      AcknowledgeWebsocketMessage ack = mapper.readValue(message, AcknowledgeWebsocketMessage.class);
+
+      synchronized (pendingMessages) {
+        pendingMessages.remove(ack.getId());
+      }
+    } catch (IOException e) {
+      logger.warn("Mapping", e);
+    }
+  }
+
+  private void handleQueryDatabase() {
+    List<String> messages = storedMessages.getMessagesForDevice(account.getId(), device.getId());
+
+    for (String message : messages) {
+      handleDeliverOutgoingMessage(message);
+    }
+  }
+
+  @Override
+  public void onWebSocketBinary(byte[] bytes, int i, int i2) {
+    logger.info("Received binary message!");
+  }
+
+  @Override
+  public void onWebSocketError(Throwable throwable) {
+    logger.info("onWebSocketError", throwable);
+  }
+
+
+  private void close(CloseStatus closeStatus) {
+    try {
+      if (this.session != null) {
+        if (closeStatus != null) this.session.close(closeStatus);
+        else                     this.session.close();
+      }
+    } catch (IOException e) {
+      logger.info("close()", e);
+    }
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/entities/AccountAttributes.java

@@ -28,11 +28,19 @@ public class AccountAttributes {
   @JsonProperty
   private boolean supportsSms;
 
+  @JsonProperty
+  private boolean fetchesMessages;
+
+  @JsonProperty
+  private int registrationId;
+
   public AccountAttributes() {}
 
-  public AccountAttributes(String signalingKey, boolean supportsSms) {
-    this.signalingKey = signalingKey;
-    this.supportsSms  = supportsSms;
+  public AccountAttributes(String signalingKey, boolean supportsSms, boolean fetchesMessages, int registrationId) {
+    this.signalingKey    = signalingKey;
+    this.supportsSms     = supportsSms;
+    this.fetchesMessages = fetchesMessages;
+    this.registrationId  = registrationId;
   }
 
   public String getSignalingKey() {
@@ -43,4 +51,11 @@ public class AccountAttributes {
     return supportsSms;
   }
 
+  public boolean getFetchesMessages() {
+    return fetchesMessages;
+  }
+
+  public int getRegistrationId() {
+    return registrationId;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/entities/AcknowledgeWebsocketMessage.java

@@ -0,0 +1,23 @@
+package org.whispersystems.textsecuregcm.entities;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AcknowledgeWebsocketMessage extends IncomingWebsocketMessage {
+
+  @JsonProperty
+  private long id;
+
+  public AcknowledgeWebsocketMessage() {}
+
+  public AcknowledgeWebsocketMessage(long id) {
+    this.type = TYPE_ACKNOWLEDGE_MESSAGE;
+    this.id   = id;
+  }
+
+  public long getId() {
+    return id;
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/entities/ClientContactTokens.java

@@ -30,4 +30,11 @@ public class ClientContactTokens {
   public List<String> getContacts() {
     return contacts;
   }
+
+  public ClientContactTokens() {}
+
+  public ClientContactTokens(List<String> contacts) {
+    this.contacts = contacts;
+  }
+
 }

src/main/java/org/whispersystems/textsecuregcm/entities/CryptoEncodingException.java

@@ -0,0 +1,13 @@
+package org.whispersystems.textsecuregcm.entities;
+
+public class CryptoEncodingException extends Exception {
+
+  public CryptoEncodingException(String s) {
+    super(s);
+  }
+
+  public CryptoEncodingException(Exception e) {
+    super(e);
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/entities/DeviceResponse.java

@@ -0,0 +1,21 @@
+package org.whispersystems.textsecuregcm.entities;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+
+public class DeviceResponse {
+
+  @JsonProperty
+  private long deviceId;
+
+  @VisibleForTesting
+  public DeviceResponse() {}
+
+  public DeviceResponse(long deviceId) {
+    this.deviceId = deviceId;
+  }
+
+  public long getDeviceId() {
+    return deviceId;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/entities/EncryptedOutgoingMessage.java

@@ -41,27 +41,30 @@ public class EncryptedOutgoingMessage {
   private static final int    MAC_KEY_SIZE    = 20;
   private static final int    MAC_SIZE        = 10;
 
-  private final OutgoingMessageSignal outgoingMessage;
-  private final String signalingKey;
+  private final String serialized;
 
   public EncryptedOutgoingMessage(OutgoingMessageSignal outgoingMessage,
                                   String signalingKey)
+      throws CryptoEncodingException
   {
-    this.outgoingMessage = outgoingMessage;
-    this.signalingKey    = signalingKey;
-  }
-
-  public String serialize() throws IOException {
     byte[]        plaintext  = outgoingMessage.toByteArray();
     SecretKeySpec cipherKey  = getCipherKey (signalingKey);
     SecretKeySpec macKey     = getMacKey(signalingKey);
     byte[]        ciphertext = getCiphertext(plaintext, cipherKey, macKey);
 
-    return Base64.encodeBytes(ciphertext);
+    this.serialized = Base64.encodeBytes(ciphertext);
+  }
+
+  public EncryptedOutgoingMessage(String serialized) {
+    this.serialized = serialized;
+  }
+
+  public String serialize() {
+    return serialized;
   }
 
   private byte[] getCiphertext(byte[] plaintext, SecretKeySpec cipherKey, SecretKeySpec macKey)
-      throws IOException
+      throws CryptoEncodingException
   {
     try {
       Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
@@ -85,31 +88,39 @@ public class EncryptedOutgoingMessage {
       throw new AssertionError(e);
     } catch (InvalidKeyException e) {
       logger.warn("Invalid Key", e);
-      throw new IOException("Invalid key!");
+      throw new CryptoEncodingException("Invalid key!");
     }
   }
 
-  private SecretKeySpec getCipherKey(String signalingKey) throws IOException {
-    byte[] signalingKeyBytes = Base64.decode(signalingKey);
-    byte[] cipherKey         = new byte[CIPHER_KEY_SIZE];
+  private SecretKeySpec getCipherKey(String signalingKey) throws CryptoEncodingException {
+    try {
+      byte[] signalingKeyBytes = Base64.decode(signalingKey);
+      byte[] cipherKey         = new byte[CIPHER_KEY_SIZE];
 
-    if (signalingKeyBytes.length < CIPHER_KEY_SIZE)
-      throw new IOException("Signaling key too short!");
+      if (signalingKeyBytes.length < CIPHER_KEY_SIZE)
+        throw new CryptoEncodingException("Signaling key too short!");
 
-    System.arraycopy(signalingKeyBytes, 0, cipherKey, 0, cipherKey.length);
-    return new SecretKeySpec(cipherKey, "AES");
+      System.arraycopy(signalingKeyBytes, 0, cipherKey, 0, cipherKey.length);
+      return new SecretKeySpec(cipherKey, "AES");
+    } catch (IOException e) {
+      throw new CryptoEncodingException(e);
+    }
   }
 
-  private SecretKeySpec getMacKey(String signalingKey) throws IOException {
-    byte[] signalingKeyBytes = Base64.decode(signalingKey);
-    byte[] macKey            = new byte[MAC_KEY_SIZE];
+  private SecretKeySpec getMacKey(String signalingKey) throws CryptoEncodingException {
+    try {
+      byte[] signalingKeyBytes = Base64.decode(signalingKey);
+      byte[] macKey            = new byte[MAC_KEY_SIZE];
 
-    if (signalingKeyBytes.length < CIPHER_KEY_SIZE + MAC_KEY_SIZE)
-      throw new IOException(("Signaling key too short!"));
+      if (signalingKeyBytes.length < CIPHER_KEY_SIZE + MAC_KEY_SIZE)
+        throw new CryptoEncodingException("Signaling key too short!");
 
-    System.arraycopy(signalingKeyBytes, CIPHER_KEY_SIZE, macKey, 0, macKey.length);
+      System.arraycopy(signalingKeyBytes, CIPHER_KEY_SIZE, macKey, 0, macKey.length);
 
-    return new SecretKeySpec(macKey, "HmacSHA256");
+      return new SecretKeySpec(macKey, "HmacSHA256");
+    } catch (IOException e) {
+      throw new CryptoEncodingException(e);
+    }
   }
 
 }

src/main/java/org/whispersystems/textsecuregcm/entities/IncomingMessage.java

@@ -25,9 +25,14 @@ public class IncomingMessage {
   private int    type;
 
   @JsonProperty
-  @NotEmpty
   private String destination;
 
+  @JsonProperty
+  private long   destinationDeviceId = 1;
+
+  @JsonProperty
+  private int destinationRegistrationId;
+
   @JsonProperty
   @NotEmpty
   private String body;
@@ -38,6 +43,7 @@ public class IncomingMessage {
   @JsonProperty
   private long   timestamp;
 
+
   public String getDestination() {
     return destination;
   }
@@ -53,4 +59,12 @@ public class IncomingMessage {
   public String getRelay() {
     return relay;
   }
+
+  public long getDestinationDeviceId() {
+    return destinationDeviceId;
+  }
+
+  public int getDestinationRegistrationId() {
+    return destinationRegistrationId;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/entities/IncomingMessageList.java

@@ -29,9 +29,20 @@ public class IncomingMessageList {
   @Valid
   private List<IncomingMessage> messages;
 
+  @JsonProperty
+  private String relay;
+
   public IncomingMessageList() {}
 
   public List<IncomingMessage> getMessages() {
     return messages;
   }
+
+  public String getRelay() {
+    return relay;
+  }
+
+  public void setRelay(String relay) {
+    this.relay = relay;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/entities/IncomingWebsocketMessage.java

@@ -0,0 +1,25 @@
+package org.whispersystems.textsecuregcm.entities;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class IncomingWebsocketMessage {
+
+  public static final int TYPE_ACKNOWLEDGE_MESSAGE = 1;
+  public static final int TYPE_PING_MESSAGE        = 2;
+  public static final int TYPE_PONG_MESSAGE        = 3;
+
+  @JsonProperty
+  protected int type;
+
+  public IncomingWebsocketMessage() {}
+
+  public IncomingWebsocketMessage(int type) {
+    this.type = type;
+  }
+
+  public int getType() {
+    return type;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/entities/MessageProtos.java

@@ -19,15 +19,14 @@ public final class MessageProtos {
     boolean hasSource();
     String getSource();
     
+    // optional uint32 sourceDevice = 7;
+    boolean hasSourceDevice();
+    int getSourceDevice();
+    
     // optional string relay = 3;
     boolean hasRelay();
     String getRelay();
     
-    // repeated string destinations = 4;
-    java.util.List<String> getDestinationsList();
-    int getDestinationsCount();
-    String getDestinations(int index);
-    
     // optional uint64 timestamp = 5;
     boolean hasTimestamp();
     long getTimestamp();
@@ -107,11 +106,21 @@ public final class MessageProtos {
       }
     }
     
+    // optional uint32 sourceDevice = 7;
+    public static final int SOURCEDEVICE_FIELD_NUMBER = 7;
+    private int sourceDevice_;
+    public boolean hasSourceDevice() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public int getSourceDevice() {
+      return sourceDevice_;
+    }
+    
     // optional string relay = 3;
     public static final int RELAY_FIELD_NUMBER = 3;
     private java.lang.Object relay_;
     public boolean hasRelay() {
-      return ((bitField0_ & 0x00000004) == 0x00000004);
+      return ((bitField0_ & 0x00000008) == 0x00000008);
     }
     public String getRelay() {
       java.lang.Object ref = relay_;
@@ -139,25 +148,11 @@ public final class MessageProtos {
       }
     }
     
-    // repeated string destinations = 4;
-    public static final int DESTINATIONS_FIELD_NUMBER = 4;
-    private com.google.protobuf.LazyStringList destinations_;
-    public java.util.List<String>
-        getDestinationsList() {
-      return destinations_;
-    }
-    public int getDestinationsCount() {
-      return destinations_.size();
-    }
-    public String getDestinations(int index) {
-      return destinations_.get(index);
-    }
-    
     // optional uint64 timestamp = 5;
     public static final int TIMESTAMP_FIELD_NUMBER = 5;
     private long timestamp_;
     public boolean hasTimestamp() {
-      return ((bitField0_ & 0x00000008) == 0x00000008);
+      return ((bitField0_ & 0x00000010) == 0x00000010);
     }
     public long getTimestamp() {
       return timestamp_;
@@ -167,7 +162,7 @@ public final class MessageProtos {
     public static final int MESSAGE_FIELD_NUMBER = 6;
     private com.google.protobuf.ByteString message_;
     public boolean hasMessage() {
-      return ((bitField0_ & 0x00000010) == 0x00000010);
+      return ((bitField0_ & 0x00000020) == 0x00000020);
     }
     public com.google.protobuf.ByteString getMessage() {
       return message_;
@@ -176,8 +171,8 @@ public final class MessageProtos {
     private void initFields() {
       type_ = 0;
       source_ = "";
+      sourceDevice_ = 0;
       relay_ = "";
-      destinations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
       timestamp_ = 0L;
       message_ = com.google.protobuf.ByteString.EMPTY;
     }
@@ -199,18 +194,18 @@ public final class MessageProtos {
       if (((bitField0_ & 0x00000002) == 0x00000002)) {
         output.writeBytes(2, getSourceBytes());
       }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
         output.writeBytes(3, getRelayBytes());
       }
-      for (int i = 0; i < destinations_.size(); i++) {
-        output.writeBytes(4, destinations_.getByteString(i));
-      }
-      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
         output.writeUInt64(5, timestamp_);
       }
-      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
         output.writeBytes(6, message_);
       }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeUInt32(7, sourceDevice_);
+      }
       getUnknownFields().writeTo(output);
     }
     
@@ -228,27 +223,22 @@ public final class MessageProtos {
         size += com.google.protobuf.CodedOutputStream
           .computeBytesSize(2, getSourceBytes());
       }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
         size += com.google.protobuf.CodedOutputStream
           .computeBytesSize(3, getRelayBytes());
       }
-      {
-        int dataSize = 0;
-        for (int i = 0; i < destinations_.size(); i++) {
-          dataSize += com.google.protobuf.CodedOutputStream
-            .computeBytesSizeNoTag(destinations_.getByteString(i));
-        }
-        size += dataSize;
-        size += 1 * getDestinationsList().size();
-      }
-      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
         size += com.google.protobuf.CodedOutputStream
           .computeUInt64Size(5, timestamp_);
       }
-      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
         size += com.google.protobuf.CodedOutputStream
           .computeBytesSize(6, message_);
       }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeUInt32Size(7, sourceDevice_);
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -377,9 +367,9 @@ public final class MessageProtos {
         bitField0_ = (bitField0_ & ~0x00000001);
         source_ = "";
         bitField0_ = (bitField0_ & ~0x00000002);
-        relay_ = "";
+        sourceDevice_ = 0;
         bitField0_ = (bitField0_ & ~0x00000004);
-        destinations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+        relay_ = "";
         bitField0_ = (bitField0_ & ~0x00000008);
         timestamp_ = 0L;
         bitField0_ = (bitField0_ & ~0x00000010);
@@ -434,19 +424,17 @@ public final class MessageProtos {
         if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
           to_bitField0_ |= 0x00000004;
         }
-        result.relay_ = relay_;
-        if (((bitField0_ & 0x00000008) == 0x00000008)) {
-          destinations_ = new com.google.protobuf.UnmodifiableLazyStringList(
-              destinations_);
-          bitField0_ = (bitField0_ & ~0x00000008);
-        }
-        result.destinations_ = destinations_;
-        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+        result.sourceDevice_ = sourceDevice_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
           to_bitField0_ |= 0x00000008;
         }
+        result.relay_ = relay_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
         result.timestamp_ = timestamp_;
         if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
-          to_bitField0_ |= 0x00000010;
+          to_bitField0_ |= 0x00000020;
         }
         result.message_ = message_;
         result.bitField0_ = to_bitField0_;
@@ -471,19 +459,12 @@ public final class MessageProtos {
         if (other.hasSource()) {
           setSource(other.getSource());
         }
+        if (other.hasSourceDevice()) {
+          setSourceDevice(other.getSourceDevice());
+        }
         if (other.hasRelay()) {
           setRelay(other.getRelay());
         }
-        if (!other.destinations_.isEmpty()) {
-          if (destinations_.isEmpty()) {
-            destinations_ = other.destinations_;
-            bitField0_ = (bitField0_ & ~0x00000008);
-          } else {
-            ensureDestinationsIsMutable();
-            destinations_.addAll(other.destinations_);
-          }
-          onChanged();
-        }
         if (other.hasTimestamp()) {
           setTimestamp(other.getTimestamp());
         }
@@ -532,15 +513,10 @@ public final class MessageProtos {
               break;
             }
             case 26: {
-              bitField0_ |= 0x00000004;
+              bitField0_ |= 0x00000008;
               relay_ = input.readBytes();
               break;
             }
-            case 34: {
-              ensureDestinationsIsMutable();
-              destinations_.add(input.readBytes());
-              break;
-            }
             case 40: {
               bitField0_ |= 0x00000010;
               timestamp_ = input.readUInt64();
@@ -551,6 +527,11 @@ public final class MessageProtos {
               message_ = input.readBytes();
               break;
             }
+            case 56: {
+              bitField0_ |= 0x00000004;
+              sourceDevice_ = input.readUInt32();
+              break;
+            }
           }
         }
       }
@@ -614,10 +595,31 @@ public final class MessageProtos {
         onChanged();
       }
       
+      // optional uint32 sourceDevice = 7;
+      private int sourceDevice_ ;
+      public boolean hasSourceDevice() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public int getSourceDevice() {
+        return sourceDevice_;
+      }
+      public Builder setSourceDevice(int value) {
+        bitField0_ |= 0x00000004;
+        sourceDevice_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSourceDevice() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        sourceDevice_ = 0;
+        onChanged();
+        return this;
+      }
+      
       // optional string relay = 3;
       private java.lang.Object relay_ = "";
       public boolean hasRelay() {
-        return ((bitField0_ & 0x00000004) == 0x00000004);
+        return ((bitField0_ & 0x00000008) == 0x00000008);
       }
       public String getRelay() {
         java.lang.Object ref = relay_;
@@ -633,79 +635,23 @@ public final class MessageProtos {
         if (value == null) {
     throw new NullPointerException();
   }
-  bitField0_ |= 0x00000004;
+  bitField0_ |= 0x00000008;
         relay_ = value;
         onChanged();
         return this;
       }
       public Builder clearRelay() {
-        bitField0_ = (bitField0_ & ~0x00000004);
+        bitField0_ = (bitField0_ & ~0x00000008);
         relay_ = getDefaultInstance().getRelay();
         onChanged();
         return this;
       }
       void setRelay(com.google.protobuf.ByteString value) {
-        bitField0_ |= 0x00000004;
+        bitField0_ |= 0x00000008;
         relay_ = value;
         onChanged();
       }
       
-      // repeated string destinations = 4;
-      private com.google.protobuf.LazyStringList destinations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
-      private void ensureDestinationsIsMutable() {
-        if (!((bitField0_ & 0x00000008) == 0x00000008)) {
-          destinations_ = new com.google.protobuf.LazyStringArrayList(destinations_);
-          bitField0_ |= 0x00000008;
-         }
-      }
-      public java.util.List<String>
-          getDestinationsList() {
-        return java.util.Collections.unmodifiableList(destinations_);
-      }
-      public int getDestinationsCount() {
-        return destinations_.size();
-      }
-      public String getDestinations(int index) {
-        return destinations_.get(index);
-      }
-      public Builder setDestinations(
-          int index, String value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  ensureDestinationsIsMutable();
-        destinations_.set(index, value);
-        onChanged();
-        return this;
-      }
-      public Builder addDestinations(String value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  ensureDestinationsIsMutable();
-        destinations_.add(value);
-        onChanged();
-        return this;
-      }
-      public Builder addAllDestinations(
-          java.lang.Iterable<String> values) {
-        ensureDestinationsIsMutable();
-        super.addAll(values, destinations_);
-        onChanged();
-        return this;
-      }
-      public Builder clearDestinations() {
-        destinations_ = com.google.protobuf.LazyStringArrayList.EMPTY;
-        bitField0_ = (bitField0_ & ~0x00000008);
-        onChanged();
-        return this;
-      }
-      void addDestinations(com.google.protobuf.ByteString value) {
-        ensureDestinationsIsMutable();
-        destinations_.add(value);
-        onChanged();
-      }
-      
       // optional uint64 timestamp = 5;
       private long timestamp_ ;
       public boolean hasTimestamp() {
@@ -778,8 +724,8 @@ public final class MessageProtos {
     java.lang.String[] descriptorData = {
       "\n\033OutgoingMessageSignal.proto\022\ntextsecur" +
       "e\"~\n\025OutgoingMessageSignal\022\014\n\004type\030\001 \001(\r" +
-      "\022\016\n\006source\030\002 \001(\t\022\r\n\005relay\030\003 \001(\t\022\024\n\014desti" +
-      "nations\030\004 \003(\t\022\021\n\ttimestamp\030\005 \001(\004\022\017\n\007mess" +
+      "\022\016\n\006source\030\002 \001(\t\022\024\n\014sourceDevice\030\007 \001(\r\022\r" +
+      "\n\005relay\030\003 \001(\t\022\021\n\ttimestamp\030\005 \001(\004\022\017\n\007mess" +
       "age\030\006 \001(\014B:\n)org.whispersystems.textsecu" +
       "regcm.entitiesB\rMessageProtos"
     };
@@ -793,7 +739,7 @@ public final class MessageProtos {
           internal_static_textsecure_OutgoingMessageSignal_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_textsecure_OutgoingMessageSignal_descriptor,
-              new java.lang.String[] { "Type", "Source", "Relay", "Destinations", "Timestamp", "Message", },
+              new java.lang.String[] { "Type", "Source", "SourceDevice", "Relay", "Timestamp", "Message", },
               org.whispersystems.textsecuregcm.entities.MessageProtos.OutgoingMessageSignal.class,
               org.whispersystems.textsecuregcm.entities.MessageProtos.OutgoingMessageSignal.Builder.class);
           return null;

src/main/java/org/whispersystems/textsecuregcm/entities/MessageResponse.java

@@ -16,15 +16,26 @@
  */
 package org.whispersystems.textsecuregcm.entities;
 
+import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
 public class MessageResponse {
   private List<String> success;
   private List<String> failure;
+  private Set<String>  missingDeviceIds;
 
   public MessageResponse(List<String> success, List<String> failure) {
-    this.success = success;
-    this.failure = failure;
+    this.success          = success;
+    this.failure          = failure;
+    this.missingDeviceIds = new HashSet<>();
+  }
+
+  public MessageResponse(Set<String> missingDeviceIds) {
+    this.success          = new LinkedList<>();
+    this.failure          = new LinkedList<>(missingDeviceIds);
+    this.missingDeviceIds = missingDeviceIds;
   }
 
   public MessageResponse() {}
@@ -33,8 +44,23 @@ public class MessageResponse {
     return success;
   }
 
+  public void setSuccess(List<String> success) {
+    this.success = success;
+  }
+
   public List<String> getFailure() {
     return failure;
   }
 
+  public void setFailure(List<String> failure) {
+    this.failure = failure;
+  }
+
+  public Set<String> getNumbersMissingDevices() {
+    return missingDeviceIds;
+  }
+
+  public void setNumbersMissingDevices(Set<String> numbersMissingDevices) {
+    this.missingDeviceIds = numbersMissingDevices;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/entities/MismatchedDevices.java

@@ -0,0 +1,24 @@
+package org.whispersystems.textsecuregcm.entities;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.util.List;
+
+public class MismatchedDevices {
+
+  @JsonProperty
+  public List<Long> missingDevices;
+
+  @JsonProperty
+  public List<Long> extraDevices;
+
+  @VisibleForTesting
+  public MismatchedDevices() {}
+
+  public MismatchedDevices(List<Long> missingDevices, List<Long> extraDevices) {
+    this.missingDevices = missingDevices;
+    this.extraDevices   = extraDevices;
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/entities/PreKey.java

@@ -20,6 +20,7 @@ package org.whispersystems.textsecuregcm.entities;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
 
 import javax.validation.constraints.NotNull;
 import javax.xml.bind.annotation.XmlTransient;
@@ -34,6 +35,9 @@ public class PreKey {
   @JsonIgnore
   private String  number;
 
+  @JsonProperty
+  private long deviceId;
+
   @JsonProperty
   @NotNull
   private long    keyId;
@@ -49,14 +53,29 @@ public class PreKey {
   @JsonProperty
   private boolean lastResort;
 
+  @JsonProperty
+  private int registrationId;
+
   public PreKey() {}
 
-  public PreKey(long id, String number, long keyId,
-                String publicKey, String identityKey,
-                boolean lastResort)
+  public PreKey(long id, String number, long deviceId, long keyId,
+                String publicKey, boolean lastResort)
   {
     this.id          = id;
     this.number      = number;
+    this.deviceId    = deviceId;
+    this.keyId       = keyId;
+    this.publicKey   = publicKey;
+    this.lastResort  = lastResort;
+  }
+
+  @VisibleForTesting
+  public PreKey(long id, String number, long deviceId, long keyId,
+                String publicKey, String identityKey, boolean lastResort)
+  {
+    this.id          = id;
+    this.number      = number;
+    this.deviceId    = deviceId;
     this.keyId       = keyId;
     this.publicKey   = publicKey;
     this.identityKey = identityKey;
@@ -113,4 +132,20 @@ public class PreKey {
   public void setLastResort(boolean lastResort) {
     this.lastResort = lastResort;
   }
+
+  public void setDeviceId(long deviceId) {
+    this.deviceId = deviceId;
+  }
+
+  public long getDeviceId() {
+    return deviceId;
+  }
+
+  public int getRegistrationId() {
+    return registrationId;
+  }
+
+  public void setRegistrationId(int registrationId) {
+    this.registrationId = registrationId;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/entities/PreKeyList.java

@@ -17,6 +17,7 @@
 package org.whispersystems.textsecuregcm.entities;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
 import org.hibernate.validator.constraints.NotEmpty;
 
 import javax.validation.Valid;
@@ -27,6 +28,7 @@ public class PreKeyList {
 
   @JsonProperty
   @NotNull
+  @Valid
   private PreKey lastResortKey;
 
   @JsonProperty
@@ -38,7 +40,17 @@ public class PreKeyList {
     return keys;
   }
 
+  @VisibleForTesting
+  public void setKeys(List<PreKey> keys) {
+    this.keys = keys;
+  }
+
   public PreKey getLastResortKey() {
     return lastResortKey;
   }
+
+  @VisibleForTesting
+  public void setLastResortKey(PreKey lastResortKey) {
+    this.lastResortKey = lastResortKey;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/entities/PreKeyStatus.java

@@ -0,0 +1,20 @@
+package org.whispersystems.textsecuregcm.entities;
+
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class PreKeyStatus {
+
+  @JsonProperty
+  private int count;
+
+  public PreKeyStatus(int count) {
+    this.count = count;
+  }
+
+  public PreKeyStatus() {}
+
+  public int getCount() {
+    return count;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/entities/RelayMessage.java

@@ -32,6 +32,10 @@ public class RelayMessage {
   @NotEmpty
   private String destination;
 
+  @JsonProperty
+  @NotEmpty
+  private long destinationDeviceId;
+
   @JsonProperty
   @NotNull
   @JsonSerialize(using = ByteArrayAdapter.Serializing.class)
@@ -40,8 +44,9 @@ public class RelayMessage {
 
   public RelayMessage() {}
 
-  public RelayMessage(String destination, byte[] outgoingMessageSignal) {
+  public RelayMessage(String destination, long destinationDeviceId, byte[] outgoingMessageSignal) {
     this.destination           = destination;
+    this.destinationDeviceId   = destinationDeviceId;
     this.outgoingMessageSignal = outgoingMessageSignal;
   }
 
@@ -49,6 +54,10 @@ public class RelayMessage {
     return destination;
   }
 
+  public long getDestinationDeviceId() {
+    return destinationDeviceId;
+  }
+
   public byte[] getOutgoingMessageSignal() {
     return outgoingMessageSignal;
   }

src/main/java/org/whispersystems/textsecuregcm/entities/StaleDevices.java

@@ -0,0 +1,18 @@
+package org.whispersystems.textsecuregcm.entities;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+public class StaleDevices {
+
+  @JsonProperty
+  private List<Long> staleDevices;
+
+  public StaleDevices() {}
+
+  public StaleDevices(List<Long> staleDevices) {
+    this.staleDevices = staleDevices;
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/entities/UnstructuredPreKeyList.java

@@ -0,0 +1,71 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.entities;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+public class UnstructuredPreKeyList {
+
+  @JsonProperty
+  @NotNull
+  @Valid
+  private List<PreKey> keys;
+
+  @VisibleForTesting
+  public UnstructuredPreKeyList() {}
+
+  public UnstructuredPreKeyList(PreKey preKey) {
+    this.keys = new LinkedList<PreKey>();
+    this.keys.add(preKey);
+  }
+
+  public UnstructuredPreKeyList(List<PreKey> preKeys) {
+    this.keys = preKeys;
+  }
+
+  public List<PreKey> getKeys() {
+    return keys;
+  }
+
+  @VisibleForTesting
+  public boolean equals(Object o) {
+    if (!(o instanceof UnstructuredPreKeyList) ||
+        ((UnstructuredPreKeyList) o).keys.size() != keys.size())
+      return false;
+    Iterator<PreKey> otherKeys = ((UnstructuredPreKeyList) o).keys.iterator();
+    for (PreKey key : keys) {
+      if (!otherKeys.next().equals(key))
+        return false;
+    }
+    return true;
+  }
+
+  public int hashCode() {
+    int ret = 0xFBA4C795 * keys.size();
+    for (PreKey key : keys)
+      ret ^= key.getPublicKey().hashCode();
+    return ret;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/federation/FederatedClient.java

@@ -17,6 +17,7 @@
 package org.whispersystems.textsecuregcm.federation;
 
 
+import com.google.common.base.Optional;
 import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientHandlerException;
 import com.sun.jersey.api.client.ClientResponse;
@@ -30,19 +31,19 @@ import org.apache.http.conn.ssl.StrictHostnameVerifier;
 import org.bouncycastle.openssl.PEMReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.controllers.NoSuchUserException;
 import org.whispersystems.textsecuregcm.entities.AccountCount;
 import org.whispersystems.textsecuregcm.entities.AttachmentUri;
 import org.whispersystems.textsecuregcm.entities.ClientContact;
 import org.whispersystems.textsecuregcm.entities.ClientContacts;
-import org.whispersystems.textsecuregcm.entities.MessageProtos.OutgoingMessageSignal;
-import org.whispersystems.textsecuregcm.entities.PreKey;
-import org.whispersystems.textsecuregcm.entities.RelayMessage;
+import org.whispersystems.textsecuregcm.entities.IncomingMessageList;
+import org.whispersystems.textsecuregcm.entities.UnstructuredPreKeyList;
 import org.whispersystems.textsecuregcm.util.Base64;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManagerFactory;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -55,6 +56,7 @@ import java.security.SecureRandom;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.util.List;
+import java.util.Map;
 
 public class FederatedClient {
 
@@ -62,8 +64,8 @@ public class FederatedClient {
 
   private static final String USER_COUNT_PATH     = "/v1/federation/user_count";
   private static final String USER_TOKENS_PATH    = "/v1/federation/user_tokens/%d";
-  private static final String RELAY_MESSAGE_PATH  = "/v1/federation/message";
-  private static final String PREKEY_PATH         = "/v1/federation/key/%s";
+  private static final String RELAY_MESSAGE_PATH  = "/v1/federation/messages/%s/%d/%s";
+  private static final String PREKEY_PATH_DEVICE  = "/v1/federation/key/%s/%s";
   private static final String ATTACHMENT_URI_PATH = "/v1/federation/attachment/%d";
 
   private final FederatedPeer peer;
@@ -89,25 +91,39 @@ public class FederatedClient {
       WebResource resource = client.resource(peer.getUrl())
                                    .path(String.format(ATTACHMENT_URI_PATH, attachmentId));
 
-      return resource.accept(MediaType.APPLICATION_JSON)
-                     .header("Authorization", authorizationHeader)
-                     .get(AttachmentUri.class)
-                     .getLocation();
+      ClientResponse response = resource.accept(MediaType.APPLICATION_JSON)
+                                        .header("Authorization", authorizationHeader)
+                                        .get(ClientResponse.class);
+
+      if (response.getStatus() < 200 || response.getStatus() >= 300) {
+        throw new WebApplicationException(clientResponseToResponse(response));
+      }
+
+      return response.getEntity(AttachmentUri.class).getLocation();
+
     } catch (UniformInterfaceException | ClientHandlerException e) {
       logger.warn("Bad URI", e);
       throw new IOException(e);
     }
   }
 
-  public PreKey getKey(String destination)  {
+  public Optional<UnstructuredPreKeyList> getKeys(String destination, String device) {
     try {
-      WebResource resource = client.resource(peer.getUrl()).path(String.format(PREKEY_PATH, destination));
-      return resource.accept(MediaType.APPLICATION_JSON)
-                     .header("Authorization", authorizationHeader)
-                     .get(PreKey.class);
+      WebResource resource = client.resource(peer.getUrl()).path(String.format(PREKEY_PATH_DEVICE, destination, device));
+
+      ClientResponse response = resource.accept(MediaType.APPLICATION_JSON)
+                                        .header("Authorization", authorizationHeader)
+                                        .get(ClientResponse.class);
+
+      if (response.getStatus() < 200 || response.getStatus() >= 300) {
+        throw new WebApplicationException(clientResponseToResponse(response));
+      }
+
+      return Optional.of(response.getEntity(UnstructuredPreKeyList.class));
+
     } catch (UniformInterfaceException | ClientHandlerException e) {
       logger.warn("PreKey", e);
-      return null;
+      return Optional.absent();
     }
   }
 
@@ -139,22 +155,18 @@ public class FederatedClient {
     }
   }
 
-  public void sendMessage(String destination, OutgoingMessageSignal message)
-      throws IOException, NoSuchUserException
+  public void sendMessages(String source, long sourceDeviceId, String destination, IncomingMessageList messages)
+      throws IOException
   {
     try {
-      WebResource    resource = client.resource(peer.getUrl()).path(RELAY_MESSAGE_PATH);
+      WebResource    resource = client.resource(peer.getUrl()).path(String.format(RELAY_MESSAGE_PATH, source, sourceDeviceId, destination));
       ClientResponse response = resource.type(MediaType.APPLICATION_JSON)
                                         .header("Authorization", authorizationHeader)
-                                        .entity(new RelayMessage(destination, message.toByteArray()))
+                                        .entity(messages)
                                         .put(ClientResponse.class);
 
-      if (response.getStatus() == 404) {
-        throw new NoSuchUserException("No remote user: " + destination);
-      }
-
       if (response.getStatus() != 200 && response.getStatus() != 204) {
-        throw new IOException("Bad response: " + response.getStatus());
+        throw new WebApplicationException(clientResponseToResponse(response));
       }
     } catch (UniformInterfaceException | ClientHandlerException e) {
       logger.warn("sendMessage", e);
@@ -206,6 +218,19 @@ public class FederatedClient {
     }
   }
 
+  private Response clientResponseToResponse(ClientResponse r) {
+    Response.ResponseBuilder rb = Response.status(r.getStatus());
+
+    for (Map.Entry<String, List<String>> entry : r.getHeaders().entrySet()) {
+      for (String value : entry.getValue()) {
+        rb.header(entry.getKey(), value);
+      }
+    }
+
+    rb.entity(r.getEntityInputStream());
+    return rb.build();
+  }
+
   public String getPeerName() {
     return peer.getName();
   }

src/main/java/org/whispersystems/textsecuregcm/federation/NonLimitedAccount.java

@@ -0,0 +1,45 @@
+package org.whispersystems.textsecuregcm.federation;
+
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.base.Optional;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.Device;
+
+public class NonLimitedAccount extends Account {
+
+  @JsonIgnore
+  private final String number;
+
+  @JsonIgnore
+  private final String relay;
+
+  @JsonIgnore
+  private final long deviceId;
+
+  public NonLimitedAccount(String number, long deviceId, String relay) {
+    this.number   = number;
+    this.deviceId = deviceId;
+    this.relay    = relay;
+  }
+
+  @Override
+  public String getNumber() {
+    return number;
+  }
+
+  @Override
+  public boolean isRateLimited() {
+    return false;
+  }
+
+  @Override
+  public Optional<String> getRelay() {
+    return Optional.of(relay);
+  }
+
+  @Override
+  public Optional<Device> getAuthenticatedDevice() {
+    return Optional.of(new Device(deviceId, null, null, null, null, null, false, 0));
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/limits/RateLimiter.java

@@ -16,13 +16,14 @@
  */
 package org.whispersystems.textsecuregcm.limits;
 
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import net.spy.memcached.MemcachedClient;
-
 import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
+import org.whispersystems.textsecuregcm.util.Constants;
 
-import java.util.concurrent.TimeUnit;
+import static com.codahale.metrics.MetricRegistry.name;
 
 public class RateLimiter {
 
@@ -35,7 +36,9 @@ public class RateLimiter {
   public RateLimiter(MemcachedClient memcachedClient, String name,
                      int bucketSize, double leakRatePerMinute)
   {
-    this.meter             = Metrics.newMeter(RateLimiter.class, name, "exceeded", TimeUnit.MINUTES);
+    MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+
+    this.meter             = metricRegistry.meter(name(getClass(), name, "exceeded"));
     this.memcachedClient   = memcachedClient;
     this.name              = name;
     this.bucketSize        = bucketSize;

src/main/java/org/whispersystems/textsecuregcm/limits/RateLimiters.java

@@ -31,6 +31,9 @@ public class RateLimiters {
   private final RateLimiter preKeysLimiter;
   private final RateLimiter messagesLimiter;
 
+  private final RateLimiter allocateDeviceLimiter;
+  private final RateLimiter verifyDeviceLimiter;
+
   public RateLimiters(RateLimitsConfiguration config, MemcachedClient memcachedClient) {
     this.smsDestinationLimiter = new RateLimiter(memcachedClient, "smsDestination",
                                                  config.getSmsDestination().getBucketSize(),
@@ -59,6 +62,23 @@ public class RateLimiters {
     this.messagesLimiter = new RateLimiter(memcachedClient, "messages",
                                            config.getMessages().getBucketSize(),
                                            config.getMessages().getLeakRatePerMinute());
+
+    this.allocateDeviceLimiter = new RateLimiter(memcachedClient, "allocateDevice",
+                                                 config.getAllocateDevice().getBucketSize(),
+                                                 config.getAllocateDevice().getLeakRatePerMinute());
+
+    this.verifyDeviceLimiter = new RateLimiter(memcachedClient, "verifyDevice",
+                                               config.getVerifyDevice().getBucketSize(),
+                                               config.getVerifyDevice().getLeakRatePerMinute());
+
+  }
+
+  public RateLimiter getAllocateDeviceLimiter() {
+    return allocateDeviceLimiter;
+  }
+
+  public RateLimiter getVerifyDeviceLimiter() {
+    return verifyDeviceLimiter;
   }
 
   public RateLimiter getMessagesLimiter() {

src/main/java/org/whispersystems/textsecuregcm/metrics/CpuUsageGauge.java

@@ -0,0 +1,16 @@
+package org.whispersystems.textsecuregcm.metrics;
+
+import com.codahale.metrics.Gauge;
+import com.sun.management.OperatingSystemMXBean;
+
+import java.lang.management.ManagementFactory;
+
+public class CpuUsageGauge implements Gauge<Integer> {
+  @Override
+  public Integer getValue() {
+    OperatingSystemMXBean mbean = (com.sun.management.OperatingSystemMXBean)
+        ManagementFactory.getOperatingSystemMXBean();
+
+    return (int) Math.ceil(mbean.getSystemCpuLoad() * 100);
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/metrics/FreeMemoryGauge.java

@@ -0,0 +1,17 @@
+package org.whispersystems.textsecuregcm.metrics;
+
+import com.codahale.metrics.Gauge;
+import com.sun.management.OperatingSystemMXBean;
+
+import java.lang.management.ManagementFactory;
+
+public class FreeMemoryGauge implements Gauge<Long> {
+
+  @Override
+  public Long getValue() {
+    OperatingSystemMXBean mbean = (com.sun.management.OperatingSystemMXBean)
+        ManagementFactory.getOperatingSystemMXBean();
+
+    return mbean.getFreePhysicalMemorySize();
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/metrics/JsonMetricsReporter.java

@@ -0,0 +1,184 @@
+package org.whispersystems.textsecuregcm.metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metered;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.InetAddress;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+/**
+ * Adapted from MetricsServlet.
+ */
+public class JsonMetricsReporter extends ScheduledReporter {
+
+  private final Logger      logger  = LoggerFactory.getLogger(JsonMetricsReporter.class);
+  private final JsonFactory factory = new JsonFactory();
+
+  private final String table;
+  private final String sunnylabsHost;
+  private final String host;
+
+  public JsonMetricsReporter(MetricRegistry registry, String token, String sunnylabsHost)
+      throws UnknownHostException
+  {
+    super(registry, "jsonmetrics-reporter", MetricFilter.ALL, TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
+    this.table         = token;
+    this.sunnylabsHost = sunnylabsHost;
+    this.host          = InetAddress.getLocalHost().getHostName();
+  }
+
+  @Override
+  public void report(SortedMap<String, Gauge>     stringGaugeSortedMap,
+                     SortedMap<String, Counter>   stringCounterSortedMap,
+                     SortedMap<String, Histogram> stringHistogramSortedMap,
+                     SortedMap<String, Meter>     stringMeterSortedMap,
+                     SortedMap<String, Timer>     stringTimerSortedMap)
+  {
+    try {
+      logger.info("Reporting metrics...");
+      URL url = new URL("https", sunnylabsHost, 443, "/report/metrics?t=" + table + "&h=" + host);
+      HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+      connection.setDoOutput(true);
+      connection.addRequestProperty("Content-Type", "application/json");
+
+      OutputStream  outputStream = connection.getOutputStream();
+      JsonGenerator json         = factory.createGenerator(outputStream, JsonEncoding.UTF8);
+
+      json.writeStartObject();
+
+      for (Map.Entry<String, Gauge> gauge : stringGaugeSortedMap.entrySet()) {
+        reportGauge(json, gauge.getKey(), gauge.getValue());
+      }
+
+      for (Map.Entry<String, Counter> counter : stringCounterSortedMap.entrySet()) {
+        reportCounter(json, counter.getKey(), counter.getValue());
+      }
+
+      for (Map.Entry<String, Histogram> histogram : stringHistogramSortedMap.entrySet()) {
+        reportHistogram(json, histogram.getKey(), histogram.getValue());
+      }
+
+      for (Map.Entry<String, Meter> meter : stringMeterSortedMap.entrySet()) {
+        reportMeter(json, meter.getKey(), meter.getValue());
+      }
+
+      for (Map.Entry<String, Timer> timer : stringTimerSortedMap.entrySet()) {
+        reportTimer(json, timer.getKey(), timer.getValue());
+      }
+
+      json.writeEndObject();
+      json.close();
+
+      outputStream.close();
+
+      logger.info("Metrics server response: " + connection.getResponseCode());
+    } catch (IOException e) {
+      logger.warn("Error sending metrics", e);
+    } catch (Exception e) {
+      logger.warn("error", e);
+    }
+  }
+
+  private void reportGauge(JsonGenerator json, String name, Gauge gauge) throws IOException {
+    Object gaugeValue = evaluateGauge(gauge);
+
+    if (gaugeValue instanceof Number) {
+      json.writeFieldName(sanitize(name));
+      json.writeObject(gaugeValue);
+    }
+  }
+
+  private void reportCounter(JsonGenerator json, String name, Counter counter) throws IOException {
+    json.writeFieldName(sanitize(name));
+    json.writeNumber(counter.getCount());
+  }
+
+  private void reportHistogram(JsonGenerator json, String name, Histogram histogram) throws IOException {
+    Snapshot snapshot = histogram.getSnapshot();
+    json.writeFieldName(sanitize(name));
+    json.writeStartObject();
+    json.writeNumberField("count", histogram.getCount());
+    writeSnapshot(json, snapshot);
+    json.writeEndObject();
+  }
+
+  private void reportMeter(JsonGenerator json, String name, Meter meter) throws IOException {
+    json.writeFieldName(sanitize(name));
+    json.writeStartObject();
+    writeMetered(json, meter);
+    json.writeEndObject();
+  }
+
+  private void reportTimer(JsonGenerator json, String name, Timer timer) throws IOException {
+    json.writeFieldName(sanitize(name));
+    json.writeStartObject();
+    json.writeFieldName("rate");
+    json.writeStartObject();
+    writeMetered(json, timer);
+    json.writeEndObject();
+    json.writeFieldName("duration");
+    json.writeStartObject();
+    writeSnapshot(json, timer.getSnapshot());
+    json.writeEndObject();
+    json.writeEndObject();
+  }
+
+  private Object evaluateGauge(Gauge gauge) {
+    try {
+      return gauge.getValue();
+    } catch (RuntimeException e) {
+      logger.warn("Error reading gauge", e);
+      return "error reading gauge";
+    }
+  }
+
+  private void writeSnapshot(JsonGenerator json, Snapshot snapshot) throws IOException {
+    json.writeNumberField("max", convertDuration(snapshot.getMax()));
+    json.writeNumberField("mean", convertDuration(snapshot.getMean()));
+    json.writeNumberField("min", convertDuration(snapshot.getMin()));
+    json.writeNumberField("stddev", convertDuration(snapshot.getStdDev()));
+    json.writeNumberField("median", convertDuration(snapshot.getMedian()));
+    json.writeNumberField("p75", convertDuration(snapshot.get75thPercentile()));
+    json.writeNumberField("p95", convertDuration(snapshot.get95thPercentile()));
+    json.writeNumberField("p98", convertDuration(snapshot.get98thPercentile()));
+    json.writeNumberField("p99", convertDuration(snapshot.get99thPercentile()));
+    json.writeNumberField("p999", convertDuration(snapshot.get999thPercentile()));
+  }
+
+  private void writeMetered(JsonGenerator json, Metered meter) throws IOException {
+    json.writeNumberField("count", convertRate(meter.getCount()));
+    json.writeNumberField("mean", convertRate(meter.getMeanRate()));
+    json.writeNumberField("m1", convertRate(meter.getOneMinuteRate()));
+    json.writeNumberField("m5", convertRate(meter.getFiveMinuteRate()));
+    json.writeNumberField("m15", convertRate(meter.getFifteenMinuteRate()));
+  }
+
+  private static final Pattern SIMPLE_NAMES = Pattern.compile("[^a-zA-Z0-9_.\\-~]");
+
+  private String sanitize(String metricName) {
+    return SIMPLE_NAMES.matcher(metricName).replaceAll("_");
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/metrics/NetworkGauge.java

@@ -0,0 +1,36 @@
+package org.whispersystems.textsecuregcm.metrics;
+
+
+import com.codahale.metrics.Gauge;
+import org.whispersystems.textsecuregcm.util.Pair;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+public abstract class NetworkGauge implements Gauge<Long> {
+
+  protected Pair<Long, Long> getSentReceived() throws IOException {
+    File           proc          = new File("/proc/net/dev");
+    BufferedReader reader        = new BufferedReader(new FileReader(proc));
+    String         header        = reader.readLine();
+    String         header2       = reader.readLine();
+
+    long           bytesSent     = 0;
+    long           bytesReceived = 0;
+
+    String interfaceStats;
+
+      while ((interfaceStats = reader.readLine()) != null) {
+        String[] stats = interfaceStats.split("\\s+");
+
+        if (!stats[1].equals("lo:")) {
+          bytesReceived += Long.parseLong(stats[2]);
+          bytesSent     += Long.parseLong(stats[10]);
+        }
+      }
+
+    return new Pair<>(bytesSent, bytesReceived);
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/metrics/NetworkReceivedGauge.java

@@ -0,0 +1,36 @@
+package org.whispersystems.textsecuregcm.metrics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.textsecuregcm.util.Pair;
+
+import java.io.IOException;
+
+public class NetworkReceivedGauge extends NetworkGauge {
+
+  private final Logger logger = LoggerFactory.getLogger(NetworkSentGauge.class);
+
+  private long lastTimestamp;
+  private long lastReceived;
+
+  @Override
+  public Long getValue() {
+    try {
+      long             timestamp       = System.currentTimeMillis();
+      Pair<Long, Long> sentAndReceived = getSentReceived();
+      long             result          = 0;
+
+      if (lastTimestamp != 0) {
+        result       = sentAndReceived.second() - lastReceived;
+        lastReceived = sentAndReceived.second();
+      }
+
+      lastTimestamp = timestamp;
+      return result;
+    } catch (IOException e) {
+      logger.warn("NetworkReceivedGauge", e);
+      return -1L;
+    }
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/metrics/NetworkSentGauge.java

@@ -0,0 +1,35 @@
+package org.whispersystems.textsecuregcm.metrics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.textsecuregcm.util.Pair;
+
+import java.io.IOException;
+
+public class NetworkSentGauge extends NetworkGauge {
+
+  private final Logger logger = LoggerFactory.getLogger(NetworkSentGauge.class);
+
+  private long lastTimestamp;
+  private long lastSent;
+
+  @Override
+  public Long getValue() {
+    try {
+      long             timestamp       = System.currentTimeMillis();
+      Pair<Long, Long> sentAndReceived = getSentReceived();
+      long             result          = 0;
+
+      if (lastTimestamp != 0) {
+        result        = sentAndReceived.first() - lastSent;
+        lastSent      = sentAndReceived.first();
+      }
+
+      lastTimestamp = timestamp;
+      return result;
+    } catch (IOException e) {
+      logger.warn("NetworkSentGauge", e);
+      return -1L;
+    }
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/providers/MemcacheHealthCheck.java

@@ -16,8 +16,7 @@
  */
 package org.whispersystems.textsecuregcm.providers;
 
-import com.yammer.metrics.core.HealthCheck;
-import com.yammer.metrics.core.HealthCheck.Result;
+import com.codahale.metrics.health.HealthCheck;
 import net.spy.memcached.MemcachedClient;
 
 import java.security.SecureRandom;
@@ -27,7 +26,6 @@ public class MemcacheHealthCheck extends HealthCheck {
   private final MemcachedClient client;
 
   public MemcacheHealthCheck(MemcachedClient client) {
-    super("memcached");
     this.client = client;
   }
 

src/main/java/org/whispersystems/textsecuregcm/providers/RedisHealthCheck.java

@@ -16,7 +16,7 @@
  */
 package org.whispersystems.textsecuregcm.providers;
 
-import com.yammer.metrics.core.HealthCheck;
+import com.codahale.metrics.health.HealthCheck;
 
 import redis.clients.jedis.Jedis;
 import redis.clients.jedis.JedisPool;
@@ -26,7 +26,6 @@ public class RedisHealthCheck extends HealthCheck {
   private final JedisPool clientPool;
 
   public RedisHealthCheck(JedisPool clientPool) {
-    super("redis");
     this.clientPool = clientPool;
   }
 

src/main/java/org/whispersystems/textsecuregcm/push/APNSender.java

@@ -16,23 +16,30 @@
  */
 package org.whispersystems.textsecuregcm.push;
 
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import com.google.common.base.Optional;
 import com.notnoop.apns.APNS;
 import com.notnoop.apns.ApnsService;
 import com.notnoop.exceptions.NetworkIOException;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
 import org.bouncycastle.openssl.PEMReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.entities.EncryptedOutgoingMessage;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.storage.PubSubManager;
+import org.whispersystems.textsecuregcm.storage.PubSubMessage;
+import org.whispersystems.textsecuregcm.storage.StoredMessages;
+import org.whispersystems.textsecuregcm.util.Constants;
 import org.whispersystems.textsecuregcm.util.Util;
+import org.whispersystems.textsecuregcm.websocket.WebsocketAddress;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.net.MalformedURLException;
 import java.security.KeyPair;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
@@ -40,21 +47,31 @@ import java.security.NoSuchAlgorithmException;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
-import java.util.concurrent.TimeUnit;
+
+import static com.codahale.metrics.MetricRegistry.name;
 
 public class APNSender {
 
-  private final Meter  success = Metrics.newMeter(APNSender.class, "sent", "success", TimeUnit.MINUTES);
-  private final Meter  failure = Metrics.newMeter(APNSender.class, "sent", "failure", TimeUnit.MINUTES);
-  private final Logger logger  = LoggerFactory.getLogger(APNSender.class);
+  private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+  private final Meter          websocketMeter = metricRegistry.meter(name(getClass(), "websocket"));
+  private final Meter          pushMeter      = metricRegistry.meter(name(getClass(), "push"));
+  private final Meter          failureMeter   = metricRegistry.meter(name(getClass(), "failure"));
+  private final Logger         logger         = LoggerFactory.getLogger(APNSender.class);
 
   private static final String MESSAGE_BODY = "m";
 
   private final Optional<ApnsService> apnService;
+  private final PubSubManager         pubSubManager;
+  private final StoredMessages        storedMessages;
 
-  public APNSender(String apnCertificate, String apnKey)
+  public APNSender(PubSubManager pubSubManager,
+                   StoredMessages storedMessages,
+                   String apnCertificate, String apnKey)
       throws CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException
   {
+    this.pubSubManager  = pubSubManager;
+    this.storedMessages = storedMessages;
+
     if (!Util.isEmpty(apnCertificate) && !Util.isEmpty(apnKey)) {
       byte[] keyStore = initializeKeyStore(apnCertificate, apnKey);
       this.apnService = Optional.of(APNS.newService()
@@ -65,31 +82,44 @@ public class APNSender {
     }
   }
 
-  public void sendMessage(String registrationId, EncryptedOutgoingMessage message)
-      throws IOException
+  public void sendMessage(Account account, Device device,
+                          String registrationId, EncryptedOutgoingMessage message)
+      throws TransientPushFailureException, NotPushRegisteredException
+  {
+    if (pubSubManager.publish(new WebsocketAddress(account.getId(), device.getId()),
+                              new PubSubMessage(PubSubMessage.TYPE_DELIVER, message.serialize())))
+    {
+      websocketMeter.mark();
+    } else {
+      storedMessages.insert(account.getId(), device.getId(), message.serialize());
+      sendPush(registrationId, message.serialize());
+    }
+  }
+
+  private void sendPush(String registrationId, String message)
+      throws TransientPushFailureException
   {
     try {
       if (!apnService.isPresent()) {
-        failure.mark();
-        throw new IOException("APN access not configured!");
+        failureMeter.mark();
+        throw new TransientPushFailureException("APN access not configured!");
       }
 
       String payload = APNS.newPayload()
                            .alertBody("Message!")
-                           .customField(MESSAGE_BODY, message.serialize())
+                           .customField(MESSAGE_BODY, message)
                            .build();
 
       logger.debug("APN Payload: " + payload);
 
       apnService.get().push(registrationId, payload);
-      success.mark();
-    } catch (MalformedURLException mue) {
-      throw new AssertionError(mue);
+      pushMeter.mark();
     } catch (NetworkIOException nioe) {
       logger.warn("Network Error", nioe);
-      failure.mark();
-      throw new IOException("Error sending APN");
+      failureMeter.mark();
+      throw new TransientPushFailureException(nioe);
     }
+
   }
 
   private static byte[] initializeKeyStore(String pemCertificate, String pemKey)

src/main/java/org/whispersystems/textsecuregcm/push/GCMSender.java

@@ -16,22 +16,24 @@
  */
 package org.whispersystems.textsecuregcm.push;
 
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import com.google.android.gcm.server.Constants;
 import com.google.android.gcm.server.Message;
 import com.google.android.gcm.server.Result;
 import com.google.android.gcm.server.Sender;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
-import org.whispersystems.textsecuregcm.controllers.NoSuchUserException;
 import org.whispersystems.textsecuregcm.entities.EncryptedOutgoingMessage;
 
 import java.io.IOException;
-import java.util.concurrent.TimeUnit;
+
+import static com.codahale.metrics.MetricRegistry.name;
 
 public class GCMSender {
 
-  private final Meter success = Metrics.newMeter(GCMSender.class, "sent", "success", TimeUnit.MINUTES);
-  private final Meter failure = Metrics.newMeter(GCMSender.class, "sent", "failure", TimeUnit.MINUTES);
+  private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(org.whispersystems.textsecuregcm.util.Constants.METRICS_NAME);
+  private final Meter          success        = metricRegistry.meter(name(getClass(), "sent", "success"));
+  private final Meter          failure        = metricRegistry.meter(name(getClass(), "sent", "failure"));
 
   private final Sender sender;
 
@@ -40,24 +42,28 @@ public class GCMSender {
   }
 
   public String sendMessage(String gcmRegistrationId, EncryptedOutgoingMessage outgoingMessage)
-      throws IOException, NoSuchUserException
+      throws NotPushRegisteredException, TransientPushFailureException
   {
-    Message gcmMessage = new Message.Builder().addData("type", "message")
-                                              .addData("message", outgoingMessage.serialize())
-                                              .build();
+    try {
+      Message gcmMessage = new Message.Builder().addData("type", "message")
+                                                .addData("message", outgoingMessage.serialize())
+                                                .build();
 
-    Result  result = sender.send(gcmMessage, gcmRegistrationId, 5);
+      Result  result = sender.send(gcmMessage, gcmRegistrationId, 5);
 
-    if (result.getMessageId() != null) {
-      success.mark();
-      return result.getCanonicalRegistrationId();
-    } else {
-      failure.mark();
-      if (result.getErrorCodeName().equals(Constants.ERROR_NOT_REGISTERED)) {
-        throw new NoSuchUserException("User no longer registered with GCM.");
+      if (result.getMessageId() != null) {
+        success.mark();
+        return result.getCanonicalRegistrationId();
       } else {
-        throw new IOException("GCM Failed: " + result.getErrorCodeName());
+        failure.mark();
+        if (result.getErrorCodeName().equals(Constants.ERROR_NOT_REGISTERED)) {
+          throw new NotPushRegisteredException("Device no longer registered with GCM.");
+        } else {
+          throw new TransientPushFailureException("GCM Failed: " + result.getErrorCodeName());
+        }
       }
+    } catch (IOException e) {
+      throw new TransientPushFailureException(e);
     }
   }
 }

src/main/java/org/whispersystems/textsecuregcm/push/NotPushRegisteredException.java

@@ -0,0 +1,11 @@
+package org.whispersystems.textsecuregcm.push;
+
+public class NotPushRegisteredException extends Exception {
+  public NotPushRegisteredException(String s) {
+    super(s);
+  }
+
+  public NotPushRegisteredException(Exception e) {
+    super(e);
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/push/PushSender.java

@@ -16,17 +16,18 @@
  */
 package org.whispersystems.textsecuregcm.push;
 
-import com.google.common.base.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.configuration.ApnConfiguration;
 import org.whispersystems.textsecuregcm.configuration.GcmConfiguration;
-import org.whispersystems.textsecuregcm.controllers.NoSuchUserException;
+import org.whispersystems.textsecuregcm.entities.CryptoEncodingException;
 import org.whispersystems.textsecuregcm.entities.EncryptedOutgoingMessage;
 import org.whispersystems.textsecuregcm.entities.MessageProtos;
 import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
-import org.whispersystems.textsecuregcm.storage.DirectoryManager;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.storage.PubSubManager;
+import org.whispersystems.textsecuregcm.storage.StoredMessages;
 
 import java.io.IOException;
 import java.security.KeyStoreException;
@@ -37,67 +38,86 @@ public class PushSender {
 
   private final Logger logger = LoggerFactory.getLogger(PushSender.class);
 
-  private final AccountsManager  accounts;
-  private final DirectoryManager directory;
-
-  private final GCMSender gcmSender;
-  private final APNSender apnSender;
+  private final AccountsManager accounts;
+  private final GCMSender       gcmSender;
+  private final APNSender       apnSender;
+  private final WebsocketSender webSocketSender;
 
   public PushSender(GcmConfiguration gcmConfiguration,
                     ApnConfiguration apnConfiguration,
-                    AccountsManager accounts,
-                    DirectoryManager directory)
+                    StoredMessages   storedMessages,
+                    PubSubManager    pubSubManager,
+                    AccountsManager  accounts)
       throws CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException
   {
-    this.accounts  = accounts;
-    this.directory = directory;
-
-    this.gcmSender = new GCMSender(gcmConfiguration.getApiKey());
-    this.apnSender = new APNSender(apnConfiguration.getCertificate(), apnConfiguration.getKey());
+    this.accounts        = accounts;
+    this.webSocketSender = new WebsocketSender(storedMessages, pubSubManager);
+    this.gcmSender       = new GCMSender(gcmConfiguration.getApiKey());
+    this.apnSender       = new APNSender(pubSubManager, storedMessages,
+                                         apnConfiguration.getCertificate(),
+                                         apnConfiguration.getKey());
   }
 
-  public void sendMessage(String destination, MessageProtos.OutgoingMessageSignal outgoingMessage)
-      throws IOException, NoSuchUserException
-  {
-    Optional<Account> account = accounts.get(destination);
-
-    if (!account.isPresent()) {
-      directory.remove(destination);
-      throw new NoSuchUserException("No such local destination: " + destination);
-    }
-
-    String signalingKey              = account.get().getSignalingKey();
-    EncryptedOutgoingMessage message = new EncryptedOutgoingMessage(outgoingMessage, signalingKey);
-
-    if      (account.get().getGcmRegistrationId() != null) sendGcmMessage(account.get(), message);
-    else if (account.get().getApnRegistrationId() != null) sendApnMessage(account.get(), message);
-    else                                             throw new NoSuchUserException("No push identifier!");
-  }
-
-  private void sendGcmMessage(Account account, EncryptedOutgoingMessage outgoingMessage)
-      throws IOException, NoSuchUserException
+  public void sendMessage(Account account, Device device, MessageProtos.OutgoingMessageSignal message)
+      throws NotPushRegisteredException, TransientPushFailureException
   {
     try {
-      String canonicalId = gcmSender.sendMessage(account.getGcmRegistrationId(),
-                                                 outgoingMessage);
+      String                   signalingKey     = device.getSignalingKey();
+      EncryptedOutgoingMessage encryptedMessage = new EncryptedOutgoingMessage(message, signalingKey);
+
+      sendMessage(account, device, encryptedMessage);
+    } catch (CryptoEncodingException e) {
+      throw new NotPushRegisteredException(e);
+    }
+  }
+
+  public void sendMessage(Account account, Device device, EncryptedOutgoingMessage message)
+      throws NotPushRegisteredException, TransientPushFailureException
+  {
+    if      (device.getGcmId() != null)   sendGcmMessage(account, device, message);
+    else if (device.getApnId() != null)   sendApnMessage(account, device, message);
+    else if (device.getFetchesMessages()) sendWebSocketMessage(account, device, message);
+    else                                  throw new NotPushRegisteredException("No delivery possible!");
+  }
+
+  private void sendGcmMessage(Account account, Device device, EncryptedOutgoingMessage outgoingMessage)
+      throws NotPushRegisteredException, TransientPushFailureException
+  {
+    try {
+      String canonicalId = gcmSender.sendMessage(device.getGcmId(), outgoingMessage);
 
       if (canonicalId != null) {
-        account.setGcmRegistrationId(canonicalId);
+        device.setGcmId(canonicalId);
         accounts.update(account);
       }
 
-    } catch (NoSuchUserException e) {
+    } catch (NotPushRegisteredException e) {
       logger.debug("No Such User", e);
-      account.setGcmRegistrationId(null);
+      device.setGcmId(null);
       accounts.update(account);
-      throw new NoSuchUserException("User no longer exists in GCM.");
+      throw new NotPushRegisteredException(e);
     }
   }
 
-  private void sendApnMessage(Account account, EncryptedOutgoingMessage outgoingMessage)
-      throws IOException
+  private void sendApnMessage(Account account, Device device, EncryptedOutgoingMessage outgoingMessage)
+      throws TransientPushFailureException, NotPushRegisteredException
   {
-    apnSender.sendMessage(account.getApnRegistrationId(), outgoingMessage);
+    try {
+      apnSender.sendMessage(account, device, device.getApnId(), outgoingMessage);
+    } catch (NotPushRegisteredException e) {
+      device.setApnId(null);
+      accounts.update(account);
+      throw new NotPushRegisteredException(e);
+    }
   }
 
+  private void sendWebSocketMessage(Account account, Device device, EncryptedOutgoingMessage outgoingMessage)
+      throws NotPushRegisteredException
+  {
+    try {
+      webSocketSender.sendMessage(account, device, outgoingMessage);
+    } catch (CryptoEncodingException e) {
+      throw new NotPushRegisteredException(e);
+    }
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/push/TransientPushFailureException.java

@@ -0,0 +1,11 @@
+package org.whispersystems.textsecuregcm.push;
+
+public class TransientPushFailureException extends Exception {
+  public TransientPushFailureException(String s) {
+    super(s);
+  }
+
+  public TransientPushFailureException(Exception e) {
+    super(e);
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/push/WebsocketSender.java

@@ -0,0 +1,68 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.push;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
+import org.whispersystems.textsecuregcm.entities.CryptoEncodingException;
+import org.whispersystems.textsecuregcm.entities.EncryptedOutgoingMessage;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.storage.PubSubManager;
+import org.whispersystems.textsecuregcm.storage.PubSubMessage;
+import org.whispersystems.textsecuregcm.storage.StoredMessages;
+import org.whispersystems.textsecuregcm.util.Constants;
+import org.whispersystems.textsecuregcm.websocket.WebsocketAddress;
+
+import java.util.List;
+
+import static com.codahale.metrics.MetricRegistry.name;
+
+public class WebsocketSender {
+
+  private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+  private final Meter          onlineMeter    = metricRegistry.meter(name(getClass(), "online"));
+  private final Meter          offlineMeter   = metricRegistry.meter(name(getClass(), "offline"));
+
+  private final StoredMessages storedMessages;
+  private final PubSubManager  pubSubManager;
+
+  public WebsocketSender(StoredMessages storedMessages, PubSubManager pubSubManager) {
+    this.storedMessages = storedMessages;
+    this.pubSubManager  = pubSubManager;
+  }
+
+  public void sendMessage(Account account, Device device, EncryptedOutgoingMessage outgoingMessage)
+      throws CryptoEncodingException
+  {
+    sendMessage(account, device, outgoingMessage.serialize());
+  }
+
+  private void sendMessage(Account account, Device device, String serializedMessage) {
+    WebsocketAddress address       = new WebsocketAddress(account.getId(), device.getId());
+    PubSubMessage    pubSubMessage = new PubSubMessage(PubSubMessage.TYPE_DELIVER, serializedMessage);
+
+    if (pubSubManager.publish(address, pubSubMessage)) {
+      onlineMeter.mark();
+    } else {
+      offlineMeter.mark();
+      storedMessages.insert(account.getId(), device.getId(), serializedMessage);
+      pubSubManager.publish(address, new PubSubMessage(PubSubMessage.TYPE_QUERY_DB, null));
+    }
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/sms/NexmoSmsSender.java

@@ -16,11 +16,13 @@
  */
 package org.whispersystems.textsecuregcm.sms;
 
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.configuration.NexmoConfiguration;
+import org.whispersystems.textsecuregcm.util.Constants;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -28,16 +30,15 @@ import java.io.InputStreamReader;
 import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLEncoder;
-import java.util.concurrent.TimeUnit;
 
-import org.whispersystems.textsecuregcm.sms.SenderFactory.VoxSender;
-import org.whispersystems.textsecuregcm.sms.SenderFactory.SmsSender;
+import static com.codahale.metrics.MetricRegistry.name;
 
-public class NexmoSmsSender implements SmsSender, VoxSender {
+public class NexmoSmsSender {
 
-  private final Meter  smsMeter = Metrics.newMeter(NexmoSmsSender.class, "sms", "delivered", TimeUnit.MINUTES);
-  private final Meter  voxMeter = Metrics.newMeter(NexmoSmsSender.class, "vox", "delivered", TimeUnit.MINUTES);
-  private final Logger logger   = LoggerFactory.getLogger(NexmoSmsSender.class);
+  private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+  private final Meter          smsMeter       = metricRegistry.meter(name(getClass(), "sms", "delivered"));
+  private final Meter          voxMeter       = metricRegistry.meter(name(getClass(), "vox", "delivered"));
+  private final Logger         logger         = LoggerFactory.getLogger(NexmoSmsSender.class);
 
   private static final String NEXMO_SMS_URL =
       "https://rest.nexmo.com/sms/json?api_key=%s&api_secret=%s&from=%s&to=%s&text=%s";
@@ -55,10 +56,9 @@ public class NexmoSmsSender implements SmsSender, VoxSender {
     this.number    = config.getNumber();
   }
 
-  @Override
   public void deliverSmsVerification(String destination, String verificationCode) throws IOException {
     URL url = new URL(String.format(NEXMO_SMS_URL, apiKey, apiSecret, number, destination,
-                                    URLEncoder.encode(SmsSender.VERIFICATION_TEXT + verificationCode, "UTF-8")));
+                                    URLEncoder.encode(SmsSender.SMS_VERIFICATION_TEXT + verificationCode, "UTF-8")));
 
     URLConnection connection = url.openConnection();
     connection.setDoInput(true);
@@ -70,10 +70,9 @@ public class NexmoSmsSender implements SmsSender, VoxSender {
     smsMeter.mark();
   }
 
-  @Override
   public void deliverVoxVerification(String destination, String message) throws IOException {
     URL url = new URL(String.format(NEXMO_VOX_URL, apiKey, apiSecret, destination,
-                                    URLEncoder.encode(VoxSender.VERIFICATION_TEXT + message, "UTF-8")));
+                                    URLEncoder.encode(SmsSender.VOX_VERIFICATION_TEXT + message, "UTF-8")));
 
     URLConnection connection = url.openConnection();
     connection.setDoInput(true);

src/main/java/org/whispersystems/textsecuregcm/sms/SenderFactory.java

@@ -1,70 +0,0 @@
-/**
- * Copyright (C) 2013 Open WhisperSystems
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-package org.whispersystems.textsecuregcm.sms;
-
-
-import com.google.common.base.Optional;
-import org.whispersystems.textsecuregcm.configuration.NexmoConfiguration;
-import org.whispersystems.textsecuregcm.configuration.TwilioConfiguration;
-
-import java.io.IOException;
-
-public class SenderFactory {
-
-  private final TwilioSmsSender          twilioSender;
-  private final Optional<NexmoSmsSender> nexmoSender;
-
-  public SenderFactory(TwilioConfiguration twilioConfig, NexmoConfiguration nexmoConfig) {
-    this.twilioSender = new TwilioSmsSender(twilioConfig);
-
-    if (nexmoConfig != null) {
-      this.nexmoSender = Optional.of(new NexmoSmsSender(nexmoConfig));
-    } else {
-      this.nexmoSender = Optional.absent();
-    }
-  }
-
-  public SmsSender getSmsSender(String number) {
-    if (nexmoSender.isPresent() && !isTwilioDestination(number)) {
-      return nexmoSender.get();
-    } else {
-      return twilioSender;
-    }
-  }
-
-  public VoxSender getVoxSender(String number) {
-    if (nexmoSender.isPresent()) {
-      return nexmoSender.get();
-    } else {
-      return twilioSender;
-    }
-  }
-
-  private boolean isTwilioDestination(String number) {
-    return number.length() == 12 && number.startsWith("+1");
-  }
-
-  public interface SmsSender {
-    public static final String VERIFICATION_TEXT = "Your TextSecure verification code: ";
-    public void deliverSmsVerification(String destination, String verificationCode) throws IOException;
-  }
-
-  public interface VoxSender {
-    public static final String VERIFICATION_TEXT = "Your TextSecure verification code is: ";
-    public void deliverVoxVerification(String destination, String verificationCode) throws IOException;
-  }
-}

src/main/java/org/whispersystems/textsecuregcm/sms/SmsSender.java

@@ -0,0 +1,84 @@
+/**
+ * Copyright (C) 2013 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.sms;
+
+
+import com.google.common.base.Optional;
+import com.twilio.sdk.TwilioRestException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class SmsSender {
+
+  static final String SMS_VERIFICATION_TEXT = "Your TextSecure verification code: ";
+  static final String VOX_VERIFICATION_TEXT = "Your TextSecure verification code is: ";
+
+  private final Logger logger = LoggerFactory.getLogger(SmsSender.class);
+
+  private final TwilioSmsSender          twilioSender;
+  private final Optional<NexmoSmsSender> nexmoSender;
+  private final boolean                  isTwilioInternational;
+
+  public SmsSender(TwilioSmsSender twilioSender,
+                   Optional<NexmoSmsSender> nexmoSender,
+                   boolean isTwilioInternational)
+  {
+    this.isTwilioInternational = isTwilioInternational;
+    this.twilioSender          = twilioSender;
+    this.nexmoSender           = nexmoSender;
+  }
+
+  public void deliverSmsVerification(String destination, String verificationCode)
+      throws IOException
+  {
+    if (!isTwilioDestination(destination) && nexmoSender.isPresent()) {
+      nexmoSender.get().deliverSmsVerification(destination, verificationCode);
+    } else {
+      try {
+        twilioSender.deliverSmsVerification(destination, verificationCode);
+      } catch (TwilioRestException e) {
+        logger.info("Twilio SMS Fallback", e);
+        if (nexmoSender.isPresent()) {
+          nexmoSender.get().deliverSmsVerification(destination, verificationCode);
+        }
+      }
+    }
+  }
+
+  public void deliverVoxVerification(String destination, String verificationCode)
+      throws IOException
+  {
+    if (!isTwilioDestination(destination) && nexmoSender.isPresent()) {
+      nexmoSender.get().deliverVoxVerification(destination, verificationCode);
+    } else {
+      try {
+        twilioSender.deliverVoxVerification(destination, verificationCode);
+      } catch (TwilioRestException e) {
+        logger.info("Twilio Vox Fallback", e);
+        if (nexmoSender.isPresent()) {
+          nexmoSender.get().deliverVoxVerification(destination, verificationCode);
+        }
+      }
+    }
+  }
+
+  private boolean isTwilioDestination(String number) {
+    return isTwilioInternational || number.length() == 12 && number.startsWith("+1");
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/sms/TwilioSmsSender.java

@@ -16,32 +16,36 @@
  */
 package org.whispersystems.textsecuregcm.sms;
 
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import com.twilio.sdk.TwilioRestClient;
 import com.twilio.sdk.TwilioRestException;
 import com.twilio.sdk.resource.factory.CallFactory;
 import com.twilio.sdk.resource.factory.MessageFactory;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Meter;
 import org.apache.http.NameValuePair;
 import org.apache.http.message.BasicNameValuePair;
 import org.whispersystems.textsecuregcm.configuration.TwilioConfiguration;
+import org.whispersystems.textsecuregcm.util.Constants;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 
-public class TwilioSmsSender implements SenderFactory.SmsSender, SenderFactory.VoxSender {
+import static com.codahale.metrics.MetricRegistry.name;
+
+public class TwilioSmsSender {
 
   public static final String SAY_TWIML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
                                          "<Response>\n" +
-                                         "    <Say voice=\"woman\" language=\"en\">%s</Say>\n" +
+                                         "    <Say voice=\"woman\" language=\"en\">" + SmsSender.VOX_VERIFICATION_TEXT + "%s</Say>\n" +
                                          "</Response>";
 
-  private final Meter smsMeter = Metrics.newMeter(TwilioSmsSender.class, "sms", "delivered", TimeUnit.MINUTES);
-  private final Meter voxMeter = Metrics.newMeter(TwilioSmsSender.class, "vox", "delivered", TimeUnit.MINUTES);
+  private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+  private final Meter          smsMeter       = metricRegistry.meter(name(getClass(), "sms", "delivered"));
+  private final Meter          voxMeter       = metricRegistry.meter(name(getClass(), "vox", "delivered"));
 
   private final String accountId;
   private final String accountToken;
@@ -55,37 +59,39 @@ public class TwilioSmsSender implements SenderFactory.SmsSender, SenderFactory.V
     this.localDomain  = config.getLocalDomain();
   }
 
-  @Override
   public void deliverSmsVerification(String destination, String verificationCode)
-      throws IOException
+      throws IOException, TwilioRestException
   {
+    TwilioRestClient    client         = new TwilioRestClient(accountId, accountToken);
+    MessageFactory      messageFactory = client.getAccount().getMessageFactory();
+    List<NameValuePair> messageParams  = new LinkedList<>();
+    messageParams.add(new BasicNameValuePair("To", destination));
+    messageParams.add(new BasicNameValuePair("From", number));
+    messageParams.add(new BasicNameValuePair("Body", SmsSender.SMS_VERIFICATION_TEXT + verificationCode));
+
     try {
-      TwilioRestClient    client         = new TwilioRestClient(accountId, accountToken);
-      MessageFactory      messageFactory = client.getAccount().getMessageFactory();
-      List<NameValuePair> messageParams  = new LinkedList<>();
-      messageParams.add(new BasicNameValuePair("To", destination));
-      messageParams.add(new BasicNameValuePair("From", number));
-      messageParams.add(new BasicNameValuePair("Body", SenderFactory.SmsSender.VERIFICATION_TEXT + verificationCode));
       messageFactory.create(messageParams);
-    } catch (TwilioRestException e) {
-      throw new IOException(e);
+    } catch (RuntimeException damnYouTwilio) {
+      throw new IOException(damnYouTwilio);
     }
 
     smsMeter.mark();
   }
 
-  @Override
-  public void deliverVoxVerification(String destination, String verificationCode) throws IOException {
+  public void deliverVoxVerification(String destination, String verificationCode)
+      throws IOException, TwilioRestException
+  {
+    TwilioRestClient    client      = new TwilioRestClient(accountId, accountToken);
+    CallFactory         callFactory = client.getAccount().getCallFactory();
+    Map<String, String> callParams  = new HashMap<>();
+    callParams.put("To", destination);
+    callParams.put("From", number);
+    callParams.put("Url", "https://" + localDomain + "/v1/accounts/voice/twiml/" + verificationCode);
+
     try {
-      TwilioRestClient    client      = new TwilioRestClient(accountId, accountToken);
-      CallFactory         callFactory = client.getAccount().getCallFactory();
-      Map<String, String> callParams  = new HashMap<>();
-      callParams.put("To", destination);
-      callParams.put("From", number);
-      callParams.put("Url", "https://" + localDomain + "/v1/accounts/voice/twiml/" + verificationCode);
       callFactory.create(callParams);
-    } catch (TwilioRestException e) {
-      throw new IOException(e);
+    } catch (RuntimeException damnYouTwilio) {
+      throw new IOException(damnYouTwilio);
     }
 
     voxMeter.mark();

src/main/java/org/whispersystems/textsecuregcm/storage/Account.java

@@ -17,53 +17,60 @@
 package org.whispersystems.textsecuregcm.storage;
 
 
-import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
 
 import java.io.Serializable;
+import java.util.LinkedList;
+import java.util.List;
 
 public class Account implements Serializable {
 
-  public static final int MEMCACHE_VERION = 1;
+  public static final int MEMCACHE_VERION = 3;
 
-  private long    id;
-  private String  number;
-  private String  hashedAuthenticationToken;
-  private String  salt;
-  private String  signalingKey;
-  private String  gcmRegistrationId;
-  private String  apnRegistrationId;
+  @JsonIgnore
+  private long id;
+
+  @JsonProperty
+  private String number;
+
+  @JsonProperty
   private boolean supportsSms;
 
+  @JsonProperty
+  private List<Device> devices = new LinkedList<>();
+
+  @JsonProperty
+  private String identityKey;
+
+  @JsonIgnore
+  private Optional<Device> authenticatedDevice;
+
   public Account() {}
 
-  public Account(long id, String number, String hashedAuthenticationToken, String salt,
-                 String signalingKey, String gcmRegistrationId, String apnRegistrationId,
-                 boolean supportsSms)
-  {
-    this.id                        = id;
-    this.number                    = number;
-    this.hashedAuthenticationToken = hashedAuthenticationToken;
-    this.salt                      = salt;
-    this.signalingKey              = signalingKey;
-    this.gcmRegistrationId         = gcmRegistrationId;
-    this.apnRegistrationId         = apnRegistrationId;
-    this.supportsSms               = supportsSms;
+  @VisibleForTesting
+  public Account(String number, boolean supportsSms, List<Device> devices) {
+    this.number      = number;
+    this.supportsSms = supportsSms;
+    this.devices     = devices;
   }
 
-  public String getApnRegistrationId() {
-    return apnRegistrationId;
+  public long getId() {
+    return id;
   }
 
-  public void setApnRegistrationId(String apnRegistrationId) {
-    this.apnRegistrationId = apnRegistrationId;
+  public void setId(long id) {
+    this.id = id;
   }
 
-  public String getGcmRegistrationId() {
-    return gcmRegistrationId;
+  public Optional<Device> getAuthenticatedDevice() {
+    return authenticatedDevice;
   }
 
-  public void setGcmRegistrationId(String gcmRegistrationId) {
-    this.gcmRegistrationId = gcmRegistrationId;
+  public void setAuthenticatedDevice(Device device) {
+    this.authenticatedDevice = Optional.of(device);
   }
 
   public void setNumber(String number) {
@@ -74,23 +81,6 @@ public class Account implements Serializable {
     return number;
   }
 
-  public void setAuthenticationCredentials(AuthenticationCredentials credentials) {
-    this.hashedAuthenticationToken = credentials.getHashedAuthenticationToken();
-    this.salt                      = credentials.getSalt();
-  }
-
-  public AuthenticationCredentials getAuthenticationCredentials() {
-    return new AuthenticationCredentials(hashedAuthenticationToken, salt);
-  }
-
-  public String getSignalingKey() {
-    return signalingKey;
-  }
-
-  public void setSignalingKey(String signalingKey) {
-    this.signalingKey = signalingKey;
-  }
-
   public boolean getSupportsSms() {
     return supportsSms;
   }
@@ -99,11 +89,63 @@ public class Account implements Serializable {
     this.supportsSms = supportsSms;
   }
 
-  public long getId() {
-    return id;
+  public void addDevice(Device device) {
+    this.devices.add(device);
   }
 
-  public void setId(long id) {
-    this.id = id;
+  public void setDevices(List<Device> devices) {
+    this.devices = devices;
+  }
+
+  public List<Device> getDevices() {
+    return devices;
+  }
+
+  public Optional<Device> getMasterDevice() {
+    return getDevice(Device.MASTER_ID);
+  }
+
+  public Optional<Device> getDevice(long deviceId) {
+    for (Device device : devices) {
+      if (device.getId() == deviceId) {
+        return Optional.of(device);
+      }
+    }
+
+    return Optional.absent();
+  }
+
+  public boolean isActive() {
+    return
+        getMasterDevice().isPresent() &&
+        getMasterDevice().get().isActive();
+  }
+
+  public long getNextDeviceId() {
+    long highestDevice = Device.MASTER_ID;
+
+    for (Device device : devices) {
+      if (device.getId() > highestDevice) {
+        highestDevice = device.getId();
+      }
+    }
+
+    return highestDevice + 1;
+  }
+
+  public boolean isRateLimited() {
+    return true;
+  }
+
+  public Optional<String> getRelay() {
+    return Optional.absent();
+  }
+
+  public void setIdentityKey(String identityKey) {
+    this.identityKey = identityKey;
+  }
+
+  public String getIdentityKey() {
+    return identityKey;
   }
 }

src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java

@@ -16,6 +16,10 @@
  */
 package org.whispersystems.textsecuregcm.storage;
 
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.skife.jdbi.v2.SQLStatement;
 import org.skife.jdbi.v2.StatementContext;
 import org.skife.jdbi.v2.TransactionIsolationLevel;
@@ -30,6 +34,7 @@ import org.skife.jdbi.v2.sqlobject.Transaction;
 import org.skife.jdbi.v2.sqlobject.customizers.Mapper;
 import org.skife.jdbi.v2.tweak.ResultSetMapper;
 
+import java.io.IOException;
 import java.lang.annotation.Annotation;
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
@@ -42,36 +47,32 @@ import java.util.List;
 
 public abstract class Accounts {
 
-  public static final String ID            = "id";
-  public static final String NUMBER        = "number";
-  public static final String AUTH_TOKEN    = "auth_token";
-  public static final String SALT          = "salt";
-  public static final String SIGNALING_KEY = "signaling_key";
-  public static final String GCM_ID        = "gcm_id";
-  public static final String APN_ID        = "apn_id";
-  public static final String SUPPORTS_SMS  = "supports_sms";
+  private static final String ID     = "id";
+  private static final String NUMBER = "number";
+  private static final String DATA   = "data";
 
-  @SqlUpdate("INSERT INTO accounts (" + NUMBER + ", " + AUTH_TOKEN + ", "  +
-                                   SALT + ", " + SIGNALING_KEY + ", " + GCM_ID + ", " +
-                                   APN_ID + ", " + SUPPORTS_SMS + ") " +
-             "VALUES (:number, :auth_token, :salt, :signaling_key, :gcm_id, :apn_id, :supports_sms)")
+  private static final ObjectMapper mapper = new ObjectMapper();
+
+  static {
+    mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
+    mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+  }
+
+  @SqlUpdate("INSERT INTO accounts (" + NUMBER + ", " + DATA + ") VALUES (:number, CAST(:data AS json))")
   @GetGeneratedKeys
-  abstract long createStep(@AccountBinder Account account);
+  abstract long insertStep(@AccountBinder Account account);
 
-  @SqlUpdate("DELETE FROM accounts WHERE number = :number")
-  abstract void removeStep(@Bind("number") String number);
+  @SqlUpdate("DELETE FROM accounts WHERE " + NUMBER + " = :number")
+  abstract void removeAccount(@Bind("number") String number);
 
-  @SqlUpdate("UPDATE accounts SET " + AUTH_TOKEN + " = :auth_token, " + SALT + " = :salt, " +
-             SIGNALING_KEY + " = :signaling_key, " + GCM_ID + " = :gcm_id, " +
-             APN_ID + " = :apn_id, " + SUPPORTS_SMS + " = :supports_sms " +
-             "WHERE " + NUMBER + " = :number")
+  @SqlUpdate("UPDATE accounts SET " + DATA + " = CAST(:data AS json) WHERE " + NUMBER + " = :number")
   abstract void update(@AccountBinder Account account);
 
   @Mapper(AccountMapper.class)
   @SqlQuery("SELECT * FROM accounts WHERE " + NUMBER + " = :number")
   abstract Account get(@Bind("number") String number);
 
-  @SqlQuery("SELECT COUNT(*) from accounts")
+  @SqlQuery("SELECT COUNT(DISTINCT " + NUMBER + ") from accounts")
   abstract long getCount();
 
   @Mapper(AccountMapper.class)
@@ -80,25 +81,27 @@ public abstract class Accounts {
 
   @Mapper(AccountMapper.class)
   @SqlQuery("SELECT * FROM accounts")
-  abstract Iterator<Account> getAll();
+  public abstract Iterator<Account> getAll();
 
-  @Transaction(TransactionIsolationLevel.REPEATABLE_READ)
+  @Transaction(TransactionIsolationLevel.SERIALIZABLE)
   public long create(Account account) {
-    removeStep(account.getNumber());
-    return createStep(account);
+    removeAccount(account.getNumber());
+    return insertStep(account);
   }
 
   public static class AccountMapper implements ResultSetMapper<Account> {
-
     @Override
     public Account map(int i, ResultSet resultSet, StatementContext statementContext)
         throws SQLException
     {
-      return new Account(resultSet.getLong(ID), resultSet.getString(NUMBER),
-                         resultSet.getString(AUTH_TOKEN), resultSet.getString(SALT),
-                         resultSet.getString(SIGNALING_KEY), resultSet.getString(GCM_ID),
-                         resultSet.getString(APN_ID),
-                         resultSet.getInt(SUPPORTS_SMS) == 1);
+      try {
+        Account account = mapper.readValue(resultSet.getString(DATA), Account.class);
+        account.setId(resultSet.getLong(ID));
+
+        return account;
+      } catch (IOException e) {
+        throw new SQLException(e);
+      }
     }
   }
 
@@ -115,15 +118,14 @@ public abstract class Accounts {
                            AccountBinder accountBinder,
                            Account account)
           {
-            sql.bind(ID, account.getId());
-            sql.bind(NUMBER, account.getNumber());
-            sql.bind(AUTH_TOKEN, account.getAuthenticationCredentials()
-                                        .getHashedAuthenticationToken());
-            sql.bind(SALT, account.getAuthenticationCredentials().getSalt());
-            sql.bind(SIGNALING_KEY, account.getSignalingKey());
-            sql.bind(GCM_ID, account.getGcmRegistrationId());
-            sql.bind(APN_ID, account.getApnRegistrationId());
-            sql.bind(SUPPORTS_SMS, account.getSupportsSms() ? 1 : 0);
+            try {
+              String serialized = mapper.writeValueAsString(account);
+
+              sql.bind(NUMBER, account.getNumber());
+              sql.bind(DATA, serialized);
+            } catch (JsonProcessingException e) {
+              throw new IllegalArgumentException(e);
+            }
           }
         };
       }

src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java

@@ -53,9 +53,7 @@ public class AccountsManager {
   }
 
   public void create(Account account) {
-    long id = accounts.create(account);
-
-    account.setId(id);
+    accounts.create(account);
 
     if (memcachedClient != null) {
       memcachedClient.set(getKey(account.getNumber()), 0, account);
@@ -92,8 +90,15 @@ public class AccountsManager {
     else                 return Optional.absent();
   }
 
+  public boolean isRelayListed(String number) {
+    byte[]                  token   = Util.getContactToken(number);
+    Optional<ClientContact> contact = directory.get(token);
+
+    return contact.isPresent() && !Util.isEmpty(contact.get().getRelay());
+  }
+
   private void updateDirectory(Account account) {
-    if (account.getGcmRegistrationId() != null || account.getApnRegistrationId() != null) {
+    if (account.isActive()) {
       byte[]        token         = Util.getContactToken(account.getNumber());
       ClientContact clientContact = new ClientContact(token, null, account.getSupportsSms());
       directory.add(clientContact);
@@ -105,4 +110,5 @@ public class AccountsManager {
   private String getKey(String number) {
     return Account.class.getSimpleName() + Account.MEMCACHE_VERION + number;
   }
+
 }

src/main/java/org/whispersystems/textsecuregcm/storage/Device.java

@@ -0,0 +1,134 @@
+/**
+ * Copyright (C) 2013 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.storage;
+
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
+import org.whispersystems.textsecuregcm.util.Util;
+
+import java.io.Serializable;
+
+public class Device implements Serializable {
+
+  public static final long MASTER_ID = 1;
+
+  @JsonProperty
+  private long    id;
+
+  @JsonProperty
+  private String  authToken;
+
+  @JsonProperty
+  private String  salt;
+
+  @JsonProperty
+  private String  signalingKey;
+
+  @JsonProperty
+  private String  gcmId;
+
+  @JsonProperty
+  private String  apnId;
+
+  @JsonProperty
+  private boolean fetchesMessages;
+
+  @JsonProperty
+  private int registrationId;
+
+  public Device() {}
+
+  public Device(long id, String authToken, String salt,
+                String signalingKey, String gcmId, String apnId,
+                boolean fetchesMessages, int registrationId)
+  {
+    this.id              = id;
+    this.authToken       = authToken;
+    this.salt            = salt;
+    this.signalingKey    = signalingKey;
+    this.gcmId           = gcmId;
+    this.apnId           = apnId;
+    this.fetchesMessages = fetchesMessages;
+    this.registrationId  = registrationId;
+  }
+
+  public String getApnId() {
+    return apnId;
+  }
+
+  public void setApnId(String apnId) {
+    this.apnId = apnId;
+  }
+
+  public String getGcmId() {
+    return gcmId;
+  }
+
+  public void setGcmId(String gcmId) {
+    this.gcmId = gcmId;
+  }
+
+  public long getId() {
+    return id;
+  }
+
+  public void setId(long id) {
+    this.id = id;
+  }
+
+  public void setAuthenticationCredentials(AuthenticationCredentials credentials) {
+    this.authToken = credentials.getHashedAuthenticationToken();
+    this.salt      = credentials.getSalt();
+  }
+
+  public AuthenticationCredentials getAuthenticationCredentials() {
+    return new AuthenticationCredentials(authToken, salt);
+  }
+
+  public String getSignalingKey() {
+    return signalingKey;
+  }
+
+  public void setSignalingKey(String signalingKey) {
+    this.signalingKey = signalingKey;
+  }
+
+  public boolean isActive() {
+    return fetchesMessages || !Util.isEmpty(getApnId()) || !Util.isEmpty(getGcmId());
+  }
+
+  public boolean getFetchesMessages() {
+    return fetchesMessages;
+  }
+
+  public void setFetchesMessages(boolean fetchesMessages) {
+    this.fetchesMessages = fetchesMessages;
+  }
+
+  public boolean isMaster() {
+    return getId() == MASTER_ID;
+  }
+
+  public int getRegistrationId() {
+    return registrationId;
+  }
+
+  public void setRegistrationId(int registrationId) {
+    this.registrationId = registrationId;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryManager.java

@@ -21,6 +21,7 @@ import com.google.gson.Gson;
 import com.google.gson.annotations.SerializedName;
 import org.whispersystems.textsecuregcm.entities.ClientContact;
 import org.whispersystems.textsecuregcm.util.IterablePair;
+import org.whispersystems.textsecuregcm.util.Pair;
 import org.whispersystems.textsecuregcm.util.Util;
 
 import java.util.LinkedList;
@@ -110,7 +111,7 @@ public class DirectoryManager {
 
       IterablePair<byte[], Response<byte[]>> lists = new IterablePair<>(tokens, futures);
 
-      for (IterablePair.Pair<byte[], Response<byte[]>> pair : lists) {
+      for (Pair<byte[], Response<byte[]>> pair : lists) {
         if (pair.second().get() != null) {
           TokenValue    tokenValue    = new Gson().fromJson(new String(pair.second().get()), TokenValue.class);
           ClientContact clientContact = new ClientContact(pair.first(), tokenValue.relay, tokenValue.supportsSms);

src/main/java/org/whispersystems/textsecuregcm/storage/Keys.java

@@ -16,6 +16,7 @@
  */
 package org.whispersystems.textsecuregcm.storage;
 
+import com.google.common.base.Optional;
 import org.skife.jdbi.v2.SQLStatement;
 import org.skife.jdbi.v2.StatementContext;
 import org.skife.jdbi.v2.TransactionIsolationLevel;
@@ -30,6 +31,7 @@ import org.skife.jdbi.v2.sqlobject.Transaction;
 import org.skife.jdbi.v2.sqlobject.customizers.Mapper;
 import org.skife.jdbi.v2.tweak.ResultSetMapper;
 import org.whispersystems.textsecuregcm.entities.PreKey;
+import org.whispersystems.textsecuregcm.entities.UnstructuredPreKeyList;
 
 import java.lang.annotation.Annotation;
 import java.lang.annotation.ElementType;
@@ -42,44 +44,73 @@ import java.util.List;
 
 public abstract class Keys {
 
-  @SqlUpdate("DELETE FROM keys WHERE number = :number")
-  abstract void removeKeys(@Bind("number") String number);
+  @SqlUpdate("DELETE FROM keys WHERE number = :number AND device_id = :device_id")
+  abstract void removeKeys(@Bind("number") String number, @Bind("device_id") long deviceId);
 
   @SqlUpdate("DELETE FROM keys WHERE id = :id")
   abstract void removeKey(@Bind("id") long id);
 
-  @SqlBatch("INSERT INTO keys (number, key_id, public_key, identity_key, last_resort) VALUES (:number, :key_id, :public_key, :identity_key, :last_resort)")
+  @SqlBatch("INSERT INTO keys (number, device_id, key_id, public_key, last_resort) VALUES " +
+      "(:number, :device_id, :key_id, :public_key, :last_resort)")
   abstract void append(@PreKeyBinder List<PreKey> preKeys);
 
-  @SqlUpdate("INSERT INTO keys (number, key_id, public_key, identity_key, last_resort) VALUES (:number, :key_id, :public_key, :identity_key, :last_resort)")
+  @SqlUpdate("INSERT INTO keys (number, device_id, key_id, public_key, last_resort) VALUES " +
+      "(:number, :device_id, :key_id, :public_key, :last_resort)")
   abstract void append(@PreKeyBinder PreKey preKey);
 
-  @SqlQuery("SELECT * FROM keys WHERE number = :number ORDER BY id LIMIT 1 FOR UPDATE")
+  @SqlQuery("SELECT * FROM keys WHERE number = :number AND device_id = :device_id ORDER BY key_id ASC FOR UPDATE")
   @Mapper(PreKeyMapper.class)
-  abstract PreKey retrieveFirst(@Bind("number") String number);
+  abstract PreKey retrieveFirst(@Bind("number") String number, @Bind("device_id") long deviceId);
+
+  @SqlQuery("SELECT DISTINCT ON (number, device_id) * FROM keys WHERE number = :number ORDER BY number, device_id, key_id ASC")
+  @Mapper(PreKeyMapper.class)
+  abstract List<PreKey> retrieveFirst(@Bind("number") String number);
+
+  @SqlQuery("SELECT COUNT(*) FROM keys WHERE number = :number AND device_id = :device_id")
+  public abstract int getCount(@Bind("number") String number, @Bind("device_id") long deviceId);
 
   @Transaction(TransactionIsolationLevel.SERIALIZABLE)
-  public void store(String number, PreKey lastResortKey, List<PreKey> keys) {
+  public void store(String number, long deviceId, List<PreKey> keys, PreKey lastResortKey) {
     for (PreKey key : keys) {
       key.setNumber(number);
+      key.setDeviceId(deviceId);
     }
 
     lastResortKey.setNumber(number);
+    lastResortKey.setDeviceId(deviceId);
+    lastResortKey.setLastResort(true);
 
-    removeKeys(number);
+    removeKeys(number, deviceId);
     append(keys);
     append(lastResortKey);
   }
 
   @Transaction(TransactionIsolationLevel.SERIALIZABLE)
-  public PreKey get(String number) {
-    PreKey preKey = retrieveFirst(number);
+  public Optional<UnstructuredPreKeyList> get(String number, long deviceId) {
+    PreKey preKey = retrieveFirst(number, deviceId);
 
     if (preKey != null && !preKey.isLastResort()) {
       removeKey(preKey.getId());
     }
 
-    return preKey;
+    if (preKey != null) return Optional.of(new UnstructuredPreKeyList(preKey));
+    else                return Optional.absent();
+  }
+
+  @Transaction(TransactionIsolationLevel.SERIALIZABLE)
+  public Optional<UnstructuredPreKeyList> get(String number) {
+    List<PreKey> preKeys = retrieveFirst(number);
+
+    if (preKeys != null) {
+      for (PreKey preKey : preKeys) {
+        if (!preKey.isLastResort()) {
+          removeKey(preKey.getId());
+        }
+      }
+    }
+
+    if (preKeys != null) return Optional.of(new UnstructuredPreKeyList(preKeys));
+    else                 return Optional.absent();
   }
 
   @BindingAnnotation(PreKeyBinder.PreKeyBinderFactory.class)
@@ -95,9 +126,9 @@ public abstract class Keys {
           {
             sql.bind("id", preKey.getId());
             sql.bind("number", preKey.getNumber());
+            sql.bind("device_id", preKey.getDeviceId());
             sql.bind("key_id", preKey.getKeyId());
             sql.bind("public_key", preKey.getPublicKey());
-            sql.bind("identity_key", preKey.getIdentityKey());
             sql.bind("last_resort", preKey.isLastResort() ? 1 : 0);
           }
         };
@@ -111,9 +142,8 @@ public abstract class Keys {
     public PreKey map(int i, ResultSet resultSet, StatementContext statementContext)
         throws SQLException
     {
-      return new PreKey(resultSet.getLong("id"), resultSet.getString("number"),
+      return new PreKey(resultSet.getLong("id"), resultSet.getString("number"), resultSet.getLong("device_id"),
                          resultSet.getLong("key_id"), resultSet.getString("public_key"),
-                         resultSet.getString("identity_key"),
                          resultSet.getInt("last_resort") == 1);
     }
   }

src/main/java/org/whispersystems/textsecuregcm/storage/PendingAccounts.java

@@ -29,4 +29,6 @@ public interface PendingAccounts {
   @SqlQuery("SELECT verification_code FROM pending_accounts WHERE number = :number")
   String getCodeForNumber(@Bind("number") String number);
 
+  @SqlUpdate("DELETE FROM pending_accounts WHERE number = :number")
+  void remove(@Bind("number") String number);
 }

src/main/java/org/whispersystems/textsecuregcm/storage/PendingAccountsManager.java

@@ -41,6 +41,12 @@ public class PendingAccountsManager {
     pendingAccounts.insert(number, code);
   }
 
+  public void remove(String number) {
+    if (memcachedClient != null)
+      memcachedClient.delete(MEMCACHE_PREFIX + number);
+    pendingAccounts.remove(number);
+  }
+
   public Optional<String> getCodeForNumber(String number) {
     String code = null;
 

src/main/java/org/whispersystems/textsecuregcm/storage/PendingDevices.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.storage;
+
+import org.skife.jdbi.v2.sqlobject.Bind;
+import org.skife.jdbi.v2.sqlobject.SqlQuery;
+import org.skife.jdbi.v2.sqlobject.SqlUpdate;
+
+public interface PendingDevices {
+
+  @SqlUpdate("WITH upsert AS (UPDATE pending_devices SET verification_code = :verification_code WHERE number = :number RETURNING *) " +
+             "INSERT INTO pending_devices (number, verification_code) SELECT :number, :verification_code WHERE NOT EXISTS (SELECT * FROM upsert)")
+  void insert(@Bind("number") String number, @Bind("verification_code") String verificationCode);
+
+  @SqlQuery("SELECT verification_code FROM pending_devices WHERE number = :number")
+  String getCodeForNumber(@Bind("number") String number);
+
+  @SqlUpdate("DELETE FROM pending_devices WHERE number = :number")
+  void remove(@Bind("number") String number);
+}

src/main/java/org/whispersystems/textsecuregcm/storage/PendingDevicesManager.java

@@ -0,0 +1,70 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.storage;
+
+import com.google.common.base.Optional;
+import net.spy.memcached.MemcachedClient;
+
+public class PendingDevicesManager {
+
+  private static final String MEMCACHE_PREFIX = "pending_devices";
+
+  private final PendingDevices  pendingDevices;
+  private final MemcachedClient memcachedClient;
+
+  public PendingDevicesManager(PendingDevices pendingDevices,
+                               MemcachedClient memcachedClient)
+  {
+    this.pendingDevices  = pendingDevices;
+    this.memcachedClient = memcachedClient;
+  }
+
+  public void store(String number, String code) {
+    if (memcachedClient != null) {
+      memcachedClient.set(MEMCACHE_PREFIX + number, 0, code);
+    }
+
+    pendingDevices.insert(number, code);
+  }
+
+  public void remove(String number) {
+    if (memcachedClient != null) {
+      memcachedClient.delete(MEMCACHE_PREFIX + number);
+    }
+
+    pendingDevices.remove(number);
+  }
+
+  public Optional<String> getCodeForNumber(String number) {
+    String code = null;
+
+    if (memcachedClient != null) {
+      code = (String)memcachedClient.get(MEMCACHE_PREFIX + number);
+    }
+
+    if (code == null) {
+      code = pendingDevices.getCodeForNumber(number);
+
+      if (code != null && memcachedClient != null) {
+        memcachedClient.set(MEMCACHE_PREFIX + number, 0, code);
+      }
+    }
+
+    if (code != null) return Optional.of(code);
+    else              return Optional.absent();
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/storage/PubSubListener.java

@@ -0,0 +1,7 @@
+package org.whispersystems.textsecuregcm.storage;
+
+public interface PubSubListener {
+
+  public void onPubSubMessage(PubSubMessage outgoingMessage);
+
+}

src/main/java/org/whispersystems/textsecuregcm/storage/PubSubManager.java

@@ -0,0 +1,158 @@
+package org.whispersystems.textsecuregcm.storage;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.textsecuregcm.websocket.InvalidWebsocketAddressException;
+import org.whispersystems.textsecuregcm.websocket.WebsocketAddress;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPubSub;
+
+public class PubSubManager {
+
+  private final Logger                                logger       = LoggerFactory.getLogger(PubSubManager.class);
+  private final ObjectMapper                          mapper       = new ObjectMapper();
+  private final SubscriptionListener                  baseListener = new SubscriptionListener();
+  private final Map<WebsocketAddress, PubSubListener> listeners    = new HashMap<>();
+
+  private final JedisPool jedisPool;
+  private boolean subscribed = false;
+
+  public PubSubManager(final JedisPool jedisPool) {
+    this.jedisPool = jedisPool;
+    initializePubSubWorker();
+    waitForSubscription();
+  }
+
+  public synchronized void subscribe(WebsocketAddress address, PubSubListener listener) {
+    listeners.put(address, listener);
+    baseListener.subscribe(address.toString());
+  }
+
+  public synchronized void unsubscribe(WebsocketAddress address, PubSubListener listener) {
+    if (listeners.get(address) == listener) {
+      listeners.remove(address);
+      baseListener.unsubscribe(address.toString());
+    }
+  }
+
+  public synchronized boolean publish(WebsocketAddress address, PubSubMessage message) {
+    try {
+      String serialized = mapper.writeValueAsString(message);
+      Jedis  jedis      = null;
+
+      try {
+        jedis = jedisPool.getResource();
+        return jedis.publish(address.toString(), serialized) != 0;
+      } finally {
+        if (jedis != null)
+          jedisPool.returnResource(jedis);
+      }
+    } catch (JsonProcessingException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  private synchronized void waitForSubscription() {
+    try {
+      while (!subscribed) {
+        wait();
+      }
+    } catch (InterruptedException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  private void initializePubSubWorker() {
+    new Thread("PubSubListener") {
+      @Override
+      public void run() {
+        for (;;) {
+          Jedis jedis = null;
+          try {
+            jedis = jedisPool.getResource();
+            jedis.subscribe(baseListener, new WebsocketAddress(0, 0).toString());
+            logger.warn("**** Unsubscribed from holding channel!!! ******");
+          } finally {
+            if (jedis != null)
+              jedisPool.returnResource(jedis);
+          }
+        }
+      }
+    }.start();
+
+    new Thread("PubSubKeepAlive") {
+      @Override
+      public void run() {
+        for (;;) {
+          try {
+            Thread.sleep(20000);
+            publish(new WebsocketAddress(0, 0), new PubSubMessage(0, "foo"));
+          } catch (InterruptedException e) {
+            throw new AssertionError(e);
+          }
+        }
+      }
+    }.start();
+  }
+
+  private class SubscriptionListener extends JedisPubSub {
+
+    @Override
+    public void onMessage(String channel, String message) {
+      try {
+        WebsocketAddress address = new WebsocketAddress(channel);
+        PubSubListener   listener;
+
+        synchronized (PubSubManager.this) {
+          listener = listeners.get(address);
+        }
+
+        if (listener != null) {
+          listener.onPubSubMessage(mapper.readValue(message, PubSubMessage.class));
+        }
+      } catch (InvalidWebsocketAddressException e) {
+        logger.warn("Address", e);
+      } catch (IOException e) {
+        logger.warn("IOE", e);
+      }
+    }
+
+    @Override
+    public void onPMessage(String s, String s2, String s3) {
+      logger.warn("Received PMessage!");
+    }
+
+    @Override
+    public void onSubscribe(String channel, int count) {
+      try {
+        WebsocketAddress address = new WebsocketAddress(channel);
+
+        if (address.getAccountId() == 0 && address.getDeviceId() == 0) {
+          synchronized (PubSubManager.this) {
+            subscribed = true;
+            PubSubManager.this.notifyAll();
+          }
+        }
+      } catch (InvalidWebsocketAddressException e) {
+        logger.warn("Weird address", e);
+      }
+    }
+
+    @Override
+    public void onUnsubscribe(String s, int i) {}
+
+    @Override
+    public void onPUnsubscribe(String s, int i) {}
+
+    @Override
+    public void onPSubscribe(String s, int i) {}
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/storage/PubSubMessage.java

@@ -0,0 +1,32 @@
+package org.whispersystems.textsecuregcm.storage;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PubSubMessage {
+
+  public static final int TYPE_QUERY_DB = 1;
+  public static final int TYPE_DELIVER  = 2;
+
+  @JsonProperty
+  private int type;
+
+  @JsonProperty
+  private String contents;
+
+  public PubSubMessage() {}
+
+  public PubSubMessage(int type, String contents) {
+    this.type     = type;
+    this.contents = contents;
+  }
+
+  public int getType() {
+    return type;
+  }
+
+  public String getContents() {
+    return contents;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/storage/StoredMessages.java

@@ -0,0 +1,86 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.storage;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
+
+import org.whispersystems.textsecuregcm.util.Constants;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import static com.codahale.metrics.MetricRegistry.name;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+
+public class StoredMessages {
+
+  private final MetricRegistry metricRegistry     = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
+  private final Histogram      queueSizeHistogram = metricRegistry.histogram(name(getClass(), "queue_size"));
+
+  private static final String QUEUE_PREFIX = "msgs";
+
+  private final JedisPool jedisPool;
+
+  public StoredMessages(JedisPool jedisPool) {
+    this.jedisPool = jedisPool;
+  }
+
+  public void insert(long accountId, long deviceId, String message) {
+    Jedis jedis = null;
+
+    try {
+      jedis = jedisPool.getResource();
+
+      long queueSize = jedis.lpush(getKey(accountId, deviceId), message);
+      queueSizeHistogram.update(queueSize);
+
+      if (queueSize > 1000) {
+        jedis.ltrim(getKey(accountId, deviceId), 0, 999);
+      }
+    } finally {
+      if (jedis != null)
+        jedisPool.returnResource(jedis);
+    }
+  }
+
+  public List<String> getMessagesForDevice(long accountId, long deviceId) {
+    List<String> messages = new LinkedList<>();
+    Jedis        jedis    = null;
+
+    try {
+      jedis = jedisPool.getResource();
+      String message;
+
+      while ((message = jedis.rpop(QUEUE_PREFIX + accountId + ":" + deviceId)) != null) {
+        messages.add(message);
+      }
+
+      return messages;
+    } finally {
+      if (jedis != null)
+        jedisPool.returnResource(jedis);
+    }
+  }
+
+  private String getKey(long accountId, long deviceId) {
+    return QUEUE_PREFIX + ":" + accountId + ":" + deviceId;
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/util/Base64.java

@@ -1241,7 +1241,7 @@ public class Base64
      * @since 1.4
      */
     public static byte[] decode( String s ) throws java.io.IOException {
-        return decode( s, NO_OPTIONS );
+        return decode( s, DONT_GUNZIP );
     }
 
     

src/main/java/org/whispersystems/textsecuregcm/util/CORSHeaderFilter.java

@@ -0,0 +1,41 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.util;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+public class CORSHeaderFilter implements Filter {
+  @Override
+  public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
+    if (response instanceof HttpServletResponse) {
+      ((HttpServletResponse) response).addHeader("Access-Control-Allow-Origin", "*");
+      ((HttpServletResponse) response).addHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE");
+      ((HttpServletResponse) response).addHeader("Access-Control-Allow-Headers", "Authorization, Content-type");
+    }
+    chain.doFilter(request, response);
+  }
+
+  @Override public void init(FilterConfig filterConfig) throws ServletException { }
+  @Override public void destroy() { }
+}

src/main/java/org/whispersystems/textsecuregcm/util/Constants.java

@@ -0,0 +1,7 @@
+package org.whispersystems.textsecuregcm.util;
+
+public class Constants {
+
+  public static final String METRICS_NAME = "textsecure";
+
+}

src/main/java/org/whispersystems/textsecuregcm/util/IterablePair.java

@@ -19,7 +19,7 @@ package org.whispersystems.textsecuregcm.util;
 import java.util.Iterator;
 import java.util.List;
 
-public class IterablePair <T1, T2> implements Iterable<IterablePair.Pair<T1,T2>> {
+public class IterablePair <T1, T2> implements Iterable<Pair<T1,T2>> {
   private final List<T1> first;
   private final List<T2> second;
 
@@ -33,24 +33,6 @@ public class IterablePair <T1, T2> implements Iterable<IterablePair.Pair<T1,T2>>
     return new ParallelIterator<>( first.iterator(), second.iterator() );
   }
 
-  public static class Pair<T1, T2> {
-    private final T1 v1;
-    private final T2 v2;
-
-    Pair(T1 v1, T2 v2) {
-      this.v1 = v1;
-      this.v2 = v2;
-    }
-
-    public T1 first(){
-      return v1;
-    }
-
-    public T2 second(){
-      return v2;
-    }
-  }
-
   public static class ParallelIterator <T1, T2> implements Iterator<Pair<T1, T2>> {
 
     private final Iterator<T1> it1;

src/main/java/org/whispersystems/textsecuregcm/util/Pair.java

@@ -0,0 +1,47 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.util;
+
+import static com.google.common.base.Objects.equal;
+
+public class Pair<T1, T2> {
+  private final T1 v1;
+  private final T2 v2;
+
+  public Pair(T1 v1, T2 v2) {
+    this.v1 = v1;
+    this.v2 = v2;
+  }
+
+  public T1 first(){
+    return v1;
+  }
+
+  public T2 second(){
+    return v2;
+  }
+
+  public boolean equals(Object o) {
+    return o instanceof Pair &&
+        equal(((Pair) o).first(), first()) &&
+        equal(((Pair) o).second(), second());
+  }
+
+  public int hashCode() {
+    return first().hashCode() ^ second().hashCode();
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/util/VerificationCode.java

@@ -16,12 +16,24 @@
  */
 package org.whispersystems.textsecuregcm.util;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.google.common.annotations.VisibleForTesting;
+
 public class VerificationCode {
 
+  @JsonProperty
   private String verificationCode;
+  @JsonIgnore
   private String verificationCodeDisplay;
+  @JsonIgnore
   private String verificationCodeSpeech;
 
+  @VisibleForTesting VerificationCode() {}
+
   public VerificationCode(int verificationCode) {
     this.verificationCode        = verificationCode + "";
     this.verificationCodeDisplay = this.verificationCode.substring(0, 3) + "-" +
@@ -54,4 +66,11 @@ public class VerificationCode {
     return delimited;
   }
 
+  @VisibleForTesting public boolean equals(Object o) {
+    return o instanceof VerificationCode && verificationCode.equals(((VerificationCode) o).verificationCode);
+  }
+
+  public int hashCode() {
+    return Integer.parseInt(verificationCode);
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/websocket/InvalidWebsocketAddressException.java

@@ -0,0 +1,11 @@
+package org.whispersystems.textsecuregcm.websocket;
+
+public class InvalidWebsocketAddressException extends Exception {
+  public InvalidWebsocketAddressException(String serialized) {
+    super(serialized);
+  }
+
+  public InvalidWebsocketAddressException(Exception e) {
+    super(e);
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/websocket/WebsocketAddress.java

@@ -0,0 +1,57 @@
+package org.whispersystems.textsecuregcm.websocket;
+
+public class WebsocketAddress {
+
+  private final long accountId;
+  private final long deviceId;
+
+  public WebsocketAddress(String serialized) throws InvalidWebsocketAddressException {
+    try {
+      String[] parts = serialized.split(":");
+
+      if (parts == null || parts.length != 2) {
+        throw new InvalidWebsocketAddressException(serialized);
+      }
+
+      this.accountId = Long.parseLong(parts[0]);
+      this.deviceId  = Long.parseLong(parts[1]);
+    } catch (NumberFormatException e) {
+      throw new InvalidWebsocketAddressException(e);
+    }
+  }
+
+  public WebsocketAddress(long accountId, long deviceId) {
+    this.accountId = accountId;
+    this.deviceId  = deviceId;
+  }
+
+  public long getAccountId() {
+    return accountId;
+  }
+
+  public long getDeviceId() {
+    return deviceId;
+  }
+
+  public String toString() {
+    return accountId + ":" + deviceId;
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) return false;
+    if (!(other instanceof WebsocketAddress)) return false;
+
+    WebsocketAddress that = (WebsocketAddress)other;
+
+    return
+        this.accountId == that.accountId &&
+        this.deviceId == that.deviceId;
+  }
+
+  @Override
+  public int hashCode() {
+    return (int)accountId ^ (int)deviceId;
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/websocket/WebsocketControllerFactory.java

@@ -0,0 +1,47 @@
+package org.whispersystems.textsecuregcm.websocket;
+
+import org.eclipse.jetty.websocket.api.UpgradeRequest;
+import org.eclipse.jetty.websocket.api.UpgradeResponse;
+import org.eclipse.jetty.websocket.servlet.WebSocketCreator;
+import org.eclipse.jetty.websocket.servlet.WebSocketServlet;
+import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
+import org.whispersystems.textsecuregcm.controllers.WebsocketController;
+import org.whispersystems.textsecuregcm.push.PushSender;
+import org.whispersystems.textsecuregcm.push.WebsocketSender;
+import org.whispersystems.textsecuregcm.storage.PubSubManager;
+import org.whispersystems.textsecuregcm.storage.StoredMessages;
+
+
+public class WebsocketControllerFactory extends WebSocketServlet implements WebSocketCreator {
+
+  private final Logger logger = LoggerFactory.getLogger(WebsocketControllerFactory.class);
+
+  private final PushSender           pushSender;
+  private final StoredMessages       storedMessages;
+  private final PubSubManager        pubSubManager;
+  private final AccountAuthenticator accountAuthenticator;
+
+  public WebsocketControllerFactory(AccountAuthenticator accountAuthenticator,
+                                    PushSender           pushSender,
+                                    StoredMessages       storedMessages,
+                                    PubSubManager        pubSubManager)
+  {
+    this.accountAuthenticator = accountAuthenticator;
+    this.pushSender           = pushSender;
+    this.storedMessages       = storedMessages;
+    this.pubSubManager        = pubSubManager;
+  }
+
+  @Override
+  public void configure(WebSocketServletFactory factory) {
+    factory.setCreator(this);
+  }
+
+  @Override
+  public Object createWebSocket(UpgradeRequest upgradeRequest, UpgradeResponse upgradeResponse) {
+    return new WebsocketController(accountAuthenticator, pushSender, pubSubManager, storedMessages);
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/websocket/WebsocketMessage.java

@@ -0,0 +1,18 @@
+package org.whispersystems.textsecuregcm.websocket;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class WebsocketMessage {
+
+  @JsonProperty
+  private long id;
+
+  @JsonProperty
+  private String message;
+
+  public WebsocketMessage(long id, String message) {
+    this.id      = id;
+    this.message = message;
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/workers/DirectoryCommand.java

@@ -16,16 +16,11 @@
  */
 package org.whispersystems.textsecuregcm.workers;
 
-import com.yammer.dropwizard.cli.ConfiguredCommand;
-import com.yammer.dropwizard.config.Bootstrap;
-import com.yammer.dropwizard.db.DatabaseConfiguration;
-import com.yammer.dropwizard.jdbi.ImmutableListContainerFactory;
-import com.yammer.dropwizard.jdbi.ImmutableSetContainerFactory;
-import com.yammer.dropwizard.jdbi.OptionalContainerFactory;
-import com.yammer.dropwizard.jdbi.args.OptionalArgumentFactory;
 import net.sourceforge.argparse4j.inf.Namespace;
 import net.spy.memcached.MemcachedClient;
 import org.skife.jdbi.v2.DBI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
 import org.whispersystems.textsecuregcm.federation.FederatedClientManager;
 import org.whispersystems.textsecuregcm.providers.MemcachedClientFactory;
@@ -34,10 +29,19 @@ import org.whispersystems.textsecuregcm.storage.Accounts;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
 import org.whispersystems.textsecuregcm.storage.DirectoryManager;
 
+import io.dropwizard.cli.ConfiguredCommand;
+import io.dropwizard.db.DataSourceFactory;
+import io.dropwizard.jdbi.ImmutableListContainerFactory;
+import io.dropwizard.jdbi.ImmutableSetContainerFactory;
+import io.dropwizard.jdbi.OptionalContainerFactory;
+import io.dropwizard.jdbi.args.OptionalArgumentFactory;
+import io.dropwizard.setup.Bootstrap;
 import redis.clients.jedis.JedisPool;
 
 public class DirectoryCommand extends ConfiguredCommand<WhisperServerConfiguration> {
 
+  private final Logger logger = LoggerFactory.getLogger(DirectoryCommand.class);
+
   public DirectoryCommand() {
     super("directory", "Update directory from DB and peers.");
   }
@@ -49,8 +53,8 @@ public class DirectoryCommand extends ConfiguredCommand<WhisperServerConfigurati
       throws Exception
   {
     try {
-      DatabaseConfiguration dbConfig = config.getDatabaseConfiguration();
-      DBI                   dbi      = new DBI(dbConfig.getUrl(), dbConfig.getUser(), dbConfig.getPassword());
+      DataSourceFactory dbConfig = config.getDataSourceFactory();
+      DBI               dbi      = new DBI(dbConfig.getUrl(), dbConfig.getUser(), dbConfig.getPassword());
 
       dbi.registerArgumentFactory(new OptionalArgumentFactory(dbConfig.getDriverClass()));
       dbi.registerContainerFactory(new ImmutableListContainerFactory());
@@ -68,6 +72,9 @@ public class DirectoryCommand extends ConfiguredCommand<WhisperServerConfigurati
 
       update.updateFromLocalDatabase();
       update.updateFromPeers();
+    } catch (Exception ex) {
+      logger.warn("Directory Exception", ex);
+      throw new RuntimeException(ex);
     } finally {
       Thread.sleep(3000);
       System.exit(0);

src/main/java/org/whispersystems/textsecuregcm/workers/DirectoryUpdater.java

@@ -27,6 +27,7 @@ import org.whispersystems.textsecuregcm.storage.AccountsManager;
 import org.whispersystems.textsecuregcm.storage.DirectoryManager;
 import org.whispersystems.textsecuregcm.storage.DirectoryManager.BatchOperationHandle;
 import org.whispersystems.textsecuregcm.util.Base64;
+import org.whispersystems.textsecuregcm.util.Hex;
 import org.whispersystems.textsecuregcm.util.Util;
 
 import java.util.Iterator;
@@ -60,7 +61,8 @@ public class DirectoryUpdater {
 
       while (accounts.hasNext()) {
         Account account = accounts.next();
-        if (account.getApnRegistrationId() != null || account.getGcmRegistrationId() != null) {
+
+        if (account.isActive()) {
           byte[]        token         = Util.getContactToken(account.getNumber());
           ClientContact clientContact = new ClientContact(token, null, account.getSupportsSms());
 
@@ -84,7 +86,7 @@ public class DirectoryUpdater {
 
     for (FederatedClient client : clients) {
       logger.info("Updating directory from peer: " + client.getPeerName());
-      BatchOperationHandle handle = directory.startBatchOperation();
+//      BatchOperationHandle handle = directory.startBatchOperation();
 
       try {
         int userCount = client.getUserCount();
@@ -93,31 +95,38 @@ public class DirectoryUpdater {
         logger.info("Remote peer user count: " + userCount);
 
         while (retrieved < userCount) {
+          logger.info("Retrieving remote tokens...");
           List<ClientContact> clientContacts = client.getUserTokens(retrieved);
 
-          if (clientContacts == null)
+          if (clientContacts == null) {
+            logger.info("Remote tokens empty, ending...");
             break;
+          } else {
+            logger.info("Retrieved " + clientContacts.size() + " remote tokens...");
+          }
 
           for (ClientContact clientContact : clientContacts) {
             clientContact.setRelay(client.getPeerName());
 
             Optional<ClientContact> existing = directory.get(clientContact.getToken());
 
-            if (!clientContact.isInactive() && (!existing.isPresent() || existing.get().getRelay().equals(client.getPeerName()))) {
-              directory.add(handle, clientContact);
+            if (!clientContact.isInactive() && (!existing.isPresent() || client.getPeerName().equals(existing.get().getRelay()))) {
+//              directory.add(handle, clientContact);
+              directory.add(clientContact);
             } else {
-              if (existing != null && client.getPeerName().equals(existing.get().getRelay())) {
+              if (existing.isPresent() && client.getPeerName().equals(existing.get().getRelay())) {
                 directory.remove(clientContact.getToken());
               }
             }
           }
 
           retrieved += clientContacts.size();
+          logger.info("Processed: " + retrieved + " remote tokens.");
         }
 
         logger.info("Update from peer complete.");
       } finally {
-        directory.stopBatchOperation(handle);
+//        directory.stopBatchOperation(handle);
       }
     }
 

src/main/resources/migrations.xml

@@ -75,4 +75,100 @@
             <column name="number"/>
         </createIndex>
     </changeSet>
+
+    <changeSet id="2" author="matt">
+
+        <addColumn tableName="accounts">
+            <column name="data" type="json" />
+        </addColumn>
+
+        <sql>CREATE type device_t AS (id integer, "authToken" varchar(255), salt varchar(255), "signalingKey" varchar(255), "gcmId" text, "apnId" text);</sql>
+        <sql>CREATE type account_t AS (number varchar(255), "supportsSms" smallint, devices device_t array);</sql>
+        <sql>UPDATE accounts SET data = row_to_json(row(number, supports_sms, array[row(1, auth_token, salt, signaling_key, gcm_id, apn_id)::device_t])::account_t)</sql>
+
+        <addNotNullConstraint tableName="accounts" columnName="data"/>
+
+        <dropColumn tableName="accounts" columnName="auth_token"/>
+        <dropColumn tableName="accounts" columnName="salt"/>
+        <dropColumn tableName="accounts" columnName="signaling_key"/>
+        <dropColumn tableName="accounts" columnName="gcm_id"/>
+        <dropColumn tableName="accounts" columnName="apn_id"/>
+        <dropColumn tableName="accounts" columnName="supports_sms"/>
+
+        <sql>DROP type account_t;</sql>
+        <sql>DROP type device_t;</sql>
+
+        <addColumn tableName="keys">
+            <column name="device_id" type="bigint" defaultValue="1">
+                <constraints nullable="false" />
+            </column>
+        </addColumn>
+
+        <createTable tableName="pending_devices">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+
+            <column name="number" type="text">
+                <constraints unique="true" nullable="false"/>
+            </column>
+
+            <column name="verification_code" type="text">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+
+        <createTable tableName="messages">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+
+            <column name="account_id" type="bigint">
+                <constraints nullable="false"/>
+            </column>
+
+            <column name="device_id" type="bigint">
+                <constraints nullable="false"/>
+            </column>
+
+            <column name="encrypted_message" type="text">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+
+        <createIndex tableName="messages" indexName="messages_account_and_device">
+            <column name="account_id"/>
+            <column name="device_id"/>
+        </createIndex>
+
+    </changeSet>
+
+    <changeSet id="3" author="moxie">
+        <sql>CREATE OR REPLACE FUNCTION "custom_json_object_set_key"(
+            "json"          json,
+            "key_to_set"    TEXT,
+            "value_to_set"  anyelement
+            )
+            RETURNS json
+            LANGUAGE sql
+            IMMUTABLE
+            STRICT
+            AS $function$
+            SELECT COALESCE(
+            (SELECT ('{' || string_agg(to_json("key") || ':' || "value", ',') || '}')
+            FROM (SELECT *
+            FROM json_each("json")
+            WHERE "key" &lt;&gt; "key_to_set"
+            UNION ALL
+            SELECT "key_to_set", to_json("value_to_set")) AS "fields"),
+            '{}'
+            )::json
+            $function$;</sql>
+        <sql>UPDATE accounts SET data = custom_json_object_set_key(data, 'identityKey', k.identity_key) FROM keys k WHERE (data->>'identityKey')::text is null AND k.number = data->>'number' AND k.last_resort = 1;</sql>
+        <sql>UPDATE accounts SET data = custom_json_object_set_key(data, 'identityKey', k.identity_key) FROM keys k WHERE (data->>'identityKey')::text is null AND k.number = data->>'number';</sql>
+    </changeSet>
+
+    <changeSet id="4" author="moxie">
+        <dropColumn tableName="keys" columnName="identity_key"/>
+    </changeSet>
 </databaseChangeLog>

src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AccountControllerTest.java

@@ -2,14 +2,14 @@ package org.whispersystems.textsecuregcm.tests.controllers;
 
 import com.google.common.base.Optional;
 import com.sun.jersey.api.client.ClientResponse;
-import com.yammer.dropwizard.testing.ResourceTest;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.whispersystems.textsecuregcm.controllers.AccountController;
 import org.whispersystems.textsecuregcm.entities.AccountAttributes;
 import org.whispersystems.textsecuregcm.limits.RateLimiter;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
-import org.whispersystems.textsecuregcm.sms.SenderFactory;
-import org.whispersystems.textsecuregcm.sms.TwilioSmsSender;
+import org.whispersystems.textsecuregcm.sms.SmsSender;
 import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
 import org.whispersystems.textsecuregcm.storage.PendingAccountsManager;
@@ -17,11 +17,12 @@ import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
 
 import javax.ws.rs.core.MediaType;
 
+import io.dropwizard.testing.junit.ResourceTestRule;
 import static org.fest.assertions.api.Assertions.assertThat;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.*;
 
-public class AccountControllerTest extends ResourceTest {
+public class AccountControllerTest {
 
   private static final String SENDER = "+14152222222";
 
@@ -29,57 +30,58 @@ public class AccountControllerTest extends ResourceTest {
   private AccountsManager        accountsManager        = mock(AccountsManager.class       );
   private RateLimiters           rateLimiters           = mock(RateLimiters.class          );
   private RateLimiter            rateLimiter            = mock(RateLimiter.class           );
-  private TwilioSmsSender        twilioSmsSender        = mock(TwilioSmsSender.class       );
-  private SenderFactory          senderFactory          = mock(SenderFactory.class         );
+  private SmsSender              smsSender              = mock(SmsSender.class             );
 
-  @Override
-  protected void setUpResources() throws Exception {
-    addProvider(AuthHelper.getAuthenticator());
+  @Rule
+  public final ResourceTestRule resources = ResourceTestRule.builder()
+                                                            .addProvider(AuthHelper.getAuthenticator())
+                                                            .addResource(new AccountController(pendingAccountsManager,
+                                                                                               accountsManager,
+                                                                                               rateLimiters,
+                                                                                               smsSender))
+                                                            .build();
 
+
+  @Before
+  public void setup() throws Exception {
     when(rateLimiters.getSmsDestinationLimiter()).thenReturn(rateLimiter);
     when(rateLimiters.getVoiceDestinationLimiter()).thenReturn(rateLimiter);
     when(rateLimiters.getVerifyLimiter()).thenReturn(rateLimiter);
 
     when(pendingAccountsManager.getCodeForNumber(SENDER)).thenReturn(Optional.of("1234"));
-    when(senderFactory.getSmsSender(SENDER)).thenReturn(twilioSmsSender);
-
-    addResource(new AccountController(pendingAccountsManager,
-                                      accountsManager,
-                                      rateLimiters,
-                                      senderFactory));
   }
 
   @Test
   public void testSendCode() throws Exception {
     ClientResponse response =
-        client().resource(String.format("/v1/accounts/sms/code/%s", SENDER))
+        resources.client().resource(String.format("/v1/accounts/sms/code/%s", SENDER))
             .get(ClientResponse.class);
 
     assertThat(response.getStatus()).isEqualTo(200);
 
-    verify(twilioSmsSender).deliverSmsVerification(eq(SENDER), anyString());
+    verify(smsSender).deliverSmsVerification(eq(SENDER), anyString());
   }
 
   @Test
   public void testVerifyCode() throws Exception {
     ClientResponse response =
-        client().resource(String.format("/v1/accounts/code/%s", "1234"))
+        resources.client().resource(String.format("/v1/accounts/code/%s", "1234"))
             .header("Authorization", AuthHelper.getAuthHeader(SENDER, "bar"))
-            .entity(new AccountAttributes("keykeykeykey", false))
+            .entity(new AccountAttributes("keykeykeykey", false, false, 2222))
             .type(MediaType.APPLICATION_JSON_TYPE)
             .put(ClientResponse.class);
 
     assertThat(response.getStatus()).isEqualTo(204);
 
-    verify(accountsManager).create(isA(Account.class));
+    verify(accountsManager, times(1)).create(isA(Account.class));
   }
 
   @Test
   public void testVerifyBadCode() throws Exception {
     ClientResponse response =
-        client().resource(String.format("/v1/accounts/code/%s", "1111"))
+        resources.client().resource(String.format("/v1/accounts/code/%s", "1111"))
             .header("Authorization", AuthHelper.getAuthHeader(SENDER, "bar"))
-            .entity(new AccountAttributes("keykeykeykey", false))
+            .entity(new AccountAttributes("keykeykeykey", false, false, 3333))
             .type(MediaType.APPLICATION_JSON_TYPE)
             .put(ClientResponse.class);
 
@@ -88,4 +90,4 @@ public class AccountControllerTest extends ResourceTest {
     verifyNoMoreInteractions(accountsManager);
   }
 
-}
+}

src/test/java/org/whispersystems/textsecuregcm/tests/controllers/DeviceControllerTest.java

@@ -0,0 +1,101 @@
+/**
+ * Copyright (C) 2014 Open WhisperSystems
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.whispersystems.textsecuregcm.tests.controllers;
+
+import com.google.common.base.Optional;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.whispersystems.textsecuregcm.controllers.DeviceController;
+import org.whispersystems.textsecuregcm.entities.AccountAttributes;
+import org.whispersystems.textsecuregcm.entities.DeviceResponse;
+import org.whispersystems.textsecuregcm.limits.RateLimiter;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.PendingDevicesManager;
+import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
+import org.whispersystems.textsecuregcm.util.VerificationCode;
+
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+
+import io.dropwizard.testing.junit.ResourceTestRule;
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.Mockito.*;
+
+public class DeviceControllerTest {
+  @Path("/v1/devices")
+  static class DumbVerificationDeviceController extends DeviceController {
+    public DumbVerificationDeviceController(PendingDevicesManager pendingDevices, AccountsManager accounts, RateLimiters rateLimiters) {
+      super(pendingDevices, accounts, rateLimiters);
+    }
+
+    @Override
+    protected VerificationCode generateVerificationCode() {
+      return new VerificationCode(5678901);
+    }
+  }
+
+  private PendingDevicesManager pendingDevicesManager = mock(PendingDevicesManager.class);
+  private AccountsManager       accountsManager       = mock(AccountsManager.class       );
+  private RateLimiters          rateLimiters          = mock(RateLimiters.class          );
+  private RateLimiter           rateLimiter           = mock(RateLimiter.class           );
+  private Account               account               = mock(Account.class               );
+
+  @Rule
+  public final ResourceTestRule resources = ResourceTestRule.builder()
+                                                            .addProvider(AuthHelper.getAuthenticator())
+                                                            .addResource(new DumbVerificationDeviceController(pendingDevicesManager,
+                                                                                                              accountsManager,
+                                                                                                              rateLimiters))
+                                                            .build();
+
+
+  @Before
+  public void setup() throws Exception {
+    when(rateLimiters.getSmsDestinationLimiter()).thenReturn(rateLimiter);
+    when(rateLimiters.getVoiceDestinationLimiter()).thenReturn(rateLimiter);
+    when(rateLimiters.getVerifyLimiter()).thenReturn(rateLimiter);
+    when(rateLimiters.getAllocateDeviceLimiter()).thenReturn(rateLimiter);
+    when(rateLimiters.getVerifyDeviceLimiter()).thenReturn(rateLimiter);
+
+    when(account.getNextDeviceId()).thenReturn(42L);
+
+    when(pendingDevicesManager.getCodeForNumber(AuthHelper.VALID_NUMBER)).thenReturn(Optional.of("5678901"));
+    when(accountsManager.get(AuthHelper.VALID_NUMBER)).thenReturn(Optional.of(account));
+  }
+
+  @Test
+  public void validDeviceRegisterTest() throws Exception {
+    VerificationCode deviceCode = resources.client().resource("/v1/devices/provisioning_code")
+        .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+        .get(VerificationCode.class);
+
+    assertThat(deviceCode).isEqualTo(new VerificationCode(5678901));
+
+    DeviceResponse response = resources.client().resource("/v1/devices/5678901")
+        .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, "password1"))
+        .entity(new AccountAttributes("keykeykeykey", false, true, 1234))
+        .type(MediaType.APPLICATION_JSON_TYPE)
+        .put(DeviceResponse.class);
+
+    assertThat(response.getDeviceId()).isEqualTo(42L);
+
+    verify(pendingDevicesManager).remove(AuthHelper.VALID_NUMBER);
+  }
+}

src/test/java/org/whispersystems/textsecuregcm/tests/controllers/DirectoryControllerTest.java

@@ -0,0 +1,80 @@
+package org.whispersystems.textsecuregcm.tests.controllers;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.whispersystems.textsecuregcm.controllers.DirectoryController;
+import org.whispersystems.textsecuregcm.entities.ClientContactTokens;
+import org.whispersystems.textsecuregcm.limits.RateLimiter;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.storage.DirectoryManager;
+import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
+import org.whispersystems.textsecuregcm.util.Base64;
+
+import javax.ws.rs.core.MediaType;
+import java.util.LinkedList;
+import java.util.List;
+
+import io.dropwizard.testing.junit.ResourceTestRule;
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DirectoryControllerTest {
+
+  private final RateLimiters     rateLimiters     = mock(RateLimiters.class    );
+  private final RateLimiter      rateLimiter      = mock(RateLimiter.class     );
+  private final DirectoryManager directoryManager = mock(DirectoryManager.class);
+
+  @Rule
+  public final ResourceTestRule resources = ResourceTestRule.builder()
+                                                            .addProvider(AuthHelper.getAuthenticator())
+                                                            .addResource(new DirectoryController(rateLimiters,
+                                                                                                 directoryManager))
+                                                            .build();
+
+
+  @Before
+  public void setup() throws Exception {
+    when(rateLimiters.getContactsLimiter()).thenReturn(rateLimiter);
+    when(directoryManager.get(anyList())).thenAnswer(new Answer<List<byte[]>>() {
+      @Override
+      public List<byte[]> answer(InvocationOnMock invocationOnMock) throws Throwable {
+        List<byte[]> query = (List<byte[]>) invocationOnMock.getArguments()[0];
+        List<byte[]> response = new LinkedList<>(query);
+        response.remove(0);
+        return response;
+      }
+    });
+  }
+
+  @Test
+  public void testContactIntersection() throws Exception {
+    List<String> tokens = new LinkedList<String>() {{
+      add(Base64.encodeBytes("foo".getBytes()));
+      add(Base64.encodeBytes("bar".getBytes()));
+      add(Base64.encodeBytes("baz".getBytes()));
+    }};
+
+    List<String> expectedResponse = new LinkedList<>(tokens);
+    expectedResponse.remove(0);
+
+    ClientResponse response =
+        resources.client().resource("/v1/directory/tokens/")
+                 .entity(new ClientContactTokens(tokens))
+                 .type(MediaType.APPLICATION_JSON_TYPE)
+                 .header("Authorization",
+                         AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER,
+                                                  AuthHelper.VALID_PASSWORD))
+                 .put(ClientResponse.class);
+
+
+    assertThat(response.getStatus()).isEqualTo(200);
+    assertThat(response.getEntity(ClientContactTokens.class).getContacts()).isEqualTo(expectedResponse);
+  }
+}

src/test/java/org/whispersystems/textsecuregcm/tests/controllers/FederatedControllerTest.java

@@ -0,0 +1,96 @@
+package org.whispersystems.textsecuregcm.tests.controllers;
+
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Optional;
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.whispersystems.textsecuregcm.controllers.FederationController;
+import org.whispersystems.textsecuregcm.controllers.MessageController;
+import org.whispersystems.textsecuregcm.entities.IncomingMessageList;
+import org.whispersystems.textsecuregcm.entities.MessageProtos;
+import org.whispersystems.textsecuregcm.federation.FederatedClientManager;
+import org.whispersystems.textsecuregcm.limits.RateLimiter;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.push.PushSender;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
+
+import javax.ws.rs.core.MediaType;
+import java.util.LinkedList;
+import java.util.List;
+
+import io.dropwizard.testing.junit.ResourceTestRule;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+import static org.whispersystems.textsecuregcm.tests.util.JsonHelpers.jsonFixture;
+
+public class FederatedControllerTest {
+
+  private static final String SINGLE_DEVICE_RECIPIENT = "+14151111111";
+  private static final String MULTI_DEVICE_RECIPIENT  = "+14152222222";
+
+  private PushSender             pushSender             = mock(PushSender.class            );
+  private FederatedClientManager federatedClientManager = mock(FederatedClientManager.class);
+  private AccountsManager        accountsManager        = mock(AccountsManager.class       );
+  private RateLimiters           rateLimiters           = mock(RateLimiters.class          );
+  private RateLimiter            rateLimiter            = mock(RateLimiter.class           );
+
+  private final ObjectMapper mapper = new ObjectMapper();
+
+  private final MessageController messageController = new MessageController(rateLimiters, pushSender, accountsManager, federatedClientManager);
+
+  @Rule
+  public final ResourceTestRule resources = ResourceTestRule.builder()
+                                                            .addProvider(AuthHelper.getAuthenticator())
+                                                            .addResource(new FederationController(accountsManager,
+                                                                                                  null, null,
+                                                                                                  messageController))
+                                                            .build();
+
+
+
+  @Before
+  public void setup() throws Exception {
+    List<Device> singleDeviceList = new LinkedList<Device>() {{
+      add(new Device(1, "foo", "bar", "baz", "isgcm", null, false, 111));
+    }};
+
+    List<Device> multiDeviceList = new LinkedList<Device>() {{
+      add(new Device(1, "foo", "bar", "baz", "isgcm", null, false, 222));
+      add(new Device(2, "foo", "bar", "baz", "isgcm", null, false, 333));
+    }};
+
+    Account singleDeviceAccount = new Account(SINGLE_DEVICE_RECIPIENT, false, singleDeviceList);
+    Account multiDeviceAccount  = new Account(MULTI_DEVICE_RECIPIENT, false, multiDeviceList);
+
+    when(accountsManager.get(eq(SINGLE_DEVICE_RECIPIENT))).thenReturn(Optional.of(singleDeviceAccount));
+    when(accountsManager.get(eq(MULTI_DEVICE_RECIPIENT))).thenReturn(Optional.of(multiDeviceAccount));
+
+    when(rateLimiters.getMessagesLimiter()).thenReturn(rateLimiter);
+  }
+
+  @Test
+  public void testSingleDeviceCurrent() throws Exception {
+    ClientResponse response =
+        resources.client().resource(String.format("/v1/federation/messages/+14152223333/1/%s", SINGLE_DEVICE_RECIPIENT))
+            .header("Authorization", AuthHelper.getAuthHeader("cyanogen", "foofoo"))
+            .entity(mapper.readValue(jsonFixture("fixtures/current_message_single_device.json"), IncomingMessageList.class))
+            .type(MediaType.APPLICATION_JSON_TYPE)
+            .put(ClientResponse.class);
+
+    assertThat("Good Response", response.getStatus(), is(equalTo(204)));
+
+    verify(pushSender).sendMessage(any(Account.class), any(Device.class), any(MessageProtos.OutgoingMessageSignal.class));
+  }
+
+
+}

src/test/java/org/whispersystems/textsecuregcm/tests/controllers/KeyControllerTest.java

@@ -0,0 +1,234 @@
+package org.whispersystems.textsecuregcm.tests.controllers;
+
+import com.google.common.base.Optional;
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.whispersystems.textsecuregcm.controllers.KeysController;
+import org.whispersystems.textsecuregcm.entities.PreKey;
+import org.whispersystems.textsecuregcm.entities.PreKeyList;
+import org.whispersystems.textsecuregcm.entities.PreKeyStatus;
+import org.whispersystems.textsecuregcm.entities.UnstructuredPreKeyList;
+import org.whispersystems.textsecuregcm.limits.RateLimiter;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.storage.Keys;
+import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
+
+import javax.ws.rs.core.MediaType;
+import java.util.LinkedList;
+import java.util.List;
+
+import io.dropwizard.testing.junit.ResourceTestRule;
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.Mockito.*;
+
+public class KeyControllerTest {
+
+  private static final String EXISTS_NUMBER     = "+14152222222";
+  private static String NOT_EXISTS_NUMBER = "+14152222220";
+
+  private static int SAMPLE_REGISTRATION_ID  =  999;
+  private static int SAMPLE_REGISTRATION_ID2 = 1002;
+
+  private final PreKey          SAMPLE_KEY    = new PreKey(1, EXISTS_NUMBER, Device.MASTER_ID, 1234, "test1", "test2", false);
+  private final PreKey          SAMPLE_KEY2   = new PreKey(2, EXISTS_NUMBER, 2, 5667, "test3", "test4,", false               );
+  private final PreKey          SAMPLE_KEY3   = new PreKey(3, EXISTS_NUMBER, 3, 334, "test5", "test6", false                );
+  private final Keys            keys          = mock(Keys.class           );
+  private final AccountsManager accounts      = mock(AccountsManager.class);
+  private final Account         existsAccount = mock(Account.class        );
+
+  private RateLimiters          rateLimiters  = mock(RateLimiters.class);
+  private RateLimiter           rateLimiter   = mock(RateLimiter.class );
+
+  @Rule
+  public final ResourceTestRule resources = ResourceTestRule.builder()
+                                                            .addProvider(AuthHelper.getAuthenticator())
+                                                            .addResource(new KeysController(rateLimiters, keys, accounts, null))
+                                                            .build();
+
+  @Before
+  public void setup() {
+    Device  sampleDevice  = mock(Device.class );
+    Device  sampleDevice2 = mock(Device.class);
+    Device  sampleDevice3 = mock(Device.class);
+
+    when(sampleDevice.getRegistrationId()).thenReturn(SAMPLE_REGISTRATION_ID);
+    when(sampleDevice2.getRegistrationId()).thenReturn(SAMPLE_REGISTRATION_ID2);
+    when(sampleDevice3.getRegistrationId()).thenReturn(SAMPLE_REGISTRATION_ID2);
+    when(sampleDevice.isActive()).thenReturn(true);
+    when(sampleDevice2.isActive()).thenReturn(true);
+    when(sampleDevice3.isActive()).thenReturn(false);
+
+    when(existsAccount.getDevice(1L)).thenReturn(Optional.of(sampleDevice));
+    when(existsAccount.getDevice(2L)).thenReturn(Optional.of(sampleDevice2));
+    when(existsAccount.getDevice(3L)).thenReturn(Optional.of(sampleDevice3));
+    when(existsAccount.isActive()).thenReturn(true);
+    when(existsAccount.getIdentityKey()).thenReturn("existsidentitykey");
+
+    when(accounts.get(EXISTS_NUMBER)).thenReturn(Optional.of(existsAccount));
+    when(accounts.get(NOT_EXISTS_NUMBER)).thenReturn(Optional.<Account>absent());
+
+    when(rateLimiters.getPreKeysLimiter()).thenReturn(rateLimiter);
+
+    when(keys.get(eq(EXISTS_NUMBER), eq(1L))).thenAnswer(new Answer<Optional<UnstructuredPreKeyList>>() {
+      @Override
+      public Optional<UnstructuredPreKeyList> answer(InvocationOnMock invocationOnMock) throws Throwable {
+        return Optional.of(new UnstructuredPreKeyList(cloneKey(SAMPLE_KEY)));
+      }
+    });
+
+    when(keys.get(eq(NOT_EXISTS_NUMBER), eq(1L))).thenReturn(Optional.<UnstructuredPreKeyList>absent());
+
+    when(keys.get(EXISTS_NUMBER)).thenAnswer(new Answer<Optional<UnstructuredPreKeyList>>() {
+      @Override
+      public Optional<UnstructuredPreKeyList> answer(InvocationOnMock invocationOnMock) throws Throwable {
+        List<PreKey> allKeys = new LinkedList<>();
+        allKeys.add(cloneKey(SAMPLE_KEY));
+        allKeys.add(cloneKey(SAMPLE_KEY2));
+        allKeys.add(cloneKey(SAMPLE_KEY3));
+
+        return Optional.of(new UnstructuredPreKeyList(allKeys));
+      }
+    });
+
+    when(keys.getCount(eq(AuthHelper.VALID_NUMBER), eq(1L))).thenReturn(5);
+    when(AuthHelper.VALID_ACCOUNT.getIdentityKey()).thenReturn(null);
+  }
+
+  @Test
+  public void validKeyStatusTest() throws Exception {
+    PreKeyStatus result = resources.client().resource("/v1/keys")
+        .header("Authorization",
+                AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+        .get(PreKeyStatus.class);
+
+    assertThat(result.getCount() == 4);
+
+    verify(keys).getCount(eq(AuthHelper.VALID_NUMBER), eq(1L));
+  }
+
+  @Test
+  public void validLegacyRequestTest() throws Exception {
+    PreKey result = resources.client().resource(String.format("/v1/keys/%s", EXISTS_NUMBER))
+        .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+        .get(PreKey.class);
+
+    assertThat(result.getKeyId()).isEqualTo(SAMPLE_KEY.getKeyId());
+    assertThat(result.getPublicKey()).isEqualTo(SAMPLE_KEY.getPublicKey());
+    assertThat(result.getIdentityKey()).isEqualTo(existsAccount.getIdentityKey());
+
+    assertThat(result.getId() == 0);
+    assertThat(result.getNumber() == null);
+
+    verify(keys).get(eq(EXISTS_NUMBER), eq(1L));
+    verifyNoMoreInteractions(keys);
+  }
+
+  @Test
+  public void validMultiRequestTest() throws Exception {
+    UnstructuredPreKeyList results = resources.client().resource(String.format("/v1/keys/%s/*", EXISTS_NUMBER))
+        .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+        .get(UnstructuredPreKeyList.class);
+
+    assertThat(results.getKeys().size()).isEqualTo(2);
+
+    PreKey result = results.getKeys().get(0);
+
+    assertThat(result.getKeyId()).isEqualTo(SAMPLE_KEY.getKeyId());
+    assertThat(result.getPublicKey()).isEqualTo(SAMPLE_KEY.getPublicKey());
+    assertThat(result.getIdentityKey()).isEqualTo(existsAccount.getIdentityKey());
+    assertThat(result.getRegistrationId()).isEqualTo(SAMPLE_REGISTRATION_ID);
+
+    assertThat(result.getId() == 0);
+    assertThat(result.getNumber() == null);
+
+    result = results.getKeys().get(1);
+    assertThat(result.getKeyId()).isEqualTo(SAMPLE_KEY2.getKeyId());
+    assertThat(result.getPublicKey()).isEqualTo(SAMPLE_KEY2.getPublicKey());
+    assertThat(result.getIdentityKey()).isEqualTo(existsAccount.getIdentityKey());
+    assertThat(result.getRegistrationId()).isEqualTo(SAMPLE_REGISTRATION_ID2);
+
+    assertThat(result.getId() == 0);
+    assertThat(result.getNumber() == null);
+
+    verify(keys).get(eq(EXISTS_NUMBER));
+    verifyNoMoreInteractions(keys);
+  }
+
+
+  @Test
+  public void invalidRequestTest() throws Exception {
+    ClientResponse response = resources.client().resource(String.format("/v1/keys/%s", NOT_EXISTS_NUMBER))
+        .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+        .get(ClientResponse.class);
+
+    assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(404);
+  }
+
+  @Test
+  public void unauthorizedRequestTest() throws Exception {
+    ClientResponse response =
+        resources.client().resource(String.format("/v1/keys/%s", NOT_EXISTS_NUMBER))
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.INVALID_PASSWORD))
+            .get(ClientResponse.class);
+
+    assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(401);
+
+    response =
+        resources.client().resource(String.format("/v1/keys/%s", NOT_EXISTS_NUMBER))
+            .get(ClientResponse.class);
+
+    assertThat(response.getStatusInfo().getStatusCode()).isEqualTo(401);
+  }
+
+  @Test
+  public void putKeysTest() throws Exception {
+    final PreKey newKey        = new PreKey(0, null, 1L, 31337, "foobar", "foobarbaz", false);
+    final PreKey lastResortKey = new PreKey(0, null, 1L, 0xFFFFFF, "fooz", "foobarbaz", false);
+
+    List<PreKey> preKeys = new LinkedList<PreKey>() {{
+      add(newKey);
+    }};
+
+    PreKeyList preKeyList = new PreKeyList();
+    preKeyList.setKeys(preKeys);
+    preKeyList.setLastResortKey(lastResortKey);
+
+    ClientResponse response =
+        resources.client().resource("/v1/keys")
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+            .type(MediaType.APPLICATION_JSON_TYPE)
+            .put(ClientResponse.class, preKeyList);
+
+    assertThat(response.getClientResponseStatus().getStatusCode()).isEqualTo(204);
+
+    ArgumentCaptor<List>   listCaptor       = ArgumentCaptor.forClass(List.class  );
+    ArgumentCaptor<PreKey> lastResortCaptor = ArgumentCaptor.forClass(PreKey.class);
+    verify(keys).store(eq(AuthHelper.VALID_NUMBER), eq(1L), listCaptor.capture(), lastResortCaptor.capture());
+
+    List<PreKey> capturedList = listCaptor.getValue();
+    assertThat(capturedList.size() == 1);
+    assertThat(capturedList.get(0).getIdentityKey().equals("foobarbaz"));
+    assertThat(capturedList.get(0).getKeyId() == 31337);
+    assertThat(capturedList.get(0).getPublicKey().equals("foobar"));
+
+    assertThat(lastResortCaptor.getValue().getPublicKey().equals("fooz"));
+    assertThat(lastResortCaptor.getValue().getIdentityKey().equals("foobarbaz"));
+
+    verify(AuthHelper.VALID_ACCOUNT).setIdentityKey(eq("foobarbaz"));
+    verify(accounts).update(AuthHelper.VALID_ACCOUNT);
+  }
+
+  private PreKey cloneKey(PreKey source) {
+    return new PreKey(source.getId(), source.getNumber(), source.getDeviceId(), source.getKeyId(),
+                      source.getPublicKey(), source.getIdentityKey(), source.isLastResort());
+  }
+
+}

src/test/java/org/whispersystems/textsecuregcm/tests/controllers/MessageControllerTest.java

@@ -0,0 +1,176 @@
+package org.whispersystems.textsecuregcm.tests.controllers;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Optional;
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.whispersystems.textsecuregcm.controllers.MessageController;
+import org.whispersystems.textsecuregcm.entities.IncomingMessageList;
+import org.whispersystems.textsecuregcm.entities.MessageProtos;
+import org.whispersystems.textsecuregcm.entities.MismatchedDevices;
+import org.whispersystems.textsecuregcm.entities.StaleDevices;
+import org.whispersystems.textsecuregcm.federation.FederatedClientManager;
+import org.whispersystems.textsecuregcm.limits.RateLimiter;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.push.PushSender;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.storage.AccountsManager;
+import org.whispersystems.textsecuregcm.storage.Device;
+import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
+
+import javax.ws.rs.core.MediaType;
+import java.util.LinkedList;
+import java.util.List;
+
+import io.dropwizard.testing.junit.ResourceTestRule;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+import static org.whispersystems.textsecuregcm.tests.util.JsonHelpers.asJson;
+import static org.whispersystems.textsecuregcm.tests.util.JsonHelpers.jsonFixture;
+
+public class MessageControllerTest {
+
+  private static final String SINGLE_DEVICE_RECIPIENT = "+14151111111";
+  private static final String MULTI_DEVICE_RECIPIENT  = "+14152222222";
+
+  private  final PushSender             pushSender             = mock(PushSender.class            );
+  private  final FederatedClientManager federatedClientManager = mock(FederatedClientManager.class);
+  private  final AccountsManager        accountsManager        = mock(AccountsManager.class       );
+  private  final RateLimiters           rateLimiters           = mock(RateLimiters.class          );
+  private  final RateLimiter            rateLimiter            = mock(RateLimiter.class           );
+
+  private  final ObjectMapper mapper = new ObjectMapper();
+
+  @Rule
+  public final ResourceTestRule resources = ResourceTestRule.builder()
+                                                            .addProvider(AuthHelper.getAuthenticator())
+                                                            .addResource(new MessageController(rateLimiters, pushSender, accountsManager,
+                                                                                               federatedClientManager))
+                                                            .build();
+
+
+  @Before
+  public void setup() throws Exception {
+    List<Device> singleDeviceList = new LinkedList<Device>() {{
+      add(new Device(1, "foo", "bar", "baz", "isgcm", null, false, 111));
+    }};
+
+    List<Device> multiDeviceList = new LinkedList<Device>() {{
+      add(new Device(1, "foo", "bar", "baz", "isgcm", null, false, 222));
+      add(new Device(2, "foo", "bar", "baz", "isgcm", null, false, 333));
+    }};
+
+    Account singleDeviceAccount = new Account(SINGLE_DEVICE_RECIPIENT, false, singleDeviceList);
+    Account multiDeviceAccount  = new Account(MULTI_DEVICE_RECIPIENT, false, multiDeviceList);
+
+    when(accountsManager.get(eq(SINGLE_DEVICE_RECIPIENT))).thenReturn(Optional.of(singleDeviceAccount));
+    when(accountsManager.get(eq(MULTI_DEVICE_RECIPIENT))).thenReturn(Optional.of(multiDeviceAccount));
+
+    when(rateLimiters.getMessagesLimiter()).thenReturn(rateLimiter);
+  }
+
+  @Test
+  public synchronized void testSingleDeviceLegacy() throws Exception {
+    ClientResponse response =
+        resources.client().resource("/v1/messages/")
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+            .entity(mapper.readValue(jsonFixture("fixtures/legacy_message_single_device.json"), IncomingMessageList.class))
+            .type(MediaType.APPLICATION_JSON_TYPE)
+            .post(ClientResponse.class);
+
+    assertThat("Good Response", response.getStatus(), is(equalTo(200)));
+
+    verify(pushSender, times(1)).sendMessage(any(Account.class), any(Device.class), any(MessageProtos.OutgoingMessageSignal.class));
+  }
+
+  @Test
+  public synchronized void testSingleDeviceCurrent() throws Exception {
+    ClientResponse response =
+        resources.client().resource(String.format("/v1/messages/%s", SINGLE_DEVICE_RECIPIENT))
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+        .entity(mapper.readValue(jsonFixture("fixtures/current_message_single_device.json"), IncomingMessageList.class))
+        .type(MediaType.APPLICATION_JSON_TYPE)
+        .put(ClientResponse.class);
+
+    assertThat("Good Response", response.getStatus(), is(equalTo(204)));
+
+    verify(pushSender, times(1)).sendMessage(any(Account.class), any(Device.class), any(MessageProtos.OutgoingMessageSignal.class));
+  }
+
+  @Test
+  public synchronized void testMultiDeviceMissing() throws Exception {
+    ClientResponse response =
+        resources.client().resource(String.format("/v1/messages/%s", MULTI_DEVICE_RECIPIENT))
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+            .entity(mapper.readValue(jsonFixture("fixtures/current_message_single_device.json"), IncomingMessageList.class))
+            .type(MediaType.APPLICATION_JSON_TYPE)
+            .put(ClientResponse.class);
+
+    assertThat("Good Response Code", response.getStatus(), is(equalTo(409)));
+
+    assertThat("Good Response Body",
+               asJson(response.getEntity(MismatchedDevices.class)),
+               is(equalTo(jsonFixture("fixtures/missing_device_response.json"))));
+
+    verifyNoMoreInteractions(pushSender);
+  }
+
+  @Test
+  public synchronized void testMultiDeviceExtra() throws Exception {
+    ClientResponse response =
+        resources.client().resource(String.format("/v1/messages/%s", MULTI_DEVICE_RECIPIENT))
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+            .entity(mapper.readValue(jsonFixture("fixtures/current_message_extra_device.json"), IncomingMessageList.class))
+            .type(MediaType.APPLICATION_JSON_TYPE)
+            .put(ClientResponse.class);
+
+    assertThat("Good Response Code", response.getStatus(), is(equalTo(409)));
+
+    assertThat("Good Response Body",
+               asJson(response.getEntity(MismatchedDevices.class)),
+               is(equalTo(jsonFixture("fixtures/missing_device_response2.json"))));
+
+    verifyNoMoreInteractions(pushSender);
+  }
+
+  @Test
+  public synchronized void testMultiDevice() throws Exception {
+    ClientResponse response =
+        resources.client().resource(String.format("/v1/messages/%s", MULTI_DEVICE_RECIPIENT))
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+            .entity(mapper.readValue(jsonFixture("fixtures/current_message_multi_device.json"), IncomingMessageList.class))
+            .type(MediaType.APPLICATION_JSON_TYPE)
+            .put(ClientResponse.class);
+
+    assertThat("Good Response Code", response.getStatus(), is(equalTo(204)));
+
+    verify(pushSender, times(2)).sendMessage(any(Account.class), any(Device.class), any(MessageProtos.OutgoingMessageSignal.class));
+  }
+
+  @Test
+  public synchronized void testRegistrationIdMismatch() throws Exception {
+    ClientResponse response =
+        resources.client().resource(String.format("/v1/messages/%s", MULTI_DEVICE_RECIPIENT))
+            .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+            .entity(mapper.readValue(jsonFixture("fixtures/current_message_registration_id.json"), IncomingMessageList.class))
+            .type(MediaType.APPLICATION_JSON_TYPE)
+            .put(ClientResponse.class);
+
+    assertThat("Good Response Code", response.getStatus(), is(equalTo(410)));
+
+    assertThat("Good Response Body",
+               asJson(response.getEntity(StaleDevices.class)),
+               is(equalTo(jsonFixture("fixtures/mismatched_registration_id.json"))));
+
+    verifyNoMoreInteractions(pushSender);
+
+  }
+
+}