Make it simpler

synapse/server.py

@@ -24,10 +24,15 @@
 import abc
 import logging
 import os
+from functools import wraps
+from typing import Any, Callable, Dict, List, Optional, TypeVar, cast
 
+from twisted.internet import tcp
 from twisted.mail.smtp import sendmail
 from twisted.web.client import BrowserLikePolicyForHTTPS
+from twisted.web.iweb import IPolicyForHTTPS
 
+import synapse
 from synapse.api.auth import Auth
 from synapse.api.filtering import Filtering
 from synapse.api.ratelimiting import Ratelimiter
@@ -104,6 +109,43 @@ from synapse.util.distributor import Distributor
 logger = logging.getLogger(__name__)
 
 
+FuncType = Callable[..., Any]
+F = TypeVar("F", bound=FuncType)
+
+
+def builder(f: F) -> F:
+    """Decorator to wrap a HomeServer method to cache result and detect
+    cyclical dependencies.
+    """
+    if not f.__name__.startswith("get_"):
+        raise Exception("Function must be named `get_*`")
+
+    depname = f.__name__[len("get_") :]  # type: str
+
+    @wraps(f)
+    def _get(self):
+        try:
+            return getattr(self, depname)
+        except AttributeError:
+            pass
+
+        # Prevent cyclic dependencies from deadlocking
+        if depname in self._building:
+            raise ValueError("Cyclic dependency while building %s" % (depname,))
+
+        try:
+            self._building[depname] = True
+            dep = f(self)
+        finally:
+            self._building.pop(depname, None)
+
+        setattr(self, self.depname, dep)
+
+        return dep
+
+    return cast(F, _get)
+
+
 class HomeServer(object):
     """A basic homeserver object without lazy component builders.
 
@@ -111,17 +153,6 @@ class HomeServer(object):
     constructor arguments, or the relevant methods overriding to create them.
     Typically this would only be used for unit tests.
 
-    For every dependency in the DEPENDENCIES list below, this class creates one
-    method,
-        def get_DEPENDENCY(self)
-    which returns the value of that dependency. If no value has yet been set
-    nor was provided to the constructor, it will attempt to call a lazy builder
-    method called
-        def build_DEPENDENCY(self)
-    which must be implemented by the subclass. This code may call any of the
-    required "get" methods on the instance to obtain the sub-dependencies that
-    one requires.
-
     Attributes:
         config (synapse.config.homeserver.HomeserverConfig):
         _listening_services (list[twisted.internet.tcp.Port]): TCP ports that
@@ -130,77 +161,6 @@ class HomeServer(object):
 
     __metaclass__ = abc.ABCMeta
 
-    DEPENDENCIES = [
-        "http_client",
-        "federation_client",
-        "federation_server",
-        "handlers",
-        "auth",
-        "room_creation_handler",
-        "state_handler",
-        "state_resolution_handler",
-        "presence_handler",
-        "sync_handler",
-        "typing_handler",
-        "room_list_handler",
-        "acme_handler",
-        "auth_handler",
-        "device_handler",
-        "stats_handler",
-        "e2e_keys_handler",
-        "e2e_room_keys_handler",
-        "event_handler",
-        "event_stream_handler",
-        "initial_sync_handler",
-        "application_service_api",
-        "application_service_scheduler",
-        "application_service_handler",
-        "device_message_handler",
-        "profile_handler",
-        "event_creation_handler",
-        "deactivate_account_handler",
-        "set_password_handler",
-        "notifier",
-        "event_sources",
-        "keyring",
-        "pusherpool",
-        "event_builder_factory",
-        "filtering",
-        "http_client_context_factory",
-        "simple_http_client",
-        "proxied_http_client",
-        "media_repository",
-        "media_repository_resource",
-        "federation_transport_client",
-        "federation_sender",
-        "receipts_handler",
-        "macaroon_generator",
-        "tcp_replication",
-        "read_marker_handler",
-        "action_generator",
-        "user_directory_handler",
-        "groups_local_handler",
-        "groups_server_handler",
-        "groups_attestation_signing",
-        "groups_attestation_renewer",
-        "secrets",
-        "spam_checker",
-        "third_party_event_rules",
-        "room_member_handler",
-        "federation_registry",
-        "server_notices_manager",
-        "server_notices_sender",
-        "message_handler",
-        "pagination_handler",
-        "room_context_handler",
-        "sendmail",
-        "registration_handler",
-        "account_validity_handler",
-        "saml_handler",
-        "event_client_serializer",
-        "storage",
-    ]
-
     REQUIRED_ON_MASTER_STARTUP = ["user_directory_handler", "stats_handler"]
 
     # This is overridden in derived application classes
@@ -215,14 +175,16 @@ class HomeServer(object):
             config: The full config for the homeserver.
         """
         if not reactor:
-            from twisted.internet import reactor
+            from twisted import internet
+
+            reactor = internet.reactor
 
         self._reactor = reactor
         self.hostname = hostname
         self.config = config
-        self._building = {}
-        self._listening_services = []
-        self.start_time = None
+        self._building = {}  # type: Dict[str, bool]
+        self._listening_services = []  # type: List[tcp.Port]
+        self.start_time = None  # type: Optional[int]
 
         self.clock = Clock(reactor)
         self.distributor = Distributor()
@@ -230,7 +192,7 @@ class HomeServer(object):
         self.admin_redaction_ratelimiter = Ratelimiter()
         self.registration_ratelimiter = Ratelimiter()
 
-        self.datastores = None
+        self.datastores = None  # type: Optional[DataStores]
 
         # Other kwargs are explicit dependencies
         for depname in kwargs:
@@ -261,182 +223,231 @@ class HomeServer(object):
         # X-Forwarded-For is handled by our custom request type.
         return request.getClientIP()
 
-    def is_mine(self, domain_specific_string):
+    def is_mine(self, domain_specific_string) -> bool:
         return domain_specific_string.domain == self.hostname
 
-    def is_mine_id(self, string):
+    def is_mine_id(self, string: str) -> bool:
         return string.split(":", 1)[1] == self.hostname
 
-    def get_clock(self):
+    def get_clock(self) -> Clock:
         return self.clock
 
     def get_datastore(self):
+        if not self.datastores:
+            raise Exception("HomeServer has not been set up yet")
+
         return self.datastores.main
 
-    def get_datastores(self):
+    def get_datastores(self) -> DataStores:
+        if not self.datastores:
+            raise Exception("HomeServer has not been set up yet")
+
         return self.datastores
 
-    def get_config(self):
+    def get_config(self) -> HomeServerConfig:
         return self.config
 
-    def get_distributor(self):
+    def get_distributor(self) -> Distributor:
         return self.distributor
 
-    def get_ratelimiter(self):
+    def get_ratelimiter(self) -> Ratelimiter:
         return self.ratelimiter
 
-    def get_registration_ratelimiter(self):
+    def get_registration_ratelimiter(self) -> Ratelimiter:
         return self.registration_ratelimiter
 
-    def get_admin_redaction_ratelimiter(self):
+    def get_admin_redaction_ratelimiter(self) -> Ratelimiter:
         return self.admin_redaction_ratelimiter
 
-    def build_federation_client(self):
+    @builder
+    def get_federation_client(self) -> FederationClient:
         return FederationClient(self)
 
-    def build_federation_server(self):
+    @builder
+    def get_federation_server(self) -> FederationServer:
         return FederationServer(self)
 
-    def build_handlers(self):
+    @builder
+    def get_handlers(self) -> Handlers:
         return Handlers(self)
 
-    def build_notifier(self):
+    @builder
+    def get_notifier(self) -> Notifier:
         return Notifier(self)
 
-    def build_auth(self):
+    @builder
+    def get_auth(self) -> Auth:
         return Auth(self)
 
-    def build_http_client_context_factory(self):
+    @builder
+    def get_http_client_context_factory(self) -> IPolicyForHTTPS:
         return (
             InsecureInterceptableContextFactory()
             if self.config.use_insecure_ssl_client_just_for_testing_do_not_use
             else BrowserLikePolicyForHTTPS()
         )
 
-    def build_simple_http_client(self):
+    @builder
+    def get_simple_http_client(self) -> SimpleHttpClient:
         return SimpleHttpClient(self)
 
-    def build_proxied_http_client(self):
+    @builder
+    def get_proxied_http_client(self) -> SimpleHttpClient:
         return SimpleHttpClient(
             self,
             http_proxy=os.getenvb(b"http_proxy"),
             https_proxy=os.getenvb(b"HTTPS_PROXY"),
         )
 
-    def build_room_creation_handler(self):
+    @builder
+    def get_room_creation_handler(self) -> RoomCreationHandler:
         return RoomCreationHandler(self)
 
-    def build_sendmail(self):
+    @builder
+    def get_sendmail(self) -> sendmail:
         return sendmail
 
-    def build_state_handler(self):
+    @builder
+    def get_state_handler(self) -> StateHandler:
         return StateHandler(self)
 
-    def build_state_resolution_handler(self):
+    @builder
+    def get_state_resolution_handler(self) -> StateResolutionHandler:
         return StateResolutionHandler(self)
 
-    def build_presence_handler(self):
+    @builder
+    def get_presence_handler(self) -> PresenceHandler:
         return PresenceHandler(self)
 
-    def build_typing_handler(self):
+    @builder
+    def get_typing_handler(self) -> TypingHandler:
         return TypingHandler(self)
 
-    def build_sync_handler(self):
+    @builder
+    def get_sync_handler(self) -> SyncHandler:
         return SyncHandler(self)
 
-    def build_room_list_handler(self):
+    @builder
+    def get_room_list_handler(self) -> RoomListHandler:
         return RoomListHandler(self)
 
-    def build_auth_handler(self):
+    @builder
+    def get_auth_handler(self) -> AuthHandler:
         return AuthHandler(self)
 
-    def build_macaroon_generator(self):
+    @builder
+    def get_macaroon_generator(self) -> MacaroonGenerator:
         return MacaroonGenerator(self)
 
-    def build_device_handler(self):
+    @builder
+    def get_device_handler(self) -> DeviceWorkerHandler:
         if self.config.worker_app:
             return DeviceWorkerHandler(self)
         else:
             return DeviceHandler(self)
 
-    def build_device_message_handler(self):
+    @builder
+    def get_device_message_handler(self) -> DeviceMessageHandler:
         return DeviceMessageHandler(self)
 
-    def build_e2e_keys_handler(self):
+    @builder
+    def get_e2e_keys_handler(self) -> E2eKeysHandler:
         return E2eKeysHandler(self)
 
-    def build_e2e_room_keys_handler(self):
+    @builder
+    def get_e2e_room_keys_handler(self) -> E2eRoomKeysHandler:
         return E2eRoomKeysHandler(self)
 
-    def build_acme_handler(self):
+    @builder
+    def get_acme_handler(self) -> AcmeHandler:
         return AcmeHandler(self)
 
-    def build_application_service_api(self):
+    @builder
+    def get_application_service_api(self) -> ApplicationServiceApi:
         return ApplicationServiceApi(self)
 
-    def build_application_service_scheduler(self):
+    @builder
+    def get_application_service_scheduler(self) -> ApplicationServiceScheduler:
         return ApplicationServiceScheduler(self)
 
-    def build_application_service_handler(self):
+    @builder
+    def get_application_service_handler(self) -> ApplicationServicesHandler:
         return ApplicationServicesHandler(self)
 
-    def build_event_handler(self):
+    @builder
+    def get_event_handler(self) -> EventHandler:
         return EventHandler(self)
 
-    def build_event_stream_handler(self):
+    @builder
+    def get_event_stream_handler(self) -> EventStreamHandler:
         return EventStreamHandler(self)
 
-    def build_initial_sync_handler(self):
+    @builder
+    def get_initial_sync_handler(self) -> InitialSyncHandler:
         return InitialSyncHandler(self)
 
-    def build_profile_handler(self):
+    @builder
+    def get_profile_handler(self):
         if self.config.worker_app:
             return BaseProfileHandler(self)
         else:
             return MasterProfileHandler(self)
 
-    def build_event_creation_handler(self):
+    @builder
+    def get_event_creation_handler(self) -> EventCreationHandler:
         return EventCreationHandler(self)
 
-    def build_deactivate_account_handler(self):
+    @builder
+    def get_deactivate_account_handler(self) -> DeactivateAccountHandler:
         return DeactivateAccountHandler(self)
 
-    def build_set_password_handler(self):
+    @builder
+    def get_set_password_handler(self) -> SetPasswordHandler:
         return SetPasswordHandler(self)
 
-    def build_event_sources(self):
+    @builder
+    def get_event_sources(self) -> EventSources:
         return EventSources(self)
 
-    def build_keyring(self):
+    @builder
+    def get_keyring(self) -> Keyring:
         return Keyring(self)
 
-    def build_event_builder_factory(self):
+    @builder
+    def get_event_builder_factory(self) -> EventBuilderFactory:
         return EventBuilderFactory(self)
 
-    def build_filtering(self):
+    @builder
+    def get_filtering(self) -> Filtering:
         return Filtering(self)
 
-    def build_pusherpool(self):
+    @builder
+    def get_pusherpool(self) -> PusherPool:
         return PusherPool(self)
 
-    def build_http_client(self):
+    @builder
+    def get_http_client(self) -> MatrixFederationHttpClient:
         tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
             self.config
         )
         return MatrixFederationHttpClient(self, tls_client_options_factory)
 
-    def build_media_repository_resource(self):
+    @builder
+    def get_media_repository_resource(self) -> MediaRepositoryResource:
         # build the media repo resource. This indirects through the HomeServer
         # to ensure that we only have a single instance of
         return MediaRepositoryResource(self)
 
-    def build_media_repository(self):
+    @builder
+    def get_media_repository(self) -> MediaRepository:
         return MediaRepository(self)
 
-    def build_federation_transport_client(self):
+    @builder
+    def get_federation_transport_client(self) -> TransportLayerClient:
         return TransportLayerClient(self)
 
-    def build_federation_sender(self):
+    @builder
+    def get_federation_sender(self):
         if self.should_send_federation():
             return FederationSender(self)
         elif not self.config.worker_app:
@@ -444,135 +455,126 @@ class HomeServer(object):
         else:
             raise Exception("Workers cannot send federation traffic")
 
-    def build_receipts_handler(self):
+    @builder
+    def get_receipts_handler(self) -> ReceiptsHandler:
         return ReceiptsHandler(self)
 
-    def build_read_marker_handler(self):
+    @builder
+    def get_read_marker_handler(self) -> ReadMarkerHandler:
         return ReadMarkerHandler(self)
 
-    def build_tcp_replication(self):
+    @builder
+    def get_tcp_replication(self):
         raise NotImplementedError()
 
-    def build_action_generator(self):
+    @builder
+    def get_action_generator(self) -> ActionGenerator:
         return ActionGenerator(self)
 
-    def build_user_directory_handler(self):
+    @builder
+    def get_user_directory_handler(self) -> UserDirectoryHandler:
         return UserDirectoryHandler(self)
 
-    def build_groups_local_handler(self):
+    @builder
+    def get_groups_local_handler(self) -> GroupsLocalHandler:
         return GroupsLocalHandler(self)
 
-    def build_groups_server_handler(self):
+    @builder
+    def get_groups_server_handler(self) -> GroupsServerHandler:
         return GroupsServerHandler(self)
 
-    def build_groups_attestation_signing(self):
+    @builder
+    def get_groups_attestation_signing(self) -> GroupAttestationSigning:
         return GroupAttestationSigning(self)
 
-    def build_groups_attestation_renewer(self):
+    @builder
+    def get_groups_attestation_renewer(self) -> GroupAttestionRenewer:
         return GroupAttestionRenewer(self)
 
-    def build_secrets(self):
+    @builder
+    def get_secrets(self):
         return Secrets()
 
-    def build_stats_handler(self):
+    @builder
+    def get_stats_handler(self) -> StatsHandler:
         return StatsHandler(self)
 
-    def build_spam_checker(self):
+    @builder
+    def get_spam_checker(self) -> SpamChecker:
         return SpamChecker(self)
 
-    def build_third_party_event_rules(self):
+    @builder
+    def get_third_party_event_rules(self) -> ThirdPartyEventRules:
         return ThirdPartyEventRules(self)
 
-    def build_room_member_handler(self):
+    @builder
+    def get_room_member_handler(self):
         if self.config.worker_app:
             return RoomMemberWorkerHandler(self)
         return RoomMemberMasterHandler(self)
 
-    def build_federation_registry(self):
+    @builder
+    def get_federation_registry(self):
         if self.config.worker_app:
             return ReplicationFederationHandlerRegistry(self)
         else:
             return FederationHandlerRegistry()
 
-    def build_server_notices_manager(self):
+    @builder
+    def get_server_notices_manager(self) -> ServerNoticesManager:
         if self.config.worker_app:
             raise Exception("Workers cannot send server notices")
         return ServerNoticesManager(self)
 
-    def build_server_notices_sender(self):
+    @builder
+    def get_server_notices_sender(self):
         if self.config.worker_app:
             return WorkerServerNoticesSender(self)
         return ServerNoticesSender(self)
 
-    def build_message_handler(self):
+    @builder
+    def get_message_handler(self) -> MessageHandler:
         return MessageHandler(self)
 
-    def build_pagination_handler(self):
+    @builder
+    def get_pagination_handler(self) -> PaginationHandler:
         return PaginationHandler(self)
 
-    def build_room_context_handler(self):
+    @builder
+    def get_room_context_handler(self) -> RoomContextHandler:
         return RoomContextHandler(self)
 
-    def build_registration_handler(self):
+    @builder
+    def get_registration_handler(self) -> RegistrationHandler:
         return RegistrationHandler(self)
 
-    def build_account_validity_handler(self):
+    @builder
+    def get_account_validity_handler(self) -> AccountValidityHandler:
         return AccountValidityHandler(self)
 
-    def build_saml_handler(self):
+    @builder
+    def get_saml_handler(self) -> "synapse.handlers.saml_handler.SamlHandler":
         from synapse.handlers.saml_handler import SamlHandler
 
         return SamlHandler(self)
 
-    def build_event_client_serializer(self):
+    @builder
+    def get_event_client_serializer(self) -> EventClientSerializer:
         return EventClientSerializer(self)
 
-    def build_storage(self) -> Storage:
+    @builder
+    def get_storage(self) -> Storage:
+        if self.datastores is None:
+            raise Exception("HomeServer has not been set up yet")
+
         return Storage(self, self.datastores)
 
-    def remove_pusher(self, app_id, push_key, user_id):
+    def remove_pusher(self, app_id: str, push_key: str, user_id: str):
         return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
 
-    def should_send_federation(self):
+    def should_send_federation(self) -> bool:
         "Should this server be sending federation traffic directly?"
         return self.config.send_federation and (
             not self.config.worker_app
             or self.config.worker_app == "synapse.app.federation_sender"
         )
-
-
-def _make_dependency_method(depname):
-    def _get(hs):
-        try:
-            return getattr(hs, depname)
-        except AttributeError:
-            pass
-
-        try:
-            builder = getattr(hs, "build_%s" % (depname))
-        except AttributeError:
-            builder = None
-
-        if builder:
-            # Prevent cyclic dependencies from deadlocking
-            if depname in hs._building:
-                raise ValueError("Cyclic dependency while building %s" % (depname,))
-            hs._building[depname] = 1
-
-            dep = builder()
-            setattr(hs, depname, dep)
-
-            del hs._building[depname]
-
-            return dep
-
-        raise NotImplementedError(
-            "%s has no %s nor a builder for it" % (type(hs).__name__, depname)
-        )
-
-    setattr(HomeServer, "get_%s" % (depname), _get)
-
-
-# Build magic accessors for every dependency
-for depname in HomeServer.DEPENDENCIES:
-    _make_dependency_method(depname)

synapse/server.pyi

@@ -1,99 +0,0 @@
-import twisted.internet
-
-import synapse.api.auth
-import synapse.config.homeserver
-import synapse.federation.sender
-import synapse.federation.transaction_queue
-import synapse.federation.transport.client
-import synapse.handlers
-import synapse.handlers.auth
-import synapse.handlers.deactivate_account
-import synapse.handlers.device
-import synapse.handlers.e2e_keys
-import synapse.handlers.message
-import synapse.handlers.presence
-import synapse.handlers.room
-import synapse.handlers.room_member
-import synapse.handlers.set_password
-import synapse.http.client
-import synapse.notifier
-import synapse.rest.media.v1.media_repository
-import synapse.server_notices.server_notices_manager
-import synapse.server_notices.server_notices_sender
-import synapse.state
-import synapse.storage
-
-class HomeServer(object):
-    @property
-    def config(self) -> synapse.config.homeserver.HomeServerConfig:
-        pass
-    def get_auth(self) -> synapse.api.auth.Auth:
-        pass
-    def get_auth_handler(self) -> synapse.handlers.auth.AuthHandler:
-        pass
-    def get_datastore(self) -> synapse.storage.DataStore:
-        pass
-    def get_device_handler(self) -> synapse.handlers.device.DeviceHandler:
-        pass
-    def get_e2e_keys_handler(self) -> synapse.handlers.e2e_keys.E2eKeysHandler:
-        pass
-    def get_handlers(self) -> synapse.handlers.Handlers:
-        pass
-    def get_state_handler(self) -> synapse.state.StateHandler:
-        pass
-    def get_state_resolution_handler(self) -> synapse.state.StateResolutionHandler:
-        pass
-    def get_simple_http_client(self) -> synapse.http.client.SimpleHttpClient:
-        """Fetch an HTTP client implementation which doesn't do any blacklisting
-        or support any HTTP_PROXY settings"""
-        pass
-    def get_proxied_http_client(self) -> synapse.http.client.SimpleHttpClient:
-        """Fetch an HTTP client implementation which doesn't do any blacklisting
-        but does support HTTP_PROXY settings"""
-        pass
-    def get_deactivate_account_handler(
-        self,
-    ) -> synapse.handlers.deactivate_account.DeactivateAccountHandler:
-        pass
-    def get_room_creation_handler(self) -> synapse.handlers.room.RoomCreationHandler:
-        pass
-    def get_room_member_handler(self) -> synapse.handlers.room_member.RoomMemberHandler:
-        pass
-    def get_event_creation_handler(
-        self,
-    ) -> synapse.handlers.message.EventCreationHandler:
-        pass
-    def get_set_password_handler(
-        self,
-    ) -> synapse.handlers.set_password.SetPasswordHandler:
-        pass
-    def get_federation_sender(self) -> synapse.federation.sender.FederationSender:
-        pass
-    def get_federation_transport_client(
-        self,
-    ) -> synapse.federation.transport.client.TransportLayerClient:
-        pass
-    def get_media_repository_resource(
-        self,
-    ) -> synapse.rest.media.v1.media_repository.MediaRepositoryResource:
-        pass
-    def get_media_repository(
-        self,
-    ) -> synapse.rest.media.v1.media_repository.MediaRepository:
-        pass
-    def get_server_notices_manager(
-        self,
-    ) -> synapse.server_notices.server_notices_manager.ServerNoticesManager:
-        pass
-    def get_server_notices_sender(
-        self,
-    ) -> synapse.server_notices.server_notices_sender.ServerNoticesSender:
-        pass
-    def get_notifier(self) -> synapse.notifier.Notifier:
-        pass
-    def get_presence_handler(self) -> synapse.handlers.presence.PresenceHandler:
-        pass
-    def get_clock(self) -> synapse.util.Clock:
-        pass
-    def get_reactor(self) -> twisted.internet.base.ReactorBase:
-        pass