conditionally import psycopg2

changelog.d/12974.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in v1.60.0 that causes synapse to fail to start when python module sqlite3 is not installed.

synapse/replication/slave/storage/_base.py

@@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, Optional
 
 from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
 from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.util.id_generators import MultiWriterIdGenerator
 
 if TYPE_CHECKING:

synapse/storage/background_updates.py

@@ -628,7 +628,7 @@ class BackgroundUpdater:
                 logger.debug("[SQL] %s", sql)
                 c.execute(sql)
 
-        if isinstance(self.db_pool.engine, engines.PostgresEngine):
+        if isinstance(self.db_pool.engine, engines.postgres.PostgresEngine):
             runner: Optional[Callable[[Connection], None]] = create_index_psql
         elif psql_only:
             runner = None

synapse/storage/database.py

@@ -55,7 +55,9 @@ from synapse.logging.context import (
 from synapse.metrics import register_threadpool
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.background_updates import BackgroundUpdater
-from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.storage.types import Connection, Cursor
 from synapse.util.async_helpers import delay_cancellation
 from synapse.util.iterutils import batch_iter

synapse/storage/databases/main/__init__.py

@@ -24,7 +24,8 @@ from synapse.storage.database import (
     LoggingTransaction,
 )
 from synapse.storage.databases.main.stats import UserSortOrder
-from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.types import Cursor
 from synapse.storage.util.id_generators import MultiWriterIdGenerator, StreamIdGenerator
 from synapse.types import JsonDict, get_domain_from_id

synapse/storage/databases/main/account_data.py

@@ -37,7 +37,7 @@ from synapse.storage.database import (
 )
 from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
 from synapse.storage.databases.main.push_rule import PushRulesWorkerStore
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.util.id_generators import (
     AbstractStreamIdGenerator,
     AbstractStreamIdTracker,

synapse/storage/databases/main/cache.py

@@ -31,7 +31,7 @@ from synapse.storage.database import (
     LoggingDatabaseConnection,
     LoggingTransaction,
 )
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.util.caches.descriptors import _CachedFunction
 from synapse.util.iterutils import batch_iter
 

synapse/storage/databases/main/deviceinbox.py

@@ -36,7 +36,7 @@ from synapse.storage.database import (
     LoggingTransaction,
     make_in_list_sql_clause,
 )
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.util.id_generators import (
     AbstractStreamIdGenerator,
     MultiWriterIdGenerator,

synapse/storage/databases/main/end_to_end_keys.py

@@ -42,7 +42,7 @@ from synapse.storage.database import (
     make_in_list_sql_clause,
 )
 from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.util.id_generators import StreamIdGenerator
 from synapse.types import JsonDict
 from synapse.util import json_encoder

synapse/storage/databases/main/event_federation.py

@@ -42,7 +42,7 @@ from synapse.storage.database import (
 )
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
 from synapse.storage.databases.main.signatures import SignatureWorkerStore
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.types import JsonDict
 from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached

synapse/storage/databases/main/events_worker.py

@@ -67,7 +67,7 @@ from synapse.storage.database import (
     LoggingDatabaseConnection,
     LoggingTransaction,
 )
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.types import Cursor
 from synapse.storage.util.id_generators import (
     AbstractStreamIdTracker,

synapse/storage/databases/main/presence.py

@@ -23,7 +23,7 @@ from synapse.storage.database import (
     LoggingTransaction,
 )
 from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.types import Connection
 from synapse.storage.util.id_generators import (
     AbstractStreamIdGenerator,

synapse/storage/databases/main/push_rule.py

@@ -31,7 +31,8 @@ from synapse.storage.databases.main.events_worker import EventsWorkerStore
 from synapse.storage.databases.main.pusher import PusherWorkerStore
 from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
-from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException
 from synapse.storage.util.id_generators import (
     AbstractStreamIdGenerator,

synapse/storage/databases/main/receipts.py

@@ -35,7 +35,7 @@ from synapse.storage.database import (
     LoggingDatabaseConnection,
     LoggingTransaction,
 )
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.util.id_generators import (
     AbstractStreamIdTracker,
     MultiWriterIdGenerator,

synapse/storage/databases/main/relations.py

@@ -33,7 +33,7 @@ from synapse.events import EventBase
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause
 from synapse.storage.databases.main.stream import generate_pagination_where_clause
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.types import JsonDict, RoomStreamToken, StreamKeyType, StreamToken
 from synapse.util.caches.descriptors import cached, cachedList
 

synapse/storage/databases/main/roommember.py

@@ -45,7 +45,7 @@ from synapse.storage.database import (
 )
 from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
-from synapse.storage.engines import Sqlite3Engine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.storage.roommember import (
     GetRoomsForUserWithStreamOrdering,
     MemberSummary,

synapse/storage/databases/main/search.py

@@ -27,7 +27,9 @@ from synapse.storage.database import (
     LoggingTransaction,
 )
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
-from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.types import JsonDict
 
 if TYPE_CHECKING:

synapse/storage/databases/main/stream.py

@@ -64,7 +64,8 @@ from synapse.storage.database import (
     make_in_list_sql_clause,
 )
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
-from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.util.id_generators import MultiWriterIdGenerator
 from synapse.types import PersistedEventPosition, RoomStreamToken
 from synapse.util.caches.descriptors import cached

synapse/storage/databases/main/user_directory.py

@@ -42,7 +42,8 @@ from synapse.storage.database import (
 )
 from synapse.storage.databases.main.state import StateFilter
 from synapse.storage.databases.main.state_deltas import StateDeltasStore
-from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.types import (
     JsonDict,
     UserProfile,

synapse/storage/databases/state/bg_updates.py

@@ -21,7 +21,7 @@ from synapse.storage.database import (
     LoggingDatabaseConnection,
     LoggingTransaction,
 )
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.state import StateFilter
 from synapse.types import MutableStateMap, StateMap
 

synapse/storage/engines/__init__.py

@@ -14,17 +14,19 @@
 from typing import Any, Mapping
 
 from ._base import BaseDatabaseEngine, IncorrectDatabaseSetup
-from .postgres import PostgresEngine
-from .sqlite import Sqlite3Engine
 
 
 def create_engine(database_config: Mapping[str, Any]) -> BaseDatabaseEngine:
     name = database_config["name"]
 
     if name == "sqlite3":
+        from .sqlite import Sqlite3Engine
+
         return Sqlite3Engine(database_config)
 
     if name == "psycopg2":
+        from .postgres import PostgresEngine
+
         return PostgresEngine(database_config)
 
     raise RuntimeError("Unsupported database engine '%s'" % (name,))

synapse/storage/engines/postgres.py

@@ -23,7 +23,7 @@ from synapse.storage.engines._base import (
 from synapse.storage.types import Cursor
 
 if TYPE_CHECKING:
-    import psycopg2  # noqa: F401
+    import psycopg2
 
     from synapse.storage.database import LoggingDatabaseConnection
 

synapse/storage/schema/main/delta/25/fts.py

@@ -14,7 +14,8 @@
 import json
 import logging
 
-from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.storage.prepare_database import get_statements
 
 logger = logging.getLogger(__name__)

synapse/storage/schema/main/delta/31/search_update.py

@@ -14,7 +14,7 @@
 import json
 import logging
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.prepare_database import get_statements
 
 logger = logging.getLogger(__name__)

synapse/storage/schema/main/delta/34/cache_stream.py

@@ -14,7 +14,7 @@
 
 import logging
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.prepare_database import get_statements
 
 logger = logging.getLogger(__name__)

synapse/storage/schema/main/delta/34/received_txn_purge.py

@@ -14,7 +14,7 @@
 
 import logging
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 
 logger = logging.getLogger(__name__)
 

synapse/storage/schema/main/delta/37/remove_auth_idx.py

@@ -14,7 +14,7 @@
 
 import logging
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.prepare_database import get_statements
 
 logger = logging.getLogger(__name__)

synapse/storage/schema/main/delta/42/user_dir.py

@@ -14,7 +14,8 @@
 
 import logging
 
-from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.storage.prepare_database import get_statements
 
 logger = logging.getLogger(__name__)

synapse/storage/schema/main/delta/48/group_unique_indexes.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.prepare_database import get_statements
 
 FIX_INDEXES = """

synapse/storage/schema/main/delta/50/make_event_content_nullable.py

@@ -53,7 +53,7 @@ SQLite:
 
 import logging
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 
 logger = logging.getLogger(__name__)
 

synapse/storage/schema/main/delta/56/unique_user_filter_index.py

@@ -1,7 +1,7 @@
 import logging
 from io import StringIO
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.prepare_database import execute_statements_from_stream
 
 logger = logging.getLogger(__name__)

synapse/storage/schema/main/delta/58/06dlols_unique_idx.py

@@ -20,7 +20,8 @@ entries, and with a UNIQUE index.
 import logging
 from io import StringIO
 
-from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.prepare_database import execute_statements_from_stream
 from synapse.storage.types import Cursor
 

synapse/storage/schema/main/delta/58/11user_id_seq.py

@@ -19,7 +19,7 @@ Adds a postgres SEQUENCE for generating guest user IDs.
 from synapse.storage.databases.main.registration import (
     find_max_generated_user_id_localpart,
 )
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 
 
 def run_create(cur, database_engine, *args, **kwargs):

synapse/storage/schema/main/delta/61/03recreate_min_depth.py

@@ -16,7 +16,8 @@
 This migration handles the process of changing the type of `room_depth.min_depth` to
 a BIGINT.
 """
-from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.storage.types import Cursor
 
 

synapse/storage/schema/main/delta/68/05partial_state_rooms_triggers.py

@@ -18,7 +18,9 @@ This migration adds triggers to the partial_state_events tables to enforce uniqu
 
 Triggers cannot be expressed in .sql files, so we have to use a separate file.
 """
-from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 from synapse.storage.types import Cursor
 
 

synapse/storage/schema/main/delta/69/01as_txn_seq.py

@@ -17,7 +17,7 @@
 Adds a postgres SEQUENCE for generating application service transaction IDs.
 """
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 
 
 def run_create(cur, database_engine, *args, **kwargs):

synapse/storage/schema/state/delta/47/state_group_seq.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 
 
 def run_create(cur, database_engine, *args, **kwargs):

synapse/storage/util/sequence.py

@@ -16,11 +16,8 @@ import logging
 import threading
 from typing import TYPE_CHECKING, Callable, List, Optional
 
-from synapse.storage.engines import (
+from synapse.storage.engines import BaseDatabaseEngine, IncorrectDatabaseSetup
-    BaseDatabaseEngine,
+from synapse.storage.engines.postgres import PostgresEngine
-    IncorrectDatabaseSetup,
-    PostgresEngine,
-)
 from synapse.storage.types import Connection, Cursor
 
 if TYPE_CHECKING:

tests/server.py

@@ -67,7 +67,8 @@ from synapse.http.site import SynapseRequest
 from synapse.logging.context import ContextResourceUsage
 from synapse.server import HomeServer
 from synapse.storage import DataStore
-from synapse.storage.engines import PostgresEngine, create_engine
+from synapse.storage.engines import create_engine
+from synapse.storage.engines.postgres import PostgresEngine
 from synapse.types import JsonDict
 from synapse.util import Clock
 

tests/storage/test_room_search.py

@@ -16,7 +16,7 @@ import synapse.rest.admin
 from synapse.api.constants import EventTypes
 from synapse.api.errors import StoreError
 from synapse.rest.client import login, room
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.postgres import PostgresEngine
 
 from tests.unittest import HomeserverTestCase, skip_unless
 from tests.utils import USE_POSTGRES_FOR_TESTS