Basic compact event

synapse/crypto/event_signing.py

@@ -22,6 +22,8 @@ from canonicaljson import encode_canonical_json
 from unpaddedbase64 import encode_base64, decode_base64
 from signedjson.sign import sign_json
 
+from frozendict import frozendict
+
 import hashlib
 import logging
 
@@ -36,7 +38,7 @@ def check_event_content_hash(event, hash_algorithm=hashlib.sha256):
     # some malformed events lack a 'hashes'. Protect against it being missing
     # or a weird type by basically treating it the same as an unhashed event.
     hashes = event.get("hashes")
-    if not isinstance(hashes, dict):
+    if not (isinstance(hashes, dict) or isinstance(hashes, frozendict)):
         raise SynapseError(400, "Malformed 'hashes'", Codes.UNAUTHORIZED)
 
     if name not in hashes:

synapse/event_auth.py

@@ -83,6 +83,7 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True):
         # FIXME
         return True
 
+    logger.info("auth_events %r", auth_events)
     creation_event = auth_events.get((EventTypes.Create, ""), None)
 
     if not creation_event:

synapse/events/__init__.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.util.frozenutils import freeze
+from synapse.util.frozenutils import freeze, unfreeze
-from synapse.util.caches import intern_dict
+from synapse.util.caches import intern_dict, intern_string
+
+import ujson as json
 
 
 # Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents
@@ -24,11 +26,30 @@ USE_FROZEN_DICTS = True
 
 
 class _EventInternalMetadata(object):
+    __slots__ = [
+        "outlier",
+        "invite_from_remote",
+        "send_on_behalf_of",
+        "stream_ordering",
+        "token_id",
+        "txn_id",
+        "before",
+        "after",
+        "order",
+    ]
+
     def __init__(self, internal_metadata_dict):
-        self.__dict__ = dict(internal_metadata_dict)
+        # self.__dict__ = dict(internal_metadata_dict)
+        for key, value in internal_metadata_dict.iteritems():
+            setattr(self, key, value)
 
     def get_dict(self):
-        return dict(self.__dict__)
+        return {
+            key: getattr(self, key)
+            for key in self.__slots__
+            if hasattr(self, key)
+        }
+        # return dict(self.__dict__)
 
     def is_outlier(self):
         return getattr(self, "outlier", False)
@@ -144,8 +165,8 @@ class FrozenEvent(EventBase):
         # Signatures is a dict of dicts, and this is faster than doing a
         # copy.deepcopy
         signatures = {
-            name: {sig_id: sig for sig_id, sig in sigs.items()}
+            name: {sig_id: sig for sig_id, sig in sigs.iteritems()}
-            for name, sigs in event_dict.pop("signatures", {}).items()
+            for name, sigs in event_dict.pop("signatures", {}).iteritems()
         }
 
         unsigned = dict(event_dict.pop("unsigned", {}))
@@ -191,3 +212,207 @@ class FrozenEvent(EventBase):
             self.get("type", None),
             self.get("state_key", None),
         )
+
+
+def _compact_property(key):
+    def getter(self):
+        try:
+            return self[key]
+        except KeyError:
+            raise AttributeError(
+                "AttributeError: '%s' object has no attribute '%s'" % (
+                    self.__name__, key,
+                )
+            )
+
+    return property(getter)
+
+
+class _Unsigned(object):
+    __slots__ = [
+        "age_ts",
+        "replaces_state",
+        "redacted_because",
+        "invite_room_state",
+        "prev_content",
+        "prev_sender",
+        "redacted_by",
+    ]
+
+    def __init__(self, **kwargs):
+        for s in self.__slots__:
+            try:
+                setattr(self, s, kwargs[s])
+            except KeyError:
+                continue
+
+    def __getitem__(self, field):
+        try:
+            return getattr(self, field)
+        except AttributeError:
+            raise KeyError(field)
+
+    def __setitem__(self, field, value):
+        try:
+            setattr(self, field, value)
+        except AttributeError:
+            raise KeyError(field)
+
+    def __delitem__(self, field):
+        try:
+            return delattr(self, field)
+        except AttributeError:
+            raise KeyError(field)
+
+    def __contains__(self, field):
+        return hasattr(self, field)
+
+    def get(self, key, default=None):
+        return getattr(self, key, default)
+
+    def pop(self, key, default):
+        r = self.get(key, default)
+        try:
+            delattr(self, key)
+        except AttributeError:
+            pass
+        return r
+
+    def __iter__(self):
+        for key in self.__slots__:
+            if hasattr(self, key):
+                yield (key, getattr(self, key))
+
+
+class CompactEvent(object):
+    __slots__ = [
+        "event_json",
+
+        "internal_metadata",
+        "rejected_reason",
+
+        "signatures",
+        "unsigned",
+
+        "event_id",
+        "room_id",
+        "type",
+        "state_key",
+        "sender",
+    ]
+
+    def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
+        event_dict = dict(unfreeze(event_dict))
+
+        object.__setattr__(self, "unsigned", _Unsigned(**event_dict.pop("unsigned", {})))
+
+        signatures = {
+            intern_string(name): {
+                intern_string(sig_id): sig.encode("utf-8")
+                for sig_id, sig in sigs.iteritems()
+            }
+            for name, sigs in event_dict.pop("signatures", {}).iteritems()
+        }
+        object.__setattr__(self, "signatures", signatures)
+
+        object.__setattr__(self, "event_json", json.dumps(event_dict))
+
+        object.__setattr__(self, "rejected_reason", rejected_reason)
+        object.__setattr__(self, "internal_metadata", _EventInternalMetadata(
+            internal_metadata_dict
+        ))
+
+        object.__setattr__(self, "event_id", event_dict["event_id"])
+        object.__setattr__(self, "room_id", event_dict["room_id"])
+        object.__setattr__(self, "type", event_dict["type"])
+        if "state_key" in event_dict:
+            object.__setattr__(self, "state_key", event_dict["state_key"])
+        object.__setattr__(self, "sender", event_dict["sender"])
+
+    auth_events = _compact_property("auth_events")
+    depth = _compact_property("depth")
+    content = _compact_property("content")
+    hashes = _compact_property("hashes")
+    origin = _compact_property("origin")
+    origin_server_ts = _compact_property("origin_server_ts")
+    prev_events = _compact_property("prev_events")
+    prev_state = _compact_property("prev_state")
+    redacts = _compact_property("redacts")
+
+    @property
+    def user_id(self):
+        return self.sender
+
+    @property
+    def membership(self):
+        return self.content["membership"]
+
+    def is_state(self):
+        return hasattr(self, "state_key") and self.state_key is not None
+
+    def get_dict(self):
+        d = json.loads(self.event_json)
+        d.update({
+            "signatures": dict(self.signatures),
+            "unsigned": dict(self.unsigned),
+        })
+
+        return d
+
+    def get_pdu_json(self, time_now=None):
+        pdu_json = self.get_dict()
+
+        if time_now is not None and "age_ts" in pdu_json["unsigned"]:
+            age = time_now - pdu_json["unsigned"]["age_ts"]
+            pdu_json.setdefault("unsigned", {})["age"] = int(age)
+            del pdu_json["unsigned"]["age_ts"]
+
+        # This may be a frozen event
+        pdu_json["unsigned"].pop("redacted_because", None)
+
+        return pdu_json
+
+    def get(self, key, default=None):
+        if key in self.__slots__:
+            return freeze(getattr(self, key, default))
+
+        d = json.loads(self.event_json)
+        return d.get(key, default)
+
+    def get_internal_metadata_dict(self):
+        return self.internal_metadata.get_dict()
+
+    def __getitem__(self, field):
+        if field in self.__slots__:
+            try:
+                return freeze(getattr(self, field))
+            except AttributeError:
+                raise KeyError(field)
+
+        d = json.loads(self.event_json)
+        return d[field]
+
+    def __contains__(self, field):
+        if field in self.__slots__:
+            return hasattr(self, field)
+
+        d = json.loads(self.event_json)
+        return field in d
+
+    @staticmethod
+    def from_event(event):
+        return CompactEvent(
+            event.get_pdu_json(),
+            event.get_internal_metadata_dict(),
+            event.rejected_reason,
+        )
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __repr__(self):
+        return "<CompactEvent event_id='%s', type='%s', state_key='%s'>" % (
+            self.get("event_id", None),
+            self.get("type", None),
+            self.get("state_key", None),
+        )

synapse/events/utils.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 from synapse.api.constants import EventTypes
-from . import EventBase
+from . import EventBase, CompactEvent
 
 from frozendict import frozendict
 
@@ -242,7 +242,7 @@ def serialize_event(e, time_now_ms, as_client_event=True,
         dict
     """
     # FIXME(erikj): To handle the case of presence events and the like
-    if not isinstance(e, EventBase):
+    if not (isinstance(e, EventBase) or isinstance(e, CompactEvent)):
         return e
 
     time_now_ms = int(time_now_ms)

synapse/federation/units.py

@@ -121,9 +121,12 @@ class Transaction(JsonEncodedObject):
                 "Require 'transaction_id' to construct a Transaction"
             )
 
+        pdu_dicts = []
         for p in pdus:
-            p.transaction_id = kwargs["transaction_id"]
+            d = p.get_pdu_json()
+            # d["transaction_id"] = kwargs["transaction_id"]
+            pdu_dicts.append(d)
 
-        kwargs["pdus"] = [p.get_pdu_json() for p in pdus]
+        kwargs["pdus"] = pdu_dicts
 
         return Transaction(**kwargs)

synapse/storage/events.py

@@ -16,7 +16,7 @@ from ._base import SQLBaseStore
 
 from twisted.internet import defer, reactor
 
-from synapse.events import FrozenEvent, USE_FROZEN_DICTS
+from synapse.events import CompactEvent, FrozenEvent, USE_FROZEN_DICTS
 from synapse.events.utils import prune_event
 
 from synapse.util.async import ObservableDeferred
@@ -1310,7 +1310,7 @@ class EventsStore(SQLBaseStore):
                 event = ev_map[row["event_id"]]
                 if not row["rejects"] and not row["redacts"]:
                     to_prefill.append(_EventCacheEntry(
-                        event=event,
+                        event=CompactEvent.from_event(event),
                         redacted_event=None,
                     ))
 
@@ -1653,7 +1653,7 @@ class EventsStore(SQLBaseStore):
                     redacted_event.unsigned["redacted_because"] = because
 
             cache_entry = _EventCacheEntry(
-                event=original_ev,
+                event=CompactEvent.from_event(original_ev),
                 redacted_event=redacted_event,
             )