Skip to content

Commit

Permalink
Add Sliding Sync /sync endpoint (initial implementation) (element-h…
Browse files Browse the repository at this point in the history
…q#17187)

Based on [MSC3575](matrix-org/matrix-spec-proposals#3575): Sliding Sync

This iteration only focuses on returning the list of room IDs in the sliding window API (without sorting/filtering).

Rooms appear in the Sliding sync response based on:

 - `invite`, `join`, `knock`, `ban` membership events
 - Kicks (`leave` membership events where `sender` is different from the `user_id`/`state_key`)
 - `newly_left` (rooms that were left during the given token range, > `from_token` and <= `to_token`)
 - In order for bans/kicks to not show up, you need to `/forget` those rooms. This doesn't modify the event itself though and only adds the `forgotten` flag to `room_memberships` in Synapse. There isn't a way to tell when a room was forgotten at the moment so we can't factor it into the from/to range.

### Example request

`POST http://localhost:8008/_matrix/client/unstable/org.matrix.msc3575/sync`

```json
{
  "lists": {
    "foo-list": {
      "ranges": [ [0, 99] ],
      "sort": [ "by_notification_level", "by_recency", "by_name" ],
      "required_state": [
        ["m.room.join_rules", ""],
        ["m.room.history_visibility", ""],
        ["m.space.child", "*"]
      ],
      "timeline_limit": 100
    }
  }
}
```

Response:
```json
{
  "next_pos": "s58_224_0_13_10_1_1_16_0_1",
  "lists": {
    "foo-list": {
      "count": 1,
      "ops": [
        {
          "op": "SYNC",
          "range": [0, 99],
          "room_ids": [
            "!MmgikIyFzsuvtnbvVG:my.synapse.linux.server"
          ]
        }
      ]
    }
  },
  "rooms": {},
  "extensions": {}
}
```
  • Loading branch information
MadLittleMods authored and Mic92 committed Jun 14, 2024
1 parent f4229fd commit df0cf9e
Show file tree
Hide file tree
Showing 11 changed files with 2,302 additions and 15 deletions.
1 change: 1 addition & 0 deletions changelog.d/17187.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add initial implementation of an experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint.
2 changes: 1 addition & 1 deletion synapse/api/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ class Membership:
KNOCK: Final = "knock"
LEAVE: Final = "leave"
BAN: Final = "ban"
LIST: Final = (INVITE, JOIN, KNOCK, LEAVE, BAN)
LIST: Final = {INVITE, JOIN, KNOCK, LEAVE, BAN}


class PresenceState:
Expand Down
610 changes: 610 additions & 0 deletions synapse/handlers/sliding_sync.py

Large diffs are not rendered by default.

19 changes: 11 additions & 8 deletions synapse/handlers/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -2002,7 +2002,7 @@ async def get_sync_result_builder(
"""
user_id = sync_config.user.to_string()

# Note: we get the users room list *before* we get the current token, this
# Note: we get the users room list *before* we get the `now_token`, this
# avoids checking back in history if rooms are joined after the token is fetched.
token_before_rooms = self.event_sources.get_current_token()
mutable_joined_room_ids = set(await self.store.get_rooms_for_user(user_id))
Expand All @@ -2014,10 +2014,10 @@ async def get_sync_result_builder(
now_token = self.event_sources.get_current_token()
log_kv({"now_token": now_token})

# Since we fetched the users room list before the token, there's a small window
# during which membership events may have been persisted, so we fetch these now
# and modify the joined room list for any changes between the get_rooms_for_user
# call and the get_current_token call.
# Since we fetched the users room list before calculating the `now_token` (see
# above), there's a small window during which membership events may have been
# persisted, so we fetch these now and modify the joined room list for any
# changes between the get_rooms_for_user call and the get_current_token call.
membership_change_events = []
if since_token:
membership_change_events = await self.store.get_membership_changes_for_user(
Expand All @@ -2027,16 +2027,19 @@ async def get_sync_result_builder(
self.rooms_to_exclude_globally,
)

mem_last_change_by_room_id: Dict[str, EventBase] = {}
last_membership_change_by_room_id: Dict[str, EventBase] = {}
for event in membership_change_events:
mem_last_change_by_room_id[event.room_id] = event
last_membership_change_by_room_id[event.room_id] = event

# For the latest membership event in each room found, add/remove the room ID
# from the joined room list accordingly. In this case we only care if the
# latest change is JOIN.

for room_id, event in mem_last_change_by_room_id.items():
for room_id, event in last_membership_change_by_room_id.items():
assert event.internal_metadata.stream_ordering
# As a shortcut, skip any events that happened before we got our
# `get_rooms_for_user()` snapshot (any changes are already represented
# in that list).
if (
event.internal_metadata.stream_ordering
< token_before_rooms.room_key.stream
Expand Down
191 changes: 188 additions & 3 deletions synapse/rest/client/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,30 @@
# [This file includes modifications made by New Vector Limited]
#
#
from typing import TYPE_CHECKING, Dict, Optional
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

from synapse._pydantic_compat import HAS_PYDANTIC_V2

if TYPE_CHECKING or HAS_PYDANTIC_V2:
from pydantic.v1 import Extra, StrictInt, StrictStr, constr, validator
from pydantic.v1 import (
Extra,
StrictBool,
StrictInt,
StrictStr,
conint,
constr,
validator,
)
else:
from pydantic import Extra, StrictInt, StrictStr, constr, validator
from pydantic import (
Extra,
StrictBool,
StrictInt,
StrictStr,
conint,
constr,
validator,
)

from synapse.rest.models import RequestBodyModel
from synapse.util.threepids import validate_email
Expand Down Expand Up @@ -97,3 +113,172 @@ class EmailRequestTokenBody(ThreepidRequestTokenBody):
class MsisdnRequestTokenBody(ThreepidRequestTokenBody):
country: ISO3116_1_Alpha_2
phone_number: StrictStr


class SlidingSyncBody(RequestBodyModel):
"""
Sliding Sync API request body.
Attributes:
lists: Sliding window API. A map of list key to list information
(:class:`SlidingSyncList`). Max lists: 100. The list keys should be
arbitrary strings which the client is using to refer to the list. Keep this
small as it needs to be sent a lot. Max length: 64 bytes.
room_subscriptions: Room subscription API. A map of room ID to room subscription
information. Used to subscribe to a specific room. Sometimes clients know
exactly which room they want to get information about e.g by following a
permalink or by refreshing a webapp currently viewing a specific room. The
sliding window API alone is insufficient for this use case because there's
no way to say "please track this room explicitly".
extensions: Extensions API. A map of extension key to extension config.
"""

class CommonRoomParameters(RequestBodyModel):
"""
Common parameters shared between the sliding window and room subscription APIs.
Attributes:
required_state: Required state for each room returned. An array of event
type and state key tuples. Elements in this array are ORd together to
produce the final set of state events to return. One unique exception is
when you request all state events via `["*", "*"]`. When used, all state
events are returned by default, and additional entries FILTER OUT the
returned set of state events. These additional entries cannot use `*`
themselves. For example, `["*", "*"], ["m.room.member",
"@alice:example.com"]` will *exclude* every `m.room.member` event
*except* for `@alice:example.com`, and include every other state event.
In addition, `["*", "*"], ["m.space.child", "*"]` is an error, the
`m.space.child` filter is not required as it would have been returned
anyway.
timeline_limit: The maximum number of timeline events to return per response.
(Max 1000 messages)
include_old_rooms: Determines if `predecessor` rooms are included in the
`rooms` response. The user MUST be joined to old rooms for them to show up
in the response.
"""

class IncludeOldRooms(RequestBodyModel):
timeline_limit: StrictInt
required_state: List[Tuple[StrictStr, StrictStr]]

required_state: List[Tuple[StrictStr, StrictStr]]
# mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884
if TYPE_CHECKING:
timeline_limit: int
else:
timeline_limit: conint(le=1000, strict=True) # type: ignore[valid-type]
include_old_rooms: Optional[IncludeOldRooms] = None

class SlidingSyncList(CommonRoomParameters):
"""
Attributes:
ranges: Sliding window ranges. If this field is missing, no sliding window
is used and all rooms are returned in this list. Integers are
*inclusive*.
sort: How the list should be sorted on the server. The first value is
applied first, then tiebreaks are performed with each subsequent sort
listed.
FIXME: Furthermore, it's not currently defined how servers should behave
if they encounter a filter or sort operation they do not recognise. If
the server rejects the request with an HTTP 400 then that will break
backwards compatibility with new clients vs old servers. However, the
client would be otherwise unaware that only some of the sort/filter
operations have taken effect. We may need to include a "warnings"
section to indicate which sort/filter operations are unrecognised,
allowing for some form of graceful degradation of service.
-- https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/sync-v3/proposals/3575-sync.md#filter-and-sort-extensions
slow_get_all_rooms: Just get all rooms (for clients that don't want to deal with
sliding windows). When true, the `ranges` and `sort` fields are ignored.
required_state: Required state for each room returned. An array of event
type and state key tuples. Elements in this array are ORd together to
produce the final set of state events to return.
One unique exception is when you request all state events via `["*",
"*"]`. When used, all state events are returned by default, and
additional entries FILTER OUT the returned set of state events. These
additional entries cannot use `*` themselves. For example, `["*", "*"],
["m.room.member", "@alice:example.com"]` will *exclude* every
`m.room.member` event *except* for `@alice:example.com`, and include
every other state event. In addition, `["*", "*"], ["m.space.child",
"*"]` is an error, the `m.space.child` filter is not required as it
would have been returned anyway.
Room members can be lazily-loaded by using the special `$LAZY` state key
(`["m.room.member", "$LAZY"]`). Typically, when you view a room, you
want to retrieve all state events except for m.room.member events which
you want to lazily load. To get this behaviour, clients can send the
following::
{
"required_state": [
// activate lazy loading
["m.room.member", "$LAZY"],
// request all state events _except_ for m.room.member
events which are lazily loaded
["*", "*"]
]
}
timeline_limit: The maximum number of timeline events to return per response.
include_old_rooms: Determines if `predecessor` rooms are included in the
`rooms` response. The user MUST be joined to old rooms for them to show up
in the response.
include_heroes: Return a stripped variant of membership events (containing
`user_id` and optionally `avatar_url` and `displayname`) for the users used
to calculate the room name.
filters: Filters to apply to the list before sorting.
bump_event_types: Allowlist of event types which should be considered recent activity
when sorting `by_recency`. By omitting event types from this field,
clients can ensure that uninteresting events (e.g. a profile rename) do
not cause a room to jump to the top of its list(s). Empty or omitted
`bump_event_types` have no effect—all events in a room will be
considered recent activity.
"""

class Filters(RequestBodyModel):
is_dm: Optional[StrictBool] = None
spaces: Optional[List[StrictStr]] = None
is_encrypted: Optional[StrictBool] = None
is_invite: Optional[StrictBool] = None
room_types: Optional[List[Union[StrictStr, None]]] = None
not_room_types: Optional[List[StrictStr]] = None
room_name_like: Optional[StrictStr] = None
tags: Optional[List[StrictStr]] = None
not_tags: Optional[List[StrictStr]] = None

# mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884
if TYPE_CHECKING:
ranges: Optional[List[Tuple[int, int]]] = None
else:
ranges: Optional[List[Tuple[conint(ge=0, strict=True), conint(ge=0, strict=True)]]] = None # type: ignore[valid-type]
sort: Optional[List[StrictStr]] = None
slow_get_all_rooms: Optional[StrictBool] = False
include_heroes: Optional[StrictBool] = False
filters: Optional[Filters] = None
bump_event_types: Optional[List[StrictStr]] = None

class RoomSubscription(CommonRoomParameters):
pass

class Extension(RequestBodyModel):
enabled: Optional[StrictBool] = False
lists: Optional[List[StrictStr]] = None
rooms: Optional[List[StrictStr]] = None

# mypy workaround via https://github.com/pydantic/pydantic/issues/156#issuecomment-1130883884
if TYPE_CHECKING:
lists: Optional[Dict[str, SlidingSyncList]] = None
else:
lists: Optional[Dict[constr(max_length=64, strict=True), SlidingSyncList]] = None # type: ignore[valid-type]
room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] = None
extensions: Optional[Dict[StrictStr, Extension]] = None

@validator("lists")
def lists_length_check(
cls, value: Optional[Dict[str, SlidingSyncList]]
) -> Optional[Dict[str, SlidingSyncList]]:
if value is not None:
assert len(value) <= 100, f"Max lists: 100 but saw {len(value)}"
return value
3 changes: 3 additions & 0 deletions synapse/rest/client/room.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,9 @@ async def on_PUT(
try:
if event_type == EventTypes.Member:
membership = content.get("membership", None)
if not isinstance(membership, str):
raise SynapseError(400, "Invalid membership (must be a string)")

event_id, _ = await self.room_member_handler.update_membership(
requester,
target=UserID.from_string(state_key),
Expand Down
Loading

0 comments on commit df0cf9e

Please sign in to comment.