Merge branch 'impl/lazy-guild-removals' into 'master'

Impl/lazy guild removals

Closes #93

See merge request litecord/litecord!75

litecord/blueprints/guild/members.py

@@ -199,6 +199,7 @@ async def modify_guild_member(guild_id, member_id):
     return "", 204
 
 
+@bp.route("/<int:guild_id>/members/@me", methods=["PATCH"])
 @bp.route("/<int:guild_id>/members/@me/nick", methods=["PATCH"])
 async def update_nickname(guild_id):
     """Update a member's nickname in a guild."""

litecord/blueprints/guilds.py

@@ -127,6 +127,8 @@ async def create_guild():
     else:
         image = None
 
+    region = j["region"] if "region" in j else next(iter(app.voice.lvsp.regions))
+
     await app.db.execute(
         """
         INSERT INTO guilds (id, name, region, icon, owner_id,
@@ -136,7 +138,7 @@ async def create_guild():
         """,
         guild_id,
         j["name"],
-        j["region"],
+        region,
         image,
         user_id,
         j.get("verification_level", 0),

litecord/common/guilds.py

@@ -329,7 +329,12 @@ async def add_member(guild_id: int, user_id: int, *, basic=False):
 
     # pubsub changes for new member
     await app.lazy_guild.new_member(guild_id, user_id)
-    states = await app.dispatcher.guild.sub_user(guild_id, user_id)
+
+    # TODO how to remove repetition between this and websocket's subscribe_all?
+    states, channels = await app.dispatcher.guild.sub_user(guild_id, user_id)
+    for channel_id in channels:
+        for state in states:
+            await app.dispatcher.channel.sub(channel_id, state.session_id)
 
     guild = await app.storage.get_guild_full(guild_id, user_id, 250)
     for state in states:

litecord/gateway/schemas.py

@@ -65,6 +65,47 @@ IDENTIFY_SCHEMA = {
                 "shard": {"type": "list", "required": False},
                 "presence": {"type": "dict", "required": False},
                 "intents": {"coerce": Intents, "required": False},
+                # TODO schema
+                "properties": {
+                    "type": "dict",
+                    "required": False,
+                    "schema": {
+                        "browser": {"type": "string", "required": False},
+                        "client_build_number": {"type": "number", "required": False},
+                        "client_event_source": {
+                            "type": "string",
+                            "required": False,
+                            "nullable": True,
+                        },
+                        "client_version": {"type": "string", "required": False},
+                        "distro": {"type": "string", "required": False},
+                        "os": {"type": "string", "required": False},
+                        "os_arch": {"type": "string", "required": False},
+                        "os_version": {"type": "string", "required": False},
+                        "release_channel": {"type": "string", "required": False},
+                        "system_locale": {"type": "string", "required": False},
+                        "window_manager": {"type": "string", "required": False},
+                    },
+                },
+                "capabilities": {"type": "number", "required": False},
+                "client_state": {
+                    "type": "dict",
+                    "required": False,
+                    "schema": {
+                        # guild_hashes is a Dict with keys being guild ids and
+                        # values being a list of 3 strings. this can not be
+                        # validated by cerberus
+                        "highest_last_message_id": {
+                            "type": "string",
+                            "required": False,
+                        },
+                        "read_state_version": {"type": "number", "required": False},
+                        "user_guild_settings_version": {
+                            "type": "number",
+                            "required": False,
+                        },
+                    },
+                },
             },
         }
     },
@@ -90,11 +131,14 @@ REQ_GUILD_SCHEMA = {
         "d": {
             "type": "dict",
             "schema": {
-                "guild_id": {"type": "string", "required": True},
+                "user_ids": {
-                "user_ids": {"type": "list", "required": False},
+                    "type": "list",
+                    "required": False,
+                    "schema": {"type": "string"},
+                },
                 "query": {"type": "string", "required": False},
                 "limit": {"type": "number", "required": False},
-                "presences": {"type": "bool", "required": False},
+                "presences": {"type": "boolean", "required": False},
             },
         }
     },

litecord/gateway/websocket.py

@@ -506,20 +506,8 @@ class GatewayWebsocket:
         channel_ids: List[int] = []
 
         for guild_id in guild_ids:
-            await app.dispatcher.guild.sub(guild_id, session_id)
+            _, channels = await app.dispatcher.guild.sub_user(guild_id, user_id)
-
+            channel_ids.extend(channels)
-            # instead of calculating which channels to subscribe to
-            # inside guild dispatcher, we calculate them in here, so that
-            # we remove complexity of the dispatcher.
-
-            guild_chan_ids = await app.storage.get_channel_ids(guild_id)
-            for channel_id in guild_chan_ids:
-                perms = await get_permissions(
-                    self.state.user_id, channel_id, storage=self.storage
-                )
-
-                if perms.bits.read_messages:
-                    channel_ids.append(channel_id)
 
         log.info("subscribing to {} guild channels", len(channel_ids))
         for channel_id in channel_ids:
@@ -664,7 +652,12 @@ class GatewayWebsocket:
 
     async def handle_2(self, payload: Dict[str, Any]):
         """Handle the OP 2 Identify packet."""
-        payload = validate(payload, IDENTIFY_SCHEMA)
+        # do not validate given guild_hashes
+        payload_copy = dict(payload)
+        payload_copy["d"].get("client_state", {"guild_hashes": None}).pop(
+            "guild_hashes"
+        )
+        validate(payload_copy, IDENTIFY_SCHEMA)
         data = payload["d"]
         token = data["token"]
 
@@ -924,9 +917,18 @@ class GatewayWebsocket:
 
     async def handle_8(self, payload: Dict):
         """Handle OP 8 Request Guild Members."""
-        payload = validate(payload, REQ_GUILD_SCHEMA)
+
+        # we do not validate guild ids because it can either be a string
+        # or a list of strings and cerberus does not validate that.
+        payload_copy = dict(payload)
+        payload_copy["d"].pop("guild_id")
+        validate(payload_copy, REQ_GUILD_SCHEMA)
+
         data = payload["d"]
-        gids = data["guild_id"]
+        gids = data.get("guild_id")
+        # Discord actually sent this??
+        if gids is None:
+            return
 
         uids, query, limit, presences = (
             data.get("user_ids", []),

litecord/pubsub/guild.py

@@ -17,7 +17,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 
-from typing import List
+from typing import List, Tuple
 
 from quart import current_app as app
 from logbook import Logger
@@ -25,6 +25,8 @@ from logbook import Logger
 from .dispatcher import DispatcherWithState, GatewayEvent
 from litecord.gateway.state import GatewayState
 from litecord.enums import EVENTS_TO_INTENTS
+from litecord.permissions import get_permissions
+
 
 log = Logger(__name__)
 
@@ -33,7 +35,7 @@ def can_dispatch(event_type, event_data, state) -> bool:
     # If we're sending to the same user for this kind of event,
     # bypass event logic (always send)
     if event_type == "GUILD_MEMBER_UPDATE":
-        user_id = int(event_data["user"])
+        user_id = int(event_data["user"]["id"])
         return user_id == state.user_id
 
     # TODO Guild Create and Req Guild Members have specific
@@ -48,12 +50,26 @@ def can_dispatch(event_type, event_data, state) -> bool:
 class GuildDispatcher(DispatcherWithState[int, str, GatewayEvent, List[str]]):
     """Guild backend for Pub/Sub."""
 
-    async def sub_user(self, guild_id: int, user_id: int) -> List[GatewayState]:
+    async def sub_user(
+        self, guild_id: int, user_id: int
+    ) -> Tuple[List[GatewayState], List[int]]:
         states = app.state_manager.fetch_states(user_id, guild_id)
         for state in states:
             await self.sub(guild_id, state.session_id)
 
-        return states
+        # instead of calculating which channels to subscribe to
+        # inside guild dispatcher, we calculate them in here, so that
+        # we remove complexity of the dispatcher.
+
+        guild_chan_ids = await app.storage.get_channel_ids(guild_id)
+        channel_ids = []
+        for channel_id in guild_chan_ids:
+            perms = await get_permissions(user_id, channel_id)
+
+            if perms.bits.read_messages:
+                channel_ids.append(channel_id)
+
+        return states, channel_ids
 
     async def dispatch_filter(
         self, guild_id: int, filter_function, event: GatewayEvent

litecord/pubsub/lazy_guild.py

@@ -870,6 +870,22 @@ class GuildMemberList:
             session_ids_new, new_user_index
         )
 
+    async def _pres_update_remove(
+        self, user_id: int, old_group: GroupID, old_index: int
+    ):
+        log.debug(
+            "removal update: uid={} old={} rel_idx={} new={}",
+            user_id,
+            old_group,
+            old_index,
+        )
+
+        old_user_index = self._get_item_index(user_id)
+        assert old_user_index is not None
+        self.list.data[old_group].remove(user_id)
+        session_ids_old = list(self._get_subs(old_user_index))
+        return await self._resync(session_ids_old, old_user_index)
+
     async def new_member(self, user_id: int):
         """Insert a new member."""
         if not self.list:
@@ -1019,14 +1035,6 @@ class GuildMemberList:
         old_presence = self.list.presences[user_id]
         has_nick = "nick" in partial_presence
 
-        # partial presences don't have 'nick'. we only use it
-        # as a flag that we're doing a mixed update (complex
-        # but without any inter-group changes)
-        try:
-            partial_presence.pop("nick")
-        except KeyError:
-            pass
-
         for group, member_ids in self.list:
             try:
                 old_index = member_ids.index(user_id)
@@ -1051,7 +1059,6 @@ class GuildMemberList:
         status = partial_presence.get("status", old_presence["status"])
 
         # calculate a possible new group
-        # TODO: handle when new_group is None (member loses perms)
         new_group = await self._get_group_for_member(user_id, roles, status)
 
         log.debug(
@@ -1071,8 +1078,16 @@ class GuildMemberList:
         # nickname changes, treat this as a simple update
         if old_group == new_group and not has_nick:
             return await self._pres_update_simple(user_id)
-
+        elif new_group is None:
-        return await self._pres_update_complex(user_id, old_group, old_index, new_group)
+            # The user is being removed from the overall list.
+            #
+            # This happens because they lost permissions to the relevant
+            # channel.
+            return await self._pres_update_remove(user_id, old_group, old_index)
+        else:
+            return await self._pres_update_complex(
+                user_id, old_group, old_index, new_group
+            )
 
     async def new_role(self, role: dict):
         """Add a new role to the list.

litecord/schemas.py

@@ -270,6 +270,12 @@ GUILD_CREATE = {
     "explicit_content_filter": {"type": "explicit", "default": 0},
     "roles": {"type": "list", "required": False, "schema": PARTIAL_ROLE_GUILD_CREATE},
     "channels": {"type": "list", "default": [], "schema": PARTIAL_CHANNEL_GUILD_CREATE},
+    # not supported
+    "system_channel_id": {"coerce": int, "required": False, "nullable": True},
+    "guild_template_code": {
+        "type": "string",
+        "required": False,
+    },
 }
 
 
@@ -424,7 +430,7 @@ INVITE = {
     "max_age": {
         "type": "number",
         "min": 0,
-        "max": 86400,
+        "max": 666666,  # TODO find correct max value
         # a day
         "default": 86400,
     },
@@ -445,6 +451,7 @@ INVITE = {
         "nullable": True,
     },  # discord client sends invite code there
     # sent by official client, unknown purpose
+    "target_type": {"type": "string", "required": False, "nullable": True},
     "target_user_id": {"type": "snowflake", "required": False, "nullable": True},
     "target_user_type": {"type": "number", "required": False, "nullable": True},
 }