From 07edc235f5b08537de46a8d3eda9f36cf4af19cb Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 21:45:43 -0300 Subject: [PATCH 01/14] lazy_guild: add handling for member losing permissions --- litecord/pubsub/lazy_guild.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/litecord/pubsub/lazy_guild.py b/litecord/pubsub/lazy_guild.py index 5bc2d38..5772089 100644 --- a/litecord/pubsub/lazy_guild.py +++ b/litecord/pubsub/lazy_guild.py @@ -870,6 +870,22 @@ class GuildMemberList: session_ids_new, new_user_index ) + async def _pres_update_remove( + self, user_id: int, old_group: GroupID, old_index: int + ): + log.debug( + "removal update: uid={} old={} rel_idx={} new={}", + user_id, + old_group, + old_index, + ) + + old_user_index = self._get_item_index(user_id) + assert old_user_index is not None + self.list.data[old_group].remove(user_id) + session_ids_old = list(self._get_subs(old_user_index)) + return await self._resync(session_ids_old, old_user_index) + async def new_member(self, user_id: int): """Insert a new member.""" if not self.list: @@ -1051,7 +1067,6 @@ class GuildMemberList: status = partial_presence.get("status", old_presence["status"]) # calculate a possible new group - # TODO: handle when new_group is None (member loses perms) new_group = await self._get_group_for_member(user_id, roles, status) log.debug( @@ -1071,8 +1086,16 @@ class GuildMemberList: # nickname changes, treat this as a simple update if old_group == new_group and not has_nick: return await self._pres_update_simple(user_id) - - return await self._pres_update_complex(user_id, old_group, old_index, new_group) + elif new_group is None: + # The user is being removed from the overall list. + # + # This happens because they lost permissions to the relevant + # channel. + return await self._pres_update_remove(user_id, old_group, old_index) + else: + return await self._pres_update_complex( + user_id, old_group, old_index, new_group + ) async def new_role(self, role: dict): """Add a new role to the list. From a830c9cb77252f5acfd10f1ac78d5c0bc06275ae Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 22:13:09 -0300 Subject: [PATCH 02/14] update gateway schemas --- litecord/gateway/schemas.py | 48 +++++++++++++++++++++++++++++++++-- litecord/gateway/websocket.py | 12 +++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/litecord/gateway/schemas.py b/litecord/gateway/schemas.py index ed8d12a..387740b 100644 --- a/litecord/gateway/schemas.py +++ b/litecord/gateway/schemas.py @@ -65,6 +65,47 @@ IDENTIFY_SCHEMA = { "shard": {"type": "list", "required": False}, "presence": {"type": "dict", "required": False}, "intents": {"coerce": Intents, "required": False}, + # TODO schema + "properties": { + "type": "dict", + "required": False, + "schema": { + "browser": {"type": "string", "required": False}, + "client_build_number": {"type": "number", "required": False}, + "client_event_source": { + "type": "string", + "required": False, + "nullable": True, + }, + "client_version": {"type": "string", "required": False}, + "distro": {"type": "string", "required": False}, + "os": {"type": "string", "required": False}, + "os_arch": {"type": "string", "required": False}, + "os_version": {"type": "string", "required": False}, + "release_channel": {"type": "string", "required": False}, + "system_locale": {"type": "string", "required": False}, + "window_manager": {"type": "string", "required": False}, + }, + }, + "capabilities": {"type": "number", "required": False}, + "client_state": { + "type": "dict", + "required": False, + "schema": { + # guild_hashes is a Dict with keys being guild ids and + # values being a list of 3 strings. this can not be + # validated by cerberus + "highest_last_message_id": { + "type": "string", + "required": False, + }, + "read_state_version": {"type": "number", "required": False}, + "user_guild_settings_version": { + "type": "number", + "required": False, + }, + }, + }, }, } }, @@ -90,8 +131,11 @@ REQ_GUILD_SCHEMA = { "d": { "type": "dict", "schema": { - "guild_id": {"type": "string", "required": True}, - "user_ids": {"type": "list", "required": False}, + "user_ids": { + "type": "list", + "required": False, + "schema": {"type": "string"}, + }, "query": {"type": "string", "required": False}, "limit": {"type": "number", "required": False}, "presences": {"type": "bool", "required": False}, diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index a0a7fef..9db80a5 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -664,7 +664,9 @@ class GatewayWebsocket: async def handle_2(self, payload: Dict[str, Any]): """Handle the OP 2 Identify packet.""" - payload = validate(payload, IDENTIFY_SCHEMA) + payload_copy = dict(payload) + payload_copy["d"].get("client_state", {}).pop("guild_hashes") + validate(payload_copy, IDENTIFY_SCHEMA) data = payload["d"] token = data["token"] @@ -924,7 +926,13 @@ class GatewayWebsocket: async def handle_8(self, payload: Dict): """Handle OP 8 Request Guild Members.""" - payload = validate(payload, REQ_GUILD_SCHEMA) + + # we do not validate guild ids because it can either be a string + # or a list of strings and cerberus does not validate that. + payload_copy = dict(payload) + payload_copy["d"].pop("guild_id") + validate(payload_copy, REQ_GUILD_SCHEMA) + data = payload["d"] gids = data["guild_id"] From a5c52f5a2c6638203bf707b75ea49adb1c086b88 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 22:24:19 -0300 Subject: [PATCH 03/14] fix typo --- litecord/gateway/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litecord/gateway/schemas.py b/litecord/gateway/schemas.py index 387740b..4626110 100644 --- a/litecord/gateway/schemas.py +++ b/litecord/gateway/schemas.py @@ -138,7 +138,7 @@ REQ_GUILD_SCHEMA = { }, "query": {"type": "string", "required": False}, "limit": {"type": "number", "required": False}, - "presences": {"type": "bool", "required": False}, + "presences": {"type": "boolean", "required": False}, }, } }, From 2b3f1ac48c6d74f633a4404734b0e3dbccfac0c6 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 22:24:28 -0300 Subject: [PATCH 04/14] gateway: add hack for Request Guild Members --- litecord/gateway/websocket.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 9db80a5..560f207 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -934,7 +934,10 @@ class GatewayWebsocket: validate(payload_copy, REQ_GUILD_SCHEMA) data = payload["d"] - gids = data["guild_id"] + gids = data.get("guild_id") + # Discord actually sent this?? + if gids is None: + return uids, query, limit, presences = ( data.get("user_ids", []), From 55032aba1e65c67b3eeb159ee81d9864b1871ad3 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 22:31:00 -0300 Subject: [PATCH 05/14] gateway: fix guild_hashes fallback --- litecord/gateway/websocket.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 560f207..05376f3 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -664,8 +664,11 @@ class GatewayWebsocket: async def handle_2(self, payload: Dict[str, Any]): """Handle the OP 2 Identify packet.""" + # do not validate given guild_hashes payload_copy = dict(payload) - payload_copy["d"].get("client_state", {}).pop("guild_hashes") + payload_copy["d"].get("client_state", {"guild_hashes": None}).pop( + "guild_hashes" + ) validate(payload_copy, IDENTIFY_SCHEMA) data = payload["d"] token = data["token"] From 36e6e078b3dbd7722e2e62e11d6068170984d403 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 22:37:36 -0300 Subject: [PATCH 06/14] guild: add secondary route for nick updating --- litecord/blueprints/guild/members.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litecord/blueprints/guild/members.py b/litecord/blueprints/guild/members.py index 9301c63..e6ec42b 100644 --- a/litecord/blueprints/guild/members.py +++ b/litecord/blueprints/guild/members.py @@ -199,6 +199,7 @@ async def modify_guild_member(guild_id, member_id): return "", 204 +@bp.route("//members/@me", methods=["PATCH"]) @bp.route("//members/@me/nick", methods=["PATCH"]) async def update_nickname(guild_id): """Update a member's nickname in a guild.""" From 8f229bf764fabbf226d02c3eb288ce41afe060bd Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 22:38:51 -0300 Subject: [PATCH 07/14] pubsub: fix typo --- litecord/pubsub/guild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litecord/pubsub/guild.py b/litecord/pubsub/guild.py index 995837f..d7477a0 100644 --- a/litecord/pubsub/guild.py +++ b/litecord/pubsub/guild.py @@ -33,7 +33,7 @@ def can_dispatch(event_type, event_data, state) -> bool: # If we're sending to the same user for this kind of event, # bypass event logic (always send) if event_type == "GUILD_MEMBER_UPDATE": - user_id = int(event_data["user"]) + user_id = int(event_data["user"]["id"]) return user_id == state.user_id # TODO Guild Create and Req Guild Members have specific From 5b0a632b9f8ca16622d12c827eb2af82b66f825e Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 23:27:07 -0300 Subject: [PATCH 08/14] handle optional voice regions on guild create --- litecord/blueprints/guilds.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litecord/blueprints/guilds.py b/litecord/blueprints/guilds.py index d419a90..9f0a04c 100644 --- a/litecord/blueprints/guilds.py +++ b/litecord/blueprints/guilds.py @@ -127,6 +127,8 @@ async def create_guild(): else: image = None + region = j["region"] if "region" in j else next(iter(app.voice.lvsp.regions)) + await app.db.execute( """ INSERT INTO guilds (id, name, region, icon, owner_id, @@ -136,7 +138,7 @@ async def create_guild(): """, guild_id, j["name"], - j["region"], + region, image, user_id, j.get("verification_level", 0), From bf5774aa03d16f49b554ec6199892292bb9d3e1b Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 23:27:23 -0300 Subject: [PATCH 09/14] schemas: update guild create --- litecord/schemas.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litecord/schemas.py b/litecord/schemas.py index 0f28a0a..a9cc109 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -270,6 +270,12 @@ GUILD_CREATE = { "explicit_content_filter": {"type": "explicit", "default": 0}, "roles": {"type": "list", "required": False, "schema": PARTIAL_ROLE_GUILD_CREATE}, "channels": {"type": "list", "default": [], "schema": PARTIAL_CHANNEL_GUILD_CREATE}, + # not supported + "system_channel_id": {"coerce": int, "required": False, "nullable": True}, + "guild_template_code": { + "type": "string", + "required": False, + }, } From 67785920bb6ba449c0e5a9d779e8f11a3396098c Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 23:27:30 -0300 Subject: [PATCH 10/14] lazy_guild: do not remove member nicks --- litecord/pubsub/lazy_guild.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/litecord/pubsub/lazy_guild.py b/litecord/pubsub/lazy_guild.py index 5772089..ac7835d 100644 --- a/litecord/pubsub/lazy_guild.py +++ b/litecord/pubsub/lazy_guild.py @@ -1035,14 +1035,6 @@ class GuildMemberList: old_presence = self.list.presences[user_id] has_nick = "nick" in partial_presence - # partial presences don't have 'nick'. we only use it - # as a flag that we're doing a mixed update (complex - # but without any inter-group changes) - try: - partial_presence.pop("nick") - except KeyError: - pass - for group, member_ids in self.list: try: old_index = member_ids.index(user_id) From 9273d8cbda2c870b89c699101325ddfd866a6983 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 23:41:07 -0300 Subject: [PATCH 11/14] schemas: update invite --- litecord/schemas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litecord/schemas.py b/litecord/schemas.py index a9cc109..f4a389a 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -430,7 +430,7 @@ INVITE = { "max_age": { "type": "number", "min": 0, - "max": 86400, + "max": 666666, # TODO find correct max value # a day "default": 86400, }, @@ -451,6 +451,7 @@ INVITE = { "nullable": True, }, # discord client sends invite code there # sent by official client, unknown purpose + "target_type": {"type": "string", "required": False, "nullable": True}, "target_user_id": {"type": "snowflake", "required": False, "nullable": True}, "target_user_type": {"type": "number", "required": False, "nullable": True}, } From d1b10e74094369b1d84d57e33d7ffef4695c3ce9 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 23:41:19 -0300 Subject: [PATCH 12/14] fix new joins not subscribing to channels --- litecord/common/guilds.py | 7 ++++++- litecord/gateway/websocket.py | 16 ++-------------- litecord/pubsub/guild.py | 22 ++++++++++++++++++++-- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/litecord/common/guilds.py b/litecord/common/guilds.py index 4dd9e81..368ba07 100644 --- a/litecord/common/guilds.py +++ b/litecord/common/guilds.py @@ -329,7 +329,12 @@ async def add_member(guild_id: int, user_id: int, *, basic=False): # pubsub changes for new member await app.lazy_guild.new_member(guild_id, user_id) - states = await app.dispatcher.guild.sub_user(guild_id, user_id) + + # TODO how to remove repetition between this and websocket's subscribe_all? + states, channels = await app.dispatcher.guild.sub_user(guild_id, user_id) + for channel_id in channel_ids: + for state in states: + await app.dispatcher.channel.sub(channel_id, state.session_id) guild = await app.storage.get_guild_full(guild_id, user_id, 250) for state in states: diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 05376f3..2c70120 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -506,20 +506,8 @@ class GatewayWebsocket: channel_ids: List[int] = [] for guild_id in guild_ids: - await app.dispatcher.guild.sub(guild_id, session_id) - - # instead of calculating which channels to subscribe to - # inside guild dispatcher, we calculate them in here, so that - # we remove complexity of the dispatcher. - - guild_chan_ids = await app.storage.get_channel_ids(guild_id) - for channel_id in guild_chan_ids: - perms = await get_permissions( - self.state.user_id, channel_id, storage=self.storage - ) - - if perms.bits.read_messages: - channel_ids.append(channel_id) + _, channels = await app.dispatcher.guild.sub_user(guild_id, session_id) + channel_ids.extend(channels) log.info("subscribing to {} guild channels", len(channel_ids)) for channel_id in channel_ids: diff --git a/litecord/pubsub/guild.py b/litecord/pubsub/guild.py index d7477a0..eeaea16 100644 --- a/litecord/pubsub/guild.py +++ b/litecord/pubsub/guild.py @@ -25,6 +25,8 @@ from logbook import Logger from .dispatcher import DispatcherWithState, GatewayEvent from litecord.gateway.state import GatewayState from litecord.enums import EVENTS_TO_INTENTS +from litecord.permissions import get_permissions + log = Logger(__name__) @@ -48,12 +50,28 @@ def can_dispatch(event_type, event_data, state) -> bool: class GuildDispatcher(DispatcherWithState[int, str, GatewayEvent, List[str]]): """Guild backend for Pub/Sub.""" - async def sub_user(self, guild_id: int, user_id: int) -> List[GatewayState]: + async def sub_user( + self, guild_id: int, user_id: int + ) -> Tuple[List[GatewayState], List[int]]: states = app.state_manager.fetch_states(user_id, guild_id) for state in states: await self.sub(guild_id, state.session_id) - return states + # instead of calculating which channels to subscribe to + # inside guild dispatcher, we calculate them in here, so that + # we remove complexity of the dispatcher. + + guild_chan_ids = await app.storage.get_channel_ids(guild_id) + channel_ids = [] + for channel_id in guild_chan_ids: + perms = await get_permissions( + self.state.user_id, channel_id, storage=self.storage + ) + + if perms.bits.read_messages: + channel_ids.append(channel_id) + + return states, channel_ids async def dispatch_filter( self, guild_id: int, filter_function, event: GatewayEvent From 0eb37554d3490086121c28650e2ec036b5d5a51d Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 14 Jul 2021 23:44:04 -0300 Subject: [PATCH 13/14] fix typos --- litecord/gateway/websocket.py | 2 +- litecord/pubsub/guild.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 2c70120..8228f51 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -506,7 +506,7 @@ class GatewayWebsocket: channel_ids: List[int] = [] for guild_id in guild_ids: - _, channels = await app.dispatcher.guild.sub_user(guild_id, session_id) + _, channels = await app.dispatcher.guild.sub_user(guild_id, user_id) channel_ids.extend(channels) log.info("subscribing to {} guild channels", len(channel_ids)) diff --git a/litecord/pubsub/guild.py b/litecord/pubsub/guild.py index eeaea16..0085acd 100644 --- a/litecord/pubsub/guild.py +++ b/litecord/pubsub/guild.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import List +from typing import List, Tuple from quart import current_app as app from logbook import Logger @@ -64,9 +64,7 @@ class GuildDispatcher(DispatcherWithState[int, str, GatewayEvent, List[str]]): guild_chan_ids = await app.storage.get_channel_ids(guild_id) channel_ids = [] for channel_id in guild_chan_ids: - perms = await get_permissions( - self.state.user_id, channel_id, storage=self.storage - ) + perms = await get_permissions(user_id, channel_id) if perms.bits.read_messages: channel_ids.append(channel_id) From b71950715c64466bafde74b7ac4efb4f66b8b185 Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 15 Jul 2021 00:20:14 -0300 Subject: [PATCH 14/14] fix typo --- litecord/common/guilds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litecord/common/guilds.py b/litecord/common/guilds.py index 368ba07..a8a960a 100644 --- a/litecord/common/guilds.py +++ b/litecord/common/guilds.py @@ -332,7 +332,7 @@ async def add_member(guild_id: int, user_id: int, *, basic=False): # TODO how to remove repetition between this and websocket's subscribe_all? states, channels = await app.dispatcher.guild.sub_user(guild_id, user_id) - for channel_id in channel_ids: + for channel_id in channels: for state in states: await app.dispatcher.channel.sub(channel_id, state.session_id)