From 0d38c7cc0c545003606935820b4dcb82061a889a Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 28 Feb 2019 20:51:09 -0300 Subject: [PATCH 01/55] gateway.websocket: add basic vsu handling this adds *theoretical* interfaces to a VoiceManager. The actual VoiceManager does not work and probably any VSU being sent over will crash the websocket with an AttributionError. - enums: add VOICE_CHANNELS - run: add App.voice attribute --- litecord/enums.py | 6 ++ litecord/gateway/websocket.py | 106 +++++++++++++++++++++++++++++----- litecord/voice/manager.py | 23 ++++++++ run.py | 3 + 4 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 litecord/voice/manager.py diff --git a/litecord/enums.py b/litecord/enums.py index 60037d6..4107fd5 100644 --- a/litecord/enums.py +++ b/litecord/enums.py @@ -83,6 +83,12 @@ GUILD_CHANS = (ChannelType.GUILD_TEXT, ChannelType.GUILD_CATEGORY) +VOICE_CHANNELS = ( + ChannelType.DM, ChannelType.GUILD_VOICE, + ChannelType.GUILD_CATEGORY +) + + class ActivityType(EasyEnum): PLAYING = 0 STREAMING = 1 diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index ca2d444..5c5fe05 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -30,12 +30,12 @@ from logbook import Logger import earl from litecord.auth import raw_token_check -from litecord.enums import RelationshipType +from litecord.enums import RelationshipType, ChannelType, VOICE_CHANNELS from litecord.schemas import validate, GW_STATUS_UPDATE from litecord.utils import ( task_wrapper, LitecordJSONEncoder, yield_chunks ) -from litecord.permissions import get_permissions +from litecord.permissions import get_permissions, ALL_PERMISSIONS from litecord.gateway.opcodes import OP from litecord.gateway.state import GatewayState @@ -48,14 +48,17 @@ from litecord.gateway.errors import ( ) log = Logger(__name__) + WebsocketProperties = collections.namedtuple( 'WebsocketProperties', 'v encoding compress zctx tasks' ) WebsocketObjects = collections.namedtuple( - 'WebsocketObjects', ('db', 'state_manager', 'storage', - 'loop', 'dispatcher', 'presence', 'ratelimiter', - 'user_storage') + 'WebsocketObjects', ( + 'db', 'state_manager', 'storage', + 'loop', 'dispatcher', 'presence', 'ratelimiter', + 'user_storage', 'voice' + ) ) @@ -113,7 +116,7 @@ class GatewayWebsocket: self.ext = WebsocketObjects( app.db, app.state_manager, app.storage, app.loop, app.dispatcher, app.presence, app.ratelimiter, - app.user_storage + app.user_storage, app.voice ) self.storage = self.ext.storage @@ -598,16 +601,93 @@ class GatewayWebsocket: # setting new presence to state await self.update_status(presence) + @property + def voice_key(self): + """Voice state key.""" + return (self.state.user_id, self.state.session_id) + + async def _voice_check(self, guild_id: int, channel_id: int): + """Check if the user can join the given guild/channel pair.""" + guild = None + if guild_id: + guild = await self.storage.get_guild(guild_id) + + channel = await self.storage.get_channel(channel_id) + ctype = ChannelType(channel['type']) + + if ctype not in VOICE_CHANNELS: + return + + if guild and channel.get(['guild_id']) != guild['id']: + return + + is_guild_voice = ctype == ChannelType.GUILD_VOICE + + states = await self.ext.voice.state_count(channel_id) + perms = (ALL_PERMISSIONS + if not is_guild_voice else + await get_permissions(self.state.user_id, + channel_id, storage=self.storage) + ) + + is_full = states >= channel['user_limit'] + is_bot = self.state.bot + + is_manager = perms.bits.manage_channels + + # if the channel is full AND: + # - user is not a bot + # - user is not manage channels + # then it fails + if not is_bot and not is_manager and is_full: + return + + # all checks passed. + return True + + async def _move_voice(self, guild_id, channel_id): + """Move an existing voice state to the given target.""" + if channel_id is None: + return await self.ext.voice.del_state(self.voice_key) + + if not await self._voice_check(guild_id, channel_id): + return + + await self.ext.voice.move_state( + self.voice_key, guild_id, channel_id) + + async def _create_voice(self, guild_id, channel_id): + """Create a voice state.""" + if not await self._voice_check(guild_id, channel_id): + return + + await self.ext.voice.create_state(self.voice_key, guild_id, channel_id) + async def handle_4(self, payload: Dict[str, Any]): """Handle OP 4 Voice Status Update.""" data = payload['d'] - # for now, ignore - log.debug('got VSU cid={} gid={} deaf={} mute={} video={}', - data.get('channel_id'), - data.get('guild_id'), - data.get('self_deaf'), - data.get('self_mute'), - data.get('self_video')) + + if not self.state: + return + + try: + channel_id = int(data['channel_id']) + guild_id = int(data['guild_id']) + + # TODO: fetch from settings if not provided + # self_deaf = bool(data['self_deaf']) + # self_mute = bool(data['self_mute']) + + # NOTE: self_video is NOT handled. + except (KeyError, ValueError): + pass + + # fetch an existing voice state + user_id, session_id = self.state.user_id, self.state.session_id + voice_state = await self.ext.voice.fetch_state(user_id, session_id) + + func = self._move_voice if voice_state else self._create_voice + await func(guild_id, channel_id) async def _handle_5(self, payload: Dict[str, Any]): """Handle OP 5 Voice Server Ping. diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py new file mode 100644 index 0000000..7457623 --- /dev/null +++ b/litecord/voice/manager.py @@ -0,0 +1,23 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +class VoiceManager: + """Main voice manager class.""" + def __init__(self, app): + self.app = app diff --git a/run.py b/run.py index ddebbe8..1c11556 100644 --- a/run.py +++ b/run.py @@ -70,6 +70,7 @@ from litecord.dispatcher import EventDispatcher from litecord.presence import PresenceManager from litecord.images import IconManager from litecord.jobs import JobManager +from litecord.voice.manager import VoiceManager from litecord.utils import LitecordJSONEncoder @@ -232,6 +233,8 @@ def init_app_managers(app_): app_.storage.presence = app_.presence + app_.voice = VoiceManager(app_) + async def api_index(app_): to_find = {} From f3bc65302dd9d70bc93a79627af169005f555a34 Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 28 Feb 2019 21:16:07 -0300 Subject: [PATCH 02/55] add voice websocket instantiation --- config.ci.py | 5 ++++ config.example.py | 5 ++++ litecord/voice/websocket.py | 27 +++++++++++++++++++ litecord/voice/websocket_starter.py | 42 +++++++++++++++++++++++++++++ run.py | 37 ++++++++++++++++++------- 5 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 litecord/voice/websocket.py create mode 100644 litecord/voice/websocket_starter.py diff --git a/config.ci.py b/config.ci.py index 24d6046..24e9224 100644 --- a/config.ci.py +++ b/config.ci.py @@ -44,6 +44,11 @@ class Config: WS_HOST = 'localhost' WS_PORT = 5001 + #: Where to host the VOICE websocket? + # (a local address the server will bind to) + VWS_HOST = 'localhost' + VWS_PORT = 5003 + # Postgres credentials POSTGRES = {} diff --git a/config.example.py b/config.example.py index e784e8e..532ae7b 100644 --- a/config.example.py +++ b/config.example.py @@ -52,6 +52,11 @@ class Config: WS_HOST = '0.0.0.0' WS_PORT = 5001 + #: Where to host the VOICE websocket? + # (a local address the server will bind to) + VWS_HOST = 'localhost' + VWS_PORT = 5003 + #: Mediaproxy URL on the internet # mediaproxy is made to prevent client IPs being leaked. MEDIA_PROXY = 'localhost:5002' diff --git a/litecord/voice/websocket.py b/litecord/voice/websocket.py new file mode 100644 index 0000000..48146af --- /dev/null +++ b/litecord/voice/websocket.py @@ -0,0 +1,27 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +class VoiceWebsocket: + """Voice websocket class.""" + def __init__(self, ws, app): + self.ws = ws + self.app = app + + async def run(self): + pass diff --git a/litecord/voice/websocket_starter.py b/litecord/voice/websocket_starter.py new file mode 100644 index 0000000..936c143 --- /dev/null +++ b/litecord/voice/websocket_starter.py @@ -0,0 +1,42 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +import urllib.urlparse +from litecord.voice.websocket import VoiceWebsocket + +async def voice_websocket_handle(app, ws, url): + """Main handler to instantiate a VoiceWebsocket + with the given url.""" + args = urllib.parse.parse_qs( + urllib.parse.urlparse(url).query + ) + + try: + gw_version = args['v'][0] + except (KeyError, IndexError): + gw_version = '3' + + if gw_version not in ('1', '2', '3'): + return await ws.close(1000, 'Invalid gateway version') + + # TODO: select a different VoiceWebsocket runner depending on + # version. however i do not have docs on voice websockets + # earlier than v3. + vws = VoiceWebsocket(ws, app) + await vws.run() diff --git a/run.py b/run.py index 1c11556..edc67fb 100644 --- a/run.py +++ b/run.py @@ -61,7 +61,6 @@ from litecord.blueprints.user.billing_job import ( from litecord.ratelimits.handler import ratelimit_handler from litecord.ratelimits.main import RatelimitManager -from litecord.gateway import websocket_handler from litecord.errors import LitecordError from litecord.gateway.state_manager import StateManager from litecord.storage import Storage @@ -72,6 +71,9 @@ from litecord.images import IconManager from litecord.jobs import JobManager from litecord.voice.manager import VoiceManager +from litecord.gateway import websocket_handler +from litecord.voice.websocket_starter import voice_websocket_handler + from litecord.utils import LitecordJSONEncoder # setup logbook @@ -295,6 +297,19 @@ async def post_app_start(app_): app_.sched.spawn(api_index(app_)) +def start_websocket(host, port, ws_handler) -> asyncio.Future: + """Start a websocket. Returns the websocket future""" + host, port = app.config['WS_HOST'], app.config['WS_PORT'] + log.info(f'starting websocket at {host} {port}') + + async def _wrapper(ws, url): + # We wrap the main websocket_handler + # so we can pass quart's app object. + await ws_handler(app, ws, url) + + return websockets.serve(_wrapper, host, port) + + @app.before_serving async def app_before_serving(): log.info('opening db') @@ -307,19 +322,21 @@ async def app_before_serving(): init_app_managers(app) - # start the websocket, etc - host, port = app.config['WS_HOST'], app.config['WS_PORT'] - log.info(f'starting websocket at {host} {port}') + # start gateway websocket and voice websocket + ws_fut = start_websocket( + app.config['WS_HOST'], app.config['WS_PORT'], + websocket_handler + ) - async def _wrapper(ws, url): - # We wrap the main websocket_handler - # so we can pass quart's app object. - await websocket_handler(app, ws, url) + vws_fut = start_websocket( + app.config['VWS_HOST'], app.config['VWS_PORT'], + voice_websocket_handler + ) - ws_future = websockets.serve(_wrapper, host, port) await post_app_start(app) - await ws_future + await ws_fut + await vws_fut @app.after_serving From ca5386d3ceb226a7f9e626c1fd1a1c2384f85e10 Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 28 Feb 2019 23:12:19 -0300 Subject: [PATCH 03/55] add VOICE_WEBSOCKET_URL to configs - gateway: remove implicit /ws path, leave it up to configs - run: fix start_websocket --- config.ci.py | 1 + config.example.py | 1 + litecord/blueprints/gateway.py | 2 +- litecord/voice/websocket_starter.py | 4 ++-- run.py | 9 +++------ tests/conftest.py | 20 +++++++++++++++++--- 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/config.ci.py b/config.ci.py index 24e9224..04f1fdd 100644 --- a/config.ci.py +++ b/config.ci.py @@ -38,6 +38,7 @@ class Config: # will hit the websocket. # e.g 'gateway.example.com' for reverse proxies. WEBSOCKET_URL = 'localhost:5001' + VOICE_WEBSOCKET_URL = 'localhost:5002' # Where to host the websocket? # (a local address the server will bind to) diff --git a/config.example.py b/config.example.py index 532ae7b..ad6de74 100644 --- a/config.example.py +++ b/config.example.py @@ -46,6 +46,7 @@ class Config: # will hit the websocket. # e.g 'gateway.example.com' for reverse proxies. WEBSOCKET_URL = 'localhost:5001' + VOICE_WEBSOCKET_URL = 'localhost:5003' #: Where to host the websocket? # (a local address the server will bind to) diff --git a/litecord/blueprints/gateway.py b/litecord/blueprints/gateway.py index 562040b..05303f0 100644 --- a/litecord/blueprints/gateway.py +++ b/litecord/blueprints/gateway.py @@ -29,7 +29,7 @@ bp = Blueprint('gateway', __name__) def get_gw(): """Get the gateway's web""" proto = 'wss://' if app.config['IS_SSL'] else 'ws://' - return f'{proto}{app.config["WEBSOCKET_URL"]}/ws' + return f'{proto}{app.config["WEBSOCKET_URL"]}' @bp.route('/gateway') diff --git a/litecord/voice/websocket_starter.py b/litecord/voice/websocket_starter.py index 936c143..002835c 100644 --- a/litecord/voice/websocket_starter.py +++ b/litecord/voice/websocket_starter.py @@ -17,10 +17,10 @@ along with this program. If not, see . """ -import urllib.urlparse +import urllib.parse from litecord.voice.websocket import VoiceWebsocket -async def voice_websocket_handle(app, ws, url): +async def voice_websocket_handler(app, ws, url): """Main handler to instantiate a VoiceWebsocket with the given url.""" args = urllib.parse.parse_qs( diff --git a/run.py b/run.py index edc67fb..f034d67 100644 --- a/run.py +++ b/run.py @@ -24,7 +24,7 @@ import asyncpg import logbook import logging import websockets -from quart import Quart, g, jsonify, request +from quart import Quart, jsonify, request from logbook import StreamHandler, Logger from logbook.compat import redirect_logging from aiohttp import ClientSession @@ -299,7 +299,6 @@ async def post_app_start(app_): def start_websocket(host, port, ws_handler) -> asyncio.Future: """Start a websocket. Returns the websocket future""" - host, port = app.config['WS_HOST'], app.config['WS_PORT'] log.info(f'starting websocket at {host} {port}') async def _wrapper(ws, url): @@ -315,12 +314,12 @@ async def app_before_serving(): log.info('opening db') await init_app_db(app) - g.app = app - g.loop = asyncio.get_event_loop() + loop = asyncio.get_event_loop() app.session = ClientSession() init_app_managers(app) + await post_app_start(app) # start gateway websocket and voice websocket ws_fut = start_websocket( @@ -333,8 +332,6 @@ async def app_before_serving(): voice_websocket_handler ) - - await post_app_start(app) await ws_fut await vws_fut diff --git a/tests/conftest.py b/tests/conftest.py index d77c012..0750091 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,7 @@ import asyncio import sys import os +import socket import pytest # this is very hacky. @@ -28,16 +29,29 @@ sys.path.append(os.getcwd()) from run import app as main_app, set_blueprints +# pytest-sanic's unused_tcp_port can't be called twice since +# pytest fixtures etc etc. +def _unused_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + @pytest.fixture(name='app') -def _test_app(unused_tcp_port, event_loop): +def _test_app(event_loop): set_blueprints(main_app) main_app.config['_testing'] = True # reassign an unused tcp port for websockets # since the config might give a used one. - main_app.config['WS_PORT'] = unused_tcp_port - main_app.config['WEBSOCKET_URL'] = f'localhost:{unused_tcp_port}' + ws_port, vws_port = _unused_port(), _unused_port() + print(ws_port, vws_port) + + main_app.config['WS_PORT'] = ws_port + main_app.config['WEBSOCKET_URL'] = f'localhost:{ws_port}' + + main_app.config['VWS_PORT'] = vws_port + main_app.config['VOICE_WEBSOCKET_URL'] = f'localhost:{vws_port}' # make sure we're calling the before_serving hooks event_loop.run_until_complete(main_app.startup()) From cbf6a3d441bbf103954c1b046ae2c83f1dea2805 Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 28 Feb 2019 23:14:56 -0300 Subject: [PATCH 04/55] run: remove unused loop variable --- run.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index f034d67..153e4eb 100644 --- a/run.py +++ b/run.py @@ -311,11 +311,13 @@ def start_websocket(host, port, ws_handler) -> asyncio.Future: @app.before_serving async def app_before_serving(): + """Callback for variable setup. + + Also sets up the websocket handlers. + """ log.info('opening db') await init_app_db(app) - loop = asyncio.get_event_loop() - app.session = ClientSession() init_app_managers(app) From 1bc04449bd97fed680ac9f12cb07d73f801c9133 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 02:25:22 -0300 Subject: [PATCH 05/55] add some dummy impl for vws v4 will be removed as this should be on the voice server instead. litecord manages simple voice state and dispatches voice state updates (such as speaking=1) --- litecord/errors.py | 10 ++ litecord/voice/errors.py | 54 +++++++++ litecord/voice/opcodes.py | 32 +++++ litecord/voice/websocket.py | 175 +++++++++++++++++++++++++++- litecord/voice/websocket_starter.py | 10 +- 5 files changed, 274 insertions(+), 7 deletions(-) create mode 100644 litecord/voice/errors.py create mode 100644 litecord/voice/opcodes.py diff --git a/litecord/errors.py b/litecord/errors.py index 1e9942a..64c0ebf 100644 --- a/litecord/errors.py +++ b/litecord/errors.py @@ -133,8 +133,18 @@ class MissingPermissions(Forbidden): class WebsocketClose(Exception): @property def code(self): + from_class = getattr(self, 'close_code', None) + + if from_class: + return from_class + return self.args[0] @property def reason(self): + from_class = getattr(self, 'close_code', None) + + if from_class: + return self.args[0] + return self.args[1] diff --git a/litecord/voice/errors.py b/litecord/voice/errors.py new file mode 100644 index 0000000..c30201f --- /dev/null +++ b/litecord/voice/errors.py @@ -0,0 +1,54 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +from litecord.errors import WebsocketClose + + +class UnknownOPCode(WebsocketClose): + close_code = 4000 + +class NotAuthenticated(WebsocketClose): + close_code = 4003 + +class AuthFailed(WebsocketClose): + close_code = 4004 + +class AlreadyAuth(WebsocketClose): + close_code = 4005 + +class InvalidSession(WebsocketClose): + close_code = 4006 + +class SessionTimeout(WebsocketClose): + close_code = 4009 + +class ServerNotFound(WebsocketClose): + close_code = 4011 + +class UnknownProtocol(WebsocketClose): + close_code = 4012 + +class Disconnected(WebsocketClose): + close_code = 4014 + +class VoiceServerCrash(WebsocketClose): + close_code = 4015 + +class UnknownEncryption(WebsocketClose): + close_code = 4016 diff --git a/litecord/voice/opcodes.py b/litecord/voice/opcodes.py new file mode 100644 index 0000000..ca1ca27 --- /dev/null +++ b/litecord/voice/opcodes.py @@ -0,0 +1,32 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +class VoiceOP: + identify = 0 + select_protocol = 1 + ready = 2 + heartbeat = 3 + session_description = 4 + speaking = 5 + heartbeat_ack = 6 + resume = 7 + hello = 8 + resumed = 9 + client_connect = 12 + client_disconnect = 13 diff --git a/litecord/voice/websocket.py b/litecord/voice/websocket.py index 48146af..5234b86 100644 --- a/litecord/voice/websocket.py +++ b/litecord/voice/websocket.py @@ -17,11 +17,182 @@ along with this program. If not, see . """ +import json +from dataclasses import dataclass, asdict, field + +import websockets.errors +from logbook import Logger + +from litecord.voice.opcodes import VoiceOP + +from litecord.errors import WebsocketClose +from litecord.voice.errors import ( + UnknownOPCode, AuthFailed, UnknownProtocol, InvalidSession +) + +from litecord.enums import ChannelType, VOICE_CHANNELS + +log = Logger(__name__) + + +@dataclass +class VoiceState: + """Store a voice websocket's state.""" + server_id: int + user_id: int + + def __bool__(self): + as_dict = asdict(self) + return all(bool(v) for v in as_dict.values()) + + class VoiceWebsocket: - """Voice websocket class.""" + """Voice websocket class. + + Implements the Discord Voice Websocket Protocol Version 4. + """ def __init__(self, ws, app): self.ws = ws self.app = app + self.voice = app.voice + self.storage = app.storage + + self.state = None + + async def send_op(self, opcode: VoiceOP, data: dict): + """Send a message through the websocket.""" + encoded = json.dumps({ + 'op': opcode, + 'd': data + }) + + await self.ws.send(encoded) + + async def _handle_0(self, msg: dict): + """Handle OP 0 Identify.""" + data = msg['d'] + + # NOTE: there is a data.video, but we don't handle video. + try: + server_id = int(data['server_id']) + user_id = int(data['user_id']) + session_id = data['session_id'] + token = data['token'] + except (KeyError, ValueError): + raise AuthFailed('Invalid identify payload') + + # server_id can be a: + # - voice channel id + # - dm id + # - group dm id + + channel = await self.storage.get_channel(server_id) + ctype = ChannelType(channel['type']) + + if ctype not in VOICE_CHANNELS: + raise AuthFailed('invalid channel id') + + v_user_id = await self.voice.authenticate(token, session_id) + + if v_user_id != user_id: + raise AuthFailed('invalid user id') + + await self.send_op(VoiceOP.hello, { + 'v': 4, + 'heartbeat_interval': 10000, + }) + + # TODO: get ourselves a place on the voice server + place = await self.voice.get_place(server_id) + + if not place: + raise InvalidSession('invalid voice place') + + self.state = VoiceState(place.server_id, user_id) + + await self.send_op(VoiceOP.ready, { + 'ssrc': place.ssrc, + 'port': place.port, + 'modes': place.modes, + 'ip': place.ip, + }) + + async def _handle_1(self, msg: dict): + """Handle 1 Select Protocol.""" + data = msg['d'] + + try: + protocol = data['protocol'] + proto_data = data['data'] + except KeyError: + raise UnknownProtocol('invalid select protocol') + + if protocol != 'udp': + raise UnknownProtocol('invalid protocol') + + try: + client_addr = proto_data['address'] + client_port = proto_data['port'] + client_mode = proto_data['mode'] + except KeyError: + raise UnknownProtocol('incomplete protocol data') + + # signal the voice server about (address, port) + mode + session = await self.voice.register( + self.state.server_id, + client_addr, client_port, client_mode + ) + + await self.send_op(VoiceOP.session_description, { + 'video_codec': 'VP8', + 'secret_key': session.key, + 'mode': session.mode, + 'media_session_id': session.sess_id, + 'audio_codec': 'opus' + }) + + async def _handle_3(self, msg: dict): + """Handle 3 Heartbeat.""" + await self.send_op(VoiceOP.heartbeat_ack, { + 'd': msg['d'] + }) + + async def _handle_5(self, msg: dict): + """Handle 5 Speaking.""" + if not self.state: + return + + await self.voice.update(self.state, msg['d']) + + async def _handle_7(self, msg: dict): + """Handle 7 Resume.""" + pass + + async def _process_msg(self): + msg = await self.ws.recv() + msg = json.loads(msg) + op_code = msg['op'] + + try: + handler = getattr(self, f'_handle_{op_code}') + except AttributeError: + raise UnknownOPCode('Unknown OP code.') + + await handler(msg) + + async def _loop(self): + while True: + await self._process_msg() async def run(self): - pass + """Main entry point for a voice websocket.""" + try: + await self._loop() + except websockets.exceptions.ConnectionClosed as err: + log.warning('conn close, state={}, err={}', self.state, err) + except WebsocketClose as err: + log.warning('ws close, state={} err={}', self.state, err) + await self.ws.close(code=err.code, reason=err.reason) + except Exception as err: + log.exception('An exception has occoured. state={}', self.state) + await self.ws.close(code=4000, reason=repr(err)) diff --git a/litecord/voice/websocket_starter.py b/litecord/voice/websocket_starter.py index 002835c..27640f5 100644 --- a/litecord/voice/websocket_starter.py +++ b/litecord/voice/websocket_starter.py @@ -20,6 +20,7 @@ along with this program. If not, see . import urllib.parse from litecord.voice.websocket import VoiceWebsocket + async def voice_websocket_handler(app, ws, url): """Main handler to instantiate a VoiceWebsocket with the given url.""" @@ -30,13 +31,12 @@ async def voice_websocket_handler(app, ws, url): try: gw_version = args['v'][0] except (KeyError, IndexError): - gw_version = '3' + gw_version = '4' - if gw_version not in ('1', '2', '3'): + if gw_version not in ('1', '2', '3', '4'): return await ws.close(1000, 'Invalid gateway version') - # TODO: select a different VoiceWebsocket runner depending on - # version. however i do not have docs on voice websockets - # earlier than v3. + # TODO: select a different VoiceWebsocket runner depending on the selected + # version. vws = VoiceWebsocket(ws, app) await vws.run() From 72704623253c87e51471db558b97302aa6c60dc8 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 03:12:50 -0300 Subject: [PATCH 06/55] add litecord voice server protocol (lvsp) draft --- docs/README.md | 3 ++ docs/lvsp/00-connection.md | 91 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 docs/README.md create mode 100644 docs/lvsp/00-connection.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..7476b25 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +# Internal documentation + +The Litecord Voice Server Protocol (LVSP) is documented here. diff --git a/docs/lvsp/00-connection.md b/docs/lvsp/00-connection.md new file mode 100644 index 0000000..fca637f --- /dev/null +++ b/docs/lvsp/00-connection.md @@ -0,0 +1,91 @@ +# Litecord Voice Server Protocol (LVSP) + +LVSP is a protocol for Litecord to communicate with an external component +dedicated for voice data. The voice server is responsible for the +Voice Websocket Discord and Voice UDP connections. + +LVSP runs over a websocket with TLS. The encoding is JSON. + +## High level + +In a high level: + - Litecord connects to the Voice Server via a URL already configured + beforehand. + - + +## OP code table + +"client" is litecord. "server" is the voice server. + +**TODO:** voice state management. + +| opcode | name | sent by | +| --: | :-- | :-- | +| 0 | HELLO | server | +| 1 | IDENTIFY | client | +| 2 | RESUME | client | +| 3 | READY | server | +| 4 | HEARTBEAT | client | +| 5 | HEARTBEAT\_ACK | server | + +## high level overview + + - connect, receive HELLO + - send IDENTIFY or RESUME + - receive READY + - start HEARTBEAT'ing + +## HELLO message + +Sent by the server when a connection is established. + +| field | type | description | +| --: | :-- | :-- | +| heartbeat\_interval | integer | amount of milliseconds to heartbeat with | + +## IDENTIFY message + +Sent by the client to identify itself. + +| field | type | description | +| --: | :-- | :-- | +| token | string | secret value kept between client and server | + +## RESUME message + +Sent by the client to resume itself from a failed websocket connection. + +The server will resend its data, then send a READY message. + +| field | type | description | +| --: | :-- | :-- | +| token | string | same value from IDENTIFY.token | +| seq | integer | last sequence number to resume from | + +## READY message + +**TODO** + +| field | type | description | +| --: | :-- | :-- | + +## HEARTBEAT message + +Sent by the client as a keepalive / health monitoring method. + +The server MUST reply with a HEARTBEAT\_ACK message back in a reasonable +time period. + +**TODO** + +| field | type | description | +| --: | :-- | :-- | + +## HEARTBEAT\_ACK message + +Sent by the server in reply to a HEARTBEAT message coming from the client. + +**TODO** + +| field | type | description | +| --: | :-- | :-- | From 2711924e03045f0b714444e37e24419be9daa401 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 04:33:09 -0300 Subject: [PATCH 07/55] move from lvsp/00-connecting.md to lvsp.md add some more stuff like INFO and VST\_REQUEST messages --- docs/lvsp.md | 169 ++++++++++++++++++++++++++++++++++++ docs/lvsp/00-connection.md | 91 ------------------- litecord/voice/websocket.py | 4 +- 3 files changed, 171 insertions(+), 93 deletions(-) create mode 100644 docs/lvsp.md delete mode 100644 docs/lvsp/00-connection.md diff --git a/docs/lvsp.md b/docs/lvsp.md new file mode 100644 index 0000000..375c54a --- /dev/null +++ b/docs/lvsp.md @@ -0,0 +1,169 @@ +# Litecord Voice Server Protocol (LVSP) + +LVSP is a protocol for Litecord to communicate with an external component +dedicated for voice data. The voice server is responsible for the +Voice Websocket Discord and Voice UDP connections. + +LVSP runs over a *long-lived* websocket with TLS. The encoding is JSON. + +**TODO:** common logic scenarios: + - initializing a voice channel + - updating a voice channel + - destroying a voice channel + - user joining to a voice channel + - user joining to a voice channel (while also initializing it, e.g + first member in the channel) + - user leaving a voice channel + +## OP code table + +"client" is litecord. "server" is the voice server. + +| opcode | name | sent by | +| --: | :-- | :-- | +| 0 | HELLO | server | +| 1 | IDENTIFY | client | +| 2 | RESUME | client | +| 3 | READY | server | +| 4 | HEARTBEAT | client | +| 5 | HEARTBEAT\_ACK | server | +| 6 | INFO | client / server | +| 7 | VST\_REQUEST | client | + +## Message structure + +Message data is defined by each opcode. + +**Note:** the `snowflake` type follows the same rules as the Discord Gateway's +snowflake type: A string encoding a Discord Snowflake. + +| field | type | description | +| --: | :-- | :-- | +| op | integer, opcode | operator code | +| d | map[string, any] | message data | + +## High level overview + + - connect, receive HELLO + - send IDENTIFY or RESUME + - if RESUME, process incoming messages as they were post-ready + - receive READY + - start HEARTBEAT'ing + - send INFO / VSU\_REQUEST messages as needed + +## Error codes + +| code | meaning | +| --: | :-- | +| 4000 | general error. Reconnect | +| 4001 | authentication failure | +| 4002 | decode error, given message failed to decode as json | + +## HELLO message + +Sent by the server when a connection is established. + +| field | type | description | +| --: | :-- | :-- | +| heartbeat\_interval | integer | amount of milliseconds to heartbeat with | + +## IDENTIFY message + +Sent by the client to identify itself. + +| field | type | description | +| --: | :-- | :-- | +| token | string | secret value kept between client and server | + +## RESUME message + +Sent by the client to resume itself from a failed websocket connection. + +The server will resend its data, then send a READY message. + +| field | type | description | +| --: | :-- | :-- | +| token | string | same value from IDENTIFY.token | +| seq | integer | last sequence number to resume from | + +## READY message + +**TODO:** does READY need any information? + +| field | type | description | +| --: | :-- | :-- | + +## HEARTBEAT message + +Sent by the client as a keepalive / health monitoring method. + +The server MUST reply with a HEARTBEAT\_ACK message back in a reasonable +time period. + +**TODO:** specify sequence numbers in INFO messages + +| field | type | description | +| --: | :-- | :-- | +| s | integer | sequence number | + +## HEARTBEAT\_ACK message + +Sent by the server in reply to a HEARTBEAT message coming from the client. + +**TODO:** add sequence numbers to ACK + +| field | type | description | +| --: | :-- | :-- | +| s | integer | sequence number | + +## INFO message + +Sent by either client or server on creation of update of a given object ( +such as a channel's bitrate setting or a user joining a channel). + +| field | type | description | +| --: | :-- | :-- | +| type | InfoType | info type | +| info | Union[ChannelInfo, VoiceInfo] | info object | + +### IntoType Enum + +| value | name | description | +| --: | :-- | :-- | +| 0 | CHANNEL | channel information | +| 1 | VST | Voice State | + +### ChannelInfo object + +| field | type | description | +| --: | :-- | :-- | +| id | snowflake | channel id | +| bitrate | integer | channel bitrate | + +### VoiceInfo object + +| field | type | description | +| --: | :-- | :-- | +| user\_id | snowflake | user id | +| channel\_id | snowflake | channel id | +| session\_id | string | session id | +| deaf | boolean | deaf status | +| mute | boolean | mute status | +| self\_deaf | boolean | self-deaf status | +| self\_mute | boolean | self-mute status | +| suppress | boolean | supress status | + +## VST\_REQUEST message + +Sent by the client to request the creation of a voice state in the voice server. + +**TODO:** verify correctness of this behavior. + +**TODO:** add logic on client connection + +The server SHALL send an INFO message containing the respective VoiceInfo data. + +| field | type | description | +| --: | :-- | :-- | +| user\_id | snowflake | user id for the voice state | +| channel\_id | snowflake | channel id for the voice state | diff --git a/docs/lvsp/00-connection.md b/docs/lvsp/00-connection.md deleted file mode 100644 index fca637f..0000000 --- a/docs/lvsp/00-connection.md +++ /dev/null @@ -1,91 +0,0 @@ -# Litecord Voice Server Protocol (LVSP) - -LVSP is a protocol for Litecord to communicate with an external component -dedicated for voice data. The voice server is responsible for the -Voice Websocket Discord and Voice UDP connections. - -LVSP runs over a websocket with TLS. The encoding is JSON. - -## High level - -In a high level: - - Litecord connects to the Voice Server via a URL already configured - beforehand. - - - -## OP code table - -"client" is litecord. "server" is the voice server. - -**TODO:** voice state management. - -| opcode | name | sent by | -| --: | :-- | :-- | -| 0 | HELLO | server | -| 1 | IDENTIFY | client | -| 2 | RESUME | client | -| 3 | READY | server | -| 4 | HEARTBEAT | client | -| 5 | HEARTBEAT\_ACK | server | - -## high level overview - - - connect, receive HELLO - - send IDENTIFY or RESUME - - receive READY - - start HEARTBEAT'ing - -## HELLO message - -Sent by the server when a connection is established. - -| field | type | description | -| --: | :-- | :-- | -| heartbeat\_interval | integer | amount of milliseconds to heartbeat with | - -## IDENTIFY message - -Sent by the client to identify itself. - -| field | type | description | -| --: | :-- | :-- | -| token | string | secret value kept between client and server | - -## RESUME message - -Sent by the client to resume itself from a failed websocket connection. - -The server will resend its data, then send a READY message. - -| field | type | description | -| --: | :-- | :-- | -| token | string | same value from IDENTIFY.token | -| seq | integer | last sequence number to resume from | - -## READY message - -**TODO** - -| field | type | description | -| --: | :-- | :-- | - -## HEARTBEAT message - -Sent by the client as a keepalive / health monitoring method. - -The server MUST reply with a HEARTBEAT\_ACK message back in a reasonable -time period. - -**TODO** - -| field | type | description | -| --: | :-- | :-- | - -## HEARTBEAT\_ACK message - -Sent by the server in reply to a HEARTBEAT message coming from the client. - -**TODO** - -| field | type | description | -| --: | :-- | :-- | diff --git a/litecord/voice/websocket.py b/litecord/voice/websocket.py index 5234b86..91241c5 100644 --- a/litecord/voice/websocket.py +++ b/litecord/voice/websocket.py @@ -18,9 +18,9 @@ along with this program. If not, see . """ import json -from dataclasses import dataclass, asdict, field +from dataclasses import dataclass, asdict -import websockets.errors +import websockets.exceptions from logbook import Logger from litecord.voice.opcodes import VoiceOP From f22a9b92a91cb9c001b98b6d699a3dc1696e1b9d Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 04:39:12 -0300 Subject: [PATCH 08/55] remove voice websocket fields from configs - run: remove vws spawn - tests.conftest: rollback to unused_tcp_port fixture --- config.ci.py | 6 ------ config.example.py | 6 ------ run.py | 10 ++-------- tests/conftest.py | 15 ++------------- 4 files changed, 4 insertions(+), 33 deletions(-) diff --git a/config.ci.py b/config.ci.py index 04f1fdd..24d6046 100644 --- a/config.ci.py +++ b/config.ci.py @@ -38,18 +38,12 @@ class Config: # will hit the websocket. # e.g 'gateway.example.com' for reverse proxies. WEBSOCKET_URL = 'localhost:5001' - VOICE_WEBSOCKET_URL = 'localhost:5002' # Where to host the websocket? # (a local address the server will bind to) WS_HOST = 'localhost' WS_PORT = 5001 - #: Where to host the VOICE websocket? - # (a local address the server will bind to) - VWS_HOST = 'localhost' - VWS_PORT = 5003 - # Postgres credentials POSTGRES = {} diff --git a/config.example.py b/config.example.py index ad6de74..e784e8e 100644 --- a/config.example.py +++ b/config.example.py @@ -46,18 +46,12 @@ class Config: # will hit the websocket. # e.g 'gateway.example.com' for reverse proxies. WEBSOCKET_URL = 'localhost:5001' - VOICE_WEBSOCKET_URL = 'localhost:5003' #: Where to host the websocket? # (a local address the server will bind to) WS_HOST = '0.0.0.0' WS_PORT = 5001 - #: Where to host the VOICE websocket? - # (a local address the server will bind to) - VWS_HOST = 'localhost' - VWS_PORT = 5003 - #: Mediaproxy URL on the internet # mediaproxy is made to prevent client IPs being leaked. MEDIA_PROXY = 'localhost:5002' diff --git a/run.py b/run.py index 153e4eb..c467aaa 100644 --- a/run.py +++ b/run.py @@ -72,7 +72,6 @@ from litecord.jobs import JobManager from litecord.voice.manager import VoiceManager from litecord.gateway import websocket_handler -from litecord.voice.websocket_starter import voice_websocket_handler from litecord.utils import LitecordJSONEncoder @@ -323,19 +322,14 @@ async def app_before_serving(): init_app_managers(app) await post_app_start(app) - # start gateway websocket and voice websocket + # start gateway websocket + # voice websocket is handled by the voice server ws_fut = start_websocket( app.config['WS_HOST'], app.config['WS_PORT'], websocket_handler ) - vws_fut = start_websocket( - app.config['VWS_HOST'], app.config['VWS_PORT'], - voice_websocket_handler - ) - await ws_fut - await vws_fut @app.after_serving diff --git a/tests/conftest.py b/tests/conftest.py index 0750091..66c5e6f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,30 +29,19 @@ sys.path.append(os.getcwd()) from run import app as main_app, set_blueprints -# pytest-sanic's unused_tcp_port can't be called twice since -# pytest fixtures etc etc. -def _unused_port(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(('127.0.0.1', 0)) - return sock.getsockname()[1] - @pytest.fixture(name='app') -def _test_app(event_loop): +def _test_app(unused_tcp_port, event_loop): set_blueprints(main_app) main_app.config['_testing'] = True # reassign an unused tcp port for websockets # since the config might give a used one. - ws_port, vws_port = _unused_port(), _unused_port() - print(ws_port, vws_port) + ws_port = unused_tcp_port main_app.config['WS_PORT'] = ws_port main_app.config['WEBSOCKET_URL'] = f'localhost:{ws_port}' - main_app.config['VWS_PORT'] = vws_port - main_app.config['VOICE_WEBSOCKET_URL'] = f'localhost:{vws_port}' - # make sure we're calling the before_serving hooks event_loop.run_until_complete(main_app.startup()) From 5d2869b53cf68c43d26588c58d86fd42ae40f840 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 04:48:05 -0300 Subject: [PATCH 09/55] remove voice websocket implementation voice websockets are left to the voice server itself. --- litecord/voice/errors.py | 54 -------- litecord/voice/opcodes.py | 32 ----- litecord/voice/websocket.py | 198 ---------------------------- litecord/voice/websocket_starter.py | 42 ------ 4 files changed, 326 deletions(-) delete mode 100644 litecord/voice/errors.py delete mode 100644 litecord/voice/opcodes.py delete mode 100644 litecord/voice/websocket.py delete mode 100644 litecord/voice/websocket_starter.py diff --git a/litecord/voice/errors.py b/litecord/voice/errors.py deleted file mode 100644 index c30201f..0000000 --- a/litecord/voice/errors.py +++ /dev/null @@ -1,54 +0,0 @@ -""" - -Litecord -Copyright (C) 2018-2019 Luna Mendes - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -""" - -from litecord.errors import WebsocketClose - - -class UnknownOPCode(WebsocketClose): - close_code = 4000 - -class NotAuthenticated(WebsocketClose): - close_code = 4003 - -class AuthFailed(WebsocketClose): - close_code = 4004 - -class AlreadyAuth(WebsocketClose): - close_code = 4005 - -class InvalidSession(WebsocketClose): - close_code = 4006 - -class SessionTimeout(WebsocketClose): - close_code = 4009 - -class ServerNotFound(WebsocketClose): - close_code = 4011 - -class UnknownProtocol(WebsocketClose): - close_code = 4012 - -class Disconnected(WebsocketClose): - close_code = 4014 - -class VoiceServerCrash(WebsocketClose): - close_code = 4015 - -class UnknownEncryption(WebsocketClose): - close_code = 4016 diff --git a/litecord/voice/opcodes.py b/litecord/voice/opcodes.py deleted file mode 100644 index ca1ca27..0000000 --- a/litecord/voice/opcodes.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - -Litecord -Copyright (C) 2018-2019 Luna Mendes - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -""" - -class VoiceOP: - identify = 0 - select_protocol = 1 - ready = 2 - heartbeat = 3 - session_description = 4 - speaking = 5 - heartbeat_ack = 6 - resume = 7 - hello = 8 - resumed = 9 - client_connect = 12 - client_disconnect = 13 diff --git a/litecord/voice/websocket.py b/litecord/voice/websocket.py deleted file mode 100644 index 91241c5..0000000 --- a/litecord/voice/websocket.py +++ /dev/null @@ -1,198 +0,0 @@ -""" - -Litecord -Copyright (C) 2018-2019 Luna Mendes - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -""" - -import json -from dataclasses import dataclass, asdict - -import websockets.exceptions -from logbook import Logger - -from litecord.voice.opcodes import VoiceOP - -from litecord.errors import WebsocketClose -from litecord.voice.errors import ( - UnknownOPCode, AuthFailed, UnknownProtocol, InvalidSession -) - -from litecord.enums import ChannelType, VOICE_CHANNELS - -log = Logger(__name__) - - -@dataclass -class VoiceState: - """Store a voice websocket's state.""" - server_id: int - user_id: int - - def __bool__(self): - as_dict = asdict(self) - return all(bool(v) for v in as_dict.values()) - - -class VoiceWebsocket: - """Voice websocket class. - - Implements the Discord Voice Websocket Protocol Version 4. - """ - def __init__(self, ws, app): - self.ws = ws - self.app = app - self.voice = app.voice - self.storage = app.storage - - self.state = None - - async def send_op(self, opcode: VoiceOP, data: dict): - """Send a message through the websocket.""" - encoded = json.dumps({ - 'op': opcode, - 'd': data - }) - - await self.ws.send(encoded) - - async def _handle_0(self, msg: dict): - """Handle OP 0 Identify.""" - data = msg['d'] - - # NOTE: there is a data.video, but we don't handle video. - try: - server_id = int(data['server_id']) - user_id = int(data['user_id']) - session_id = data['session_id'] - token = data['token'] - except (KeyError, ValueError): - raise AuthFailed('Invalid identify payload') - - # server_id can be a: - # - voice channel id - # - dm id - # - group dm id - - channel = await self.storage.get_channel(server_id) - ctype = ChannelType(channel['type']) - - if ctype not in VOICE_CHANNELS: - raise AuthFailed('invalid channel id') - - v_user_id = await self.voice.authenticate(token, session_id) - - if v_user_id != user_id: - raise AuthFailed('invalid user id') - - await self.send_op(VoiceOP.hello, { - 'v': 4, - 'heartbeat_interval': 10000, - }) - - # TODO: get ourselves a place on the voice server - place = await self.voice.get_place(server_id) - - if not place: - raise InvalidSession('invalid voice place') - - self.state = VoiceState(place.server_id, user_id) - - await self.send_op(VoiceOP.ready, { - 'ssrc': place.ssrc, - 'port': place.port, - 'modes': place.modes, - 'ip': place.ip, - }) - - async def _handle_1(self, msg: dict): - """Handle 1 Select Protocol.""" - data = msg['d'] - - try: - protocol = data['protocol'] - proto_data = data['data'] - except KeyError: - raise UnknownProtocol('invalid select protocol') - - if protocol != 'udp': - raise UnknownProtocol('invalid protocol') - - try: - client_addr = proto_data['address'] - client_port = proto_data['port'] - client_mode = proto_data['mode'] - except KeyError: - raise UnknownProtocol('incomplete protocol data') - - # signal the voice server about (address, port) + mode - session = await self.voice.register( - self.state.server_id, - client_addr, client_port, client_mode - ) - - await self.send_op(VoiceOP.session_description, { - 'video_codec': 'VP8', - 'secret_key': session.key, - 'mode': session.mode, - 'media_session_id': session.sess_id, - 'audio_codec': 'opus' - }) - - async def _handle_3(self, msg: dict): - """Handle 3 Heartbeat.""" - await self.send_op(VoiceOP.heartbeat_ack, { - 'd': msg['d'] - }) - - async def _handle_5(self, msg: dict): - """Handle 5 Speaking.""" - if not self.state: - return - - await self.voice.update(self.state, msg['d']) - - async def _handle_7(self, msg: dict): - """Handle 7 Resume.""" - pass - - async def _process_msg(self): - msg = await self.ws.recv() - msg = json.loads(msg) - op_code = msg['op'] - - try: - handler = getattr(self, f'_handle_{op_code}') - except AttributeError: - raise UnknownOPCode('Unknown OP code.') - - await handler(msg) - - async def _loop(self): - while True: - await self._process_msg() - - async def run(self): - """Main entry point for a voice websocket.""" - try: - await self._loop() - except websockets.exceptions.ConnectionClosed as err: - log.warning('conn close, state={}, err={}', self.state, err) - except WebsocketClose as err: - log.warning('ws close, state={} err={}', self.state, err) - await self.ws.close(code=err.code, reason=err.reason) - except Exception as err: - log.exception('An exception has occoured. state={}', self.state) - await self.ws.close(code=4000, reason=repr(err)) diff --git a/litecord/voice/websocket_starter.py b/litecord/voice/websocket_starter.py deleted file mode 100644 index 27640f5..0000000 --- a/litecord/voice/websocket_starter.py +++ /dev/null @@ -1,42 +0,0 @@ -""" - -Litecord -Copyright (C) 2018-2019 Luna Mendes - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -""" - -import urllib.parse -from litecord.voice.websocket import VoiceWebsocket - - -async def voice_websocket_handler(app, ws, url): - """Main handler to instantiate a VoiceWebsocket - with the given url.""" - args = urllib.parse.parse_qs( - urllib.parse.urlparse(url).query - ) - - try: - gw_version = args['v'][0] - except (KeyError, IndexError): - gw_version = '4' - - if gw_version not in ('1', '2', '3', '4'): - return await ws.close(1000, 'Invalid gateway version') - - # TODO: select a different VoiceWebsocket runner depending on the selected - # version. - vws = VoiceWebsocket(ws, app) - await vws.run() From e2258ed728b45fbfe585de32d78aa1e25d80a7f2 Mon Sep 17 00:00:00 2001 From: gabixdev Date: Fri, 1 Mar 2019 15:45:49 +0000 Subject: [PATCH 10/55] Make the secret actually secret, use HMAC for authentication --- docs/lvsp.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index 375c54a..3e94116 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -66,6 +66,7 @@ Sent by the server when a connection is established. | field | type | description | | --: | :-- | :-- | | heartbeat\_interval | integer | amount of milliseconds to heartbeat with | +| nonce | string | random 10-character string used as a message in HMAC authentication | ## IDENTIFY message @@ -73,7 +74,7 @@ Sent by the client to identify itself. | field | type | description | | --: | :-- | :-- | -| token | string | secret value kept between client and server | +| token | string | `HMAC(SHA256, key=[secret shared between server and client]), data=[nonce from HELLO]` | ## RESUME message From 919d8be2cbe6faaa82a3270072d8fdff36dacb8d Mon Sep 17 00:00:00 2001 From: gabixdev Date: Fri, 1 Mar 2019 15:48:29 +0000 Subject: [PATCH 11/55] Update lvsp.md --- docs/lvsp.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index 3e94116..744af0e 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -66,7 +66,7 @@ Sent by the server when a connection is established. | field | type | description | | --: | :-- | :-- | | heartbeat\_interval | integer | amount of milliseconds to heartbeat with | -| nonce | string | random 10-character string used as a message in HMAC authentication | +| nonce | string | random 10-character string used in authentication | ## IDENTIFY message @@ -74,7 +74,7 @@ Sent by the client to identify itself. | field | type | description | | --: | :-- | :-- | -| token | string | `HMAC(SHA256, key=[secret shared between server and client]), data=[nonce from HELLO]` | +| token | string | `HMAC(SHA256, key=[secret shared between server and client]), message=[nonce from HELLO]` | ## RESUME message From e22deb316f1c68e498605ad7010f5792aab3aec9 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 17:29:40 -0300 Subject: [PATCH 12/55] gateway.websocket: add voice state property fetch - gateway.websocket: handle when vsu.channel_id and vsu.guild_id are none --- litecord/gateway/websocket.py | 59 ++++++++++++++++++++++++++--------- litecord/storage.py | 6 ++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 5c5fe05..4967417 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -47,6 +47,8 @@ from litecord.gateway.errors import ( DecodeError, UnknownOPCode, InvalidShard, ShardingRequired ) +from litecord.storage import int_, bool_ + log = Logger(__name__) WebsocketProperties = collections.namedtuple( @@ -645,23 +647,60 @@ class GatewayWebsocket: # all checks passed. return True - async def _move_voice(self, guild_id, channel_id): + async def _vsu_get_prop(self, state, data): + """Get voice state properties from data, fallbacking to + user settings.""" + try: + # TODO: fetch from settings if not provided + self_deaf = bool(data['self_deaf']) + self_mute = bool(data['self_mute']) + except (KeyError, ValueError): + pass + + return { + 'deaf': state.deaf, + 'mute': state.mute, + 'self_deaf': self_deaf, + 'self_mute': self_mute, + } + + async def _move_voice(self, guild_id, channel_id, state, data): """Move an existing voice state to the given target.""" + # first case: consider when the user is leaving the + # voice channel. if channel_id is None: return await self.ext.voice.del_state(self.voice_key) + # second case: an update of voice state while being in + # the same channel + if channel_id == state.channel_id: + # we are moving to the same channel, so a simple update + # to the self_deaf / self_mute should suffice. + prop = await self._vsu_get_prop(state, data) + return await self.ext.voice.update_state( + self.voice_key, prop) + + # third case: moving between channels, check if the + # user can join the targeted channel first if not await self._voice_check(guild_id, channel_id): return + # if they can join, move the state to there. await self.ext.voice.move_state( self.voice_key, guild_id, channel_id) - async def _create_voice(self, guild_id, channel_id): + async def _create_voice(self, guild_id, channel_id, _state, data): """Create a voice state.""" + # we ignore the given existing state as it'll be basically + # none, lol. + + # check if we can join the channel if not await self._voice_check(guild_id, channel_id): return - await self.ext.voice.create_state(self.voice_key, guild_id, channel_id) + # if yes, create the state + await self.ext.voice.create_state( + self.voice_key, guild_id, channel_id, data) async def handle_4(self, payload: Dict[str, Any]): """Handle OP 4 Voice Status Update.""" @@ -670,24 +709,16 @@ class GatewayWebsocket: if not self.state: return - try: - channel_id = int(data['channel_id']) - guild_id = int(data['guild_id']) + channel_id = int_(data.get('channel_id')) + guild_id = int_(data.get('guild_id')) - # TODO: fetch from settings if not provided - # self_deaf = bool(data['self_deaf']) - # self_mute = bool(data['self_mute']) - - # NOTE: self_video is NOT handled. - except (KeyError, ValueError): - pass # fetch an existing voice state user_id, session_id = self.state.user_id, self.state.session_id voice_state = await self.ext.voice.fetch_state(user_id, session_id) func = self._move_voice if voice_state else self._create_voice - await func(guild_id, channel_id) + await func(guild_id, channel_id, voice_state, data) async def _handle_5(self, payload: Dict[str, Any]): """Handle OP 5 Voice Server Ping. diff --git a/litecord/storage.py b/litecord/storage.py index 83f98fc..d792025 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -52,6 +52,12 @@ def str_(val): return maybe(str, val) +def int_(val): + return maybe(int, val) + + +def bool_(val): + return maybe(int, val) def _filter_recipients(recipients: List[Dict[str, Any]], user_id: int): """Filter recipients in a list of recipients, removing the one that is reundant (ourselves).""" From f8b808c7b4217e8faa168d57da58b2da8dc2445c Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 17:30:44 -0300 Subject: [PATCH 13/55] remove unused import --- litecord/gateway/websocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 4967417..26fb5ef 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -47,7 +47,7 @@ from litecord.gateway.errors import ( DecodeError, UnknownOPCode, InvalidShard, ShardingRequired ) -from litecord.storage import int_, bool_ +from litecord.storage import int_ log = Logger(__name__) From 8cf6a28b584ddf21c7c4c14d92d9e9fee82236cb Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 17:32:21 -0300 Subject: [PATCH 14/55] storage: formatting --- litecord/storage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litecord/storage.py b/litecord/storage.py index d792025..7638853 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -58,6 +58,8 @@ def int_(val): def bool_(val): return maybe(int, val) + + def _filter_recipients(recipients: List[Dict[str, Any]], user_id: int): """Filter recipients in a list of recipients, removing the one that is reundant (ourselves).""" From ec738cd41e4c6c8d7b2b0ee9728b3a2e0f32646a Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 18:17:07 -0300 Subject: [PATCH 15/55] voice.manager: add some functions - add voice.state with VoiceState dataclass --- litecord/gateway/websocket.py | 7 ++-- litecord/voice/manager.py | 73 +++++++++++++++++++++++++++++++++++ litecord/voice/state.py | 52 +++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 litecord/voice/state.py diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 26fb5ef..74acee2 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -686,8 +686,8 @@ class GatewayWebsocket: return # if they can join, move the state to there. - await self.ext.voice.move_state( - self.voice_key, guild_id, channel_id) + # this will delete the old one and construct a new one. + await self.ext.voice.move_channels(self.voice_key, channel_id) async def _create_voice(self, guild_id, channel_id, _state, data): """Create a voice state.""" @@ -699,8 +699,7 @@ class GatewayWebsocket: return # if yes, create the state - await self.ext.voice.create_state( - self.voice_key, guild_id, channel_id, data) + await self.ext.voice.create_state(self.voice_key, channel_id, data) async def handle_4(self, payload: Dict[str, Any]): """Handle OP 4 Voice Status Update.""" diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 7457623..7a5c66f 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -17,7 +17,80 @@ along with this program. If not, see . """ +from typing import Tuple +from collections import defaultdict +from dataclasses import fields + +from logbook import Logger + +from litecord.voice.state import VoiceState + + +VoiceKey = Tuple[int, int] +log = Logger(__name__) + + +def _construct_state(state_dict: dict) -> VoiceState: + """Create a VoiceState instance out of a dictionary with the + VoiceState fields as keys.""" + fields = fields(VoiceState) + args = [state_dict[field.name] for field in fields] + return VoiceState(*args) + + class VoiceManager: """Main voice manager class.""" def __init__(self, app): self.app = app + + self.states = defaultdict(dict) + + # TODO: hold voice server LVSP connections + # TODO: map channel ids to voice servers + + async def state_count(self, channel_id: int) -> int: + """Get the current amount of voice states in a channel.""" + return len(self.states[channel_id]) + + async def del_state(self, voice_key: VoiceKey): + """Delete a given voice state.""" + chan_id, user_id = voice_key + + try: + # TODO: tell that to the voice server of the channel. + self.states[chan_id].pop(user_id) + except KeyError: + pass + + async def update_state(self, voice_key: VoiceKey, prop: dict): + """Update a state in a channel""" + chan_id, user_id = voice_key + + try: + state = self.states[chan_id][user_id] + except KeyError: + return + + # construct a new state based on the old one + properties + new_state_dict = dict(state.as_json) + + for field in prop: + # NOTE: this should not happen, ever. + if field in ('channel_id', 'user_id'): + raise ValueError('properties are updating channel or user') + + new_state_dict[field] = prop[field] + + new_state = _construct_state(new_state_dict) + + # TODO: dispatch to voice server + self.states[chan_id][user_id] = new_state + + async def move_channels(self, old_voice_key: VoiceKey, channel_id: int): + """Move a user between channels.""" + await self.del_state(old_voice_key) + await self.create_state(old_voice_key, channel_id, {}) + + async def create_state(self, voice_key: VoiceKey, channel_id: int, + data: dict): + pass diff --git a/litecord/voice/state.py b/litecord/voice/state.py new file mode 100644 index 0000000..72708af --- /dev/null +++ b/litecord/voice/state.py @@ -0,0 +1,52 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +from dataclasses import dataclass, asdict + + +@dataclass +class VoiceState: + """Represents a voice state.""" + channel_id: int + user_id: int + session_id: str + deaf: bool + mute: bool + self_deaf: bool + self_mute: bool + suppressed_by: int + + @property + def as_json(self): + """Return JSON-serializable dict.""" + return asdict(self) + + def as_json_for(self, user_id: int): + """Generate JSON-serializable version, given a user ID.""" + self_dict = asdict(self) + + # state.suppress is defined by the user + # that is currently viewing the state. + + # a better approach would be actually using + # the suppressed_by field for backend efficiency. + self_dict['suppress'] = user_id == self.suppressed_by + self_dict.pop('suppressed_by') + + return self_dict From 75a52a5ac8b2ae70a706903db8f3004cbe065c6f Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 18:25:22 -0300 Subject: [PATCH 16/55] storage: add voice state fetch --- litecord/storage.py | 25 ++++++++++++++++++++++--- litecord/voice/manager.py | 4 ++-- litecord/voice/state.py | 3 +++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/litecord/storage.py b/litecord/storage.py index 7638853..3911312 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -535,6 +535,27 @@ class Storage: return list(map(dict, roledata)) + async def guild_voice_states(self, guild_id: int, + user_id=None) -> List[Dict[str, Any]]: + """Get a list of voice states for the given guild.""" + channel_ids = await self.get_channel_ids(guild_id) + + res = [] + + for channel_id in channel_ids: + states = await self.app.voice.fetch_states(channel_id, user_id) + + jsonified = [s.as_json_for(user_id) for s in states] + + # discord does NOT insert guild_id to voice states on the + # guild voice state list. + for state in jsonified: + state.pop('guild_id') + + res.extend(jsonified) + + return res + async def get_guild_extra(self, guild_id: int, user_id=None, large=None) -> Dict: """Get extra information about a guild.""" @@ -575,9 +596,7 @@ class Storage: ), 'emojis': await self.get_guild_emojis(guild_id), - - # TODO: voice state management - 'voice_states': [], + 'voice_states': await self.guild_voice_states(guild_id), }} async def get_guild_full(self, guild_id: int, diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 7a5c66f..3b0c77c 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -33,8 +33,8 @@ log = Logger(__name__) def _construct_state(state_dict: dict) -> VoiceState: """Create a VoiceState instance out of a dictionary with the VoiceState fields as keys.""" - fields = fields(VoiceState) - args = [state_dict[field.name] for field in fields] + state_fields = fields(VoiceState) + args = [state_dict[field.name] for field in state_fields] return VoiceState(*args) diff --git a/litecord/voice/state.py b/litecord/voice/state.py index 72708af..adb3c20 100644 --- a/litecord/voice/state.py +++ b/litecord/voice/state.py @@ -41,6 +41,9 @@ class VoiceState: """Generate JSON-serializable version, given a user ID.""" self_dict = asdict(self) + if user_id is None: + return self_dict + # state.suppress is defined by the user # that is currently viewing the state. From 22402fd4cb02601e4fec9dc42a937cb7f86d6069 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 18:29:41 -0300 Subject: [PATCH 17/55] voice.manager: add fetch_states() impl - storage: follow the impl --- litecord/storage.py | 4 ++-- litecord/voice/manager.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/litecord/storage.py b/litecord/storage.py index 3911312..0d56c28 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -543,9 +543,9 @@ class Storage: res = [] for channel_id in channel_ids: - states = await self.app.voice.fetch_states(channel_id, user_id) + states = await self.app.voice.fetch_states(channel_id) - jsonified = [s.as_json_for(user_id) for s in states] + jsonified = [s.as_json_for(user_id) for s in states.values()] # discord does NOT insert guild_id to voice states on the # guild voice state list. diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 3b0c77c..569f607 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import Tuple +from typing import Tuple, Dict from collections import defaultdict from dataclasses import fields @@ -52,6 +52,15 @@ class VoiceManager: """Get the current amount of voice states in a channel.""" return len(self.states[channel_id]) + async def fetch_states(self, channel_id: int) -> Dict[int, VoiceState]: + """Fetch the states of the given channel.""" + # NOTE: maybe we *could* optimize by just returning a reference to the + # states dict instead of calling dict()... + + # however I'm really worried about state inconsistencies caused + # by this, so i'll just use dict(). + return dict(self.states[channel_id]) + async def del_state(self, voice_key: VoiceKey): """Delete a given voice state.""" chan_id, user_id = voice_key From d47d79977b133ff85e80719614ac0828e03d8c19 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 1 Mar 2019 18:35:46 -0300 Subject: [PATCH 18/55] gateway.websocket: handle chan_id=None - gateway.websocket: change fetch_state to get_state - voice.manager: add get_state() --- litecord/gateway/websocket.py | 13 ++++++++++--- litecord/voice/manager.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 74acee2..2fb6000 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -691,6 +691,15 @@ class GatewayWebsocket: async def _create_voice(self, guild_id, channel_id, _state, data): """Create a voice state.""" + + # if we are trying to create a voice state pointing torwards + # nowhere, we ignore it. + + # NOTE: HOWEVER, shouldn't we update the users' settings for + # self_mute and self_deaf? + if channel_id is None: + return + # we ignore the given existing state as it'll be basically # none, lol. @@ -711,10 +720,8 @@ class GatewayWebsocket: channel_id = int_(data.get('channel_id')) guild_id = int_(data.get('guild_id')) - # fetch an existing voice state - user_id, session_id = self.state.user_id, self.state.session_id - voice_state = await self.ext.voice.fetch_state(user_id, session_id) + voice_state = await self.ext.voice.get_state(self.voice_key) func = self._move_voice if voice_state else self._create_voice await func(guild_id, channel_id, voice_state, data) diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 569f607..e3301d1 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -61,6 +61,16 @@ class VoiceManager: # by this, so i'll just use dict(). return dict(self.states[channel_id]) + async def get_state(self, voice_key: VoiceKey) -> VoiceState: + """Get a single VoiceState for a user in a channel. Returns None + if no VoiceState is found.""" + channel_id, user_id = voice_key + + try: + return self.states[channel_id][user_id] + except KeyError: + return None + async def del_state(self, voice_key: VoiceKey): """Delete a given voice state.""" chan_id, user_id = voice_key From c6aea0b7b1cdecd4f09c83620bca38d88491ec7a Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 00:04:08 -0300 Subject: [PATCH 19/55] gateway.websocket: rewrite op 4 handler mostly we moved the permission checks from the websocket to the voice manager. - voice.manager: add leave_all(), leave() - voice.state add VoiceState.guild_id, VoiceState.key --- litecord/gateway/websocket.py | 143 ++++++++++++---------------------- litecord/storage.py | 2 +- litecord/voice/manager.py | 103 +++++++++++++++++++----- litecord/voice/state.py | 6 ++ litecord/voice/utils.py | 0 5 files changed, 142 insertions(+), 112 deletions(-) create mode 100644 litecord/voice/utils.py diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 2fb6000..e999875 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -30,12 +30,12 @@ from logbook import Logger import earl from litecord.auth import raw_token_check -from litecord.enums import RelationshipType, ChannelType, VOICE_CHANNELS +from litecord.enums import RelationshipType, ChannelType from litecord.schemas import validate, GW_STATUS_UPDATE from litecord.utils import ( task_wrapper, LitecordJSONEncoder, yield_chunks ) -from litecord.permissions import get_permissions, ALL_PERMISSIONS +from litecord.permissions import get_permissions from litecord.gateway.opcodes import OP from litecord.gateway.state import GatewayState @@ -603,50 +603,10 @@ class GatewayWebsocket: # setting new presence to state await self.update_status(presence) - @property - def voice_key(self): + def voice_key(self, channel_id: int, guild_id: int): """Voice state key.""" return (self.state.user_id, self.state.session_id) - async def _voice_check(self, guild_id: int, channel_id: int): - """Check if the user can join the given guild/channel pair.""" - guild = None - if guild_id: - guild = await self.storage.get_guild(guild_id) - - channel = await self.storage.get_channel(channel_id) - ctype = ChannelType(channel['type']) - - if ctype not in VOICE_CHANNELS: - return - - if guild and channel.get(['guild_id']) != guild['id']: - return - - is_guild_voice = ctype == ChannelType.GUILD_VOICE - - states = await self.ext.voice.state_count(channel_id) - perms = (ALL_PERMISSIONS - if not is_guild_voice else - await get_permissions(self.state.user_id, - channel_id, storage=self.storage) - ) - - is_full = states >= channel['user_limit'] - is_bot = self.state.bot - - is_manager = perms.bits.manage_channels - - # if the channel is full AND: - # - user is not a bot - # - user is not manage channels - # then it fails - if not is_bot and not is_manager and is_full: - return - - # all checks passed. - return True - async def _vsu_get_prop(self, state, data): """Get voice state properties from data, fallbacking to user settings.""" @@ -664,52 +624,6 @@ class GatewayWebsocket: 'self_mute': self_mute, } - async def _move_voice(self, guild_id, channel_id, state, data): - """Move an existing voice state to the given target.""" - # first case: consider when the user is leaving the - # voice channel. - if channel_id is None: - return await self.ext.voice.del_state(self.voice_key) - - # second case: an update of voice state while being in - # the same channel - if channel_id == state.channel_id: - # we are moving to the same channel, so a simple update - # to the self_deaf / self_mute should suffice. - prop = await self._vsu_get_prop(state, data) - return await self.ext.voice.update_state( - self.voice_key, prop) - - # third case: moving between channels, check if the - # user can join the targeted channel first - if not await self._voice_check(guild_id, channel_id): - return - - # if they can join, move the state to there. - # this will delete the old one and construct a new one. - await self.ext.voice.move_channels(self.voice_key, channel_id) - - async def _create_voice(self, guild_id, channel_id, _state, data): - """Create a voice state.""" - - # if we are trying to create a voice state pointing torwards - # nowhere, we ignore it. - - # NOTE: HOWEVER, shouldn't we update the users' settings for - # self_mute and self_deaf? - if channel_id is None: - return - - # we ignore the given existing state as it'll be basically - # none, lol. - - # check if we can join the channel - if not await self._voice_check(guild_id, channel_id): - return - - # if yes, create the state - await self.ext.voice.create_state(self.voice_key, channel_id, data) - async def handle_4(self, payload: Dict[str, Any]): """Handle OP 4 Voice Status Update.""" data = payload['d'] @@ -720,11 +634,54 @@ class GatewayWebsocket: channel_id = int_(data.get('channel_id')) guild_id = int_(data.get('guild_id')) - # fetch an existing voice state - voice_state = await self.ext.voice.get_state(self.voice_key) + # if its null and null, disconnect the user from any voice + # TODO: maybe just leave from DMs? idk... + if channel_id is None and guild_id is None: + await self.ext.voice.leave_all(self.state.user_id) - func = self._move_voice if voice_state else self._create_voice - await func(guild_id, channel_id, voice_state, data) + # if guild is not none but channel is, we are leaving + # a guild's channel + if channel_id is None: + await self.ext.voice.leave(guild_id, self.state.user_id) + + # fetch an existing state given user and guild OR user and channel + chan_type = ChannelType( + await self.storage.get_chan_type(channel_id) + ) + + state_id2 = channel_id + + if chan_type == ChannelType.GUILD_VOICE: + state_id2 = guild_id + + # a voice state key is a Tuple[int, int] + # - [0] is the user id + # - [1] is the channel id or guild id + + # the old approach was a (user_id, session_id), but + # that does not work. + + # this works since users can be connected to many channels + # using a single gateway websocket connection. HOWEVER, + # they CAN NOT enter two channels in a single guild. + + # this state id format takes care of that. + voice_key = (self.state.user_id, state_id2) + voice_state = await self.ext.voice.get_state(voice_key) + + if voice_state is None: + await self.ext.voice.create_state(voice_key) + + same_guild = guild_id == voice_state.guild_id + same_channel = channel_id == voice_state.channel_id + + prop = await self._vsu_get_prop(voice_state, data) + + if same_guild and same_channel: + await self.ext.voice.update_state(voice_state, prop) + + if same_guild and not same_channel: + await self.ext.voice.move_state(voice_state, channel_id) async def _handle_5(self, payload: Dict[str, Any]): """Handle OP 5 Voice Server Ping. diff --git a/litecord/storage.py b/litecord/storage.py index 0d56c28..0910a65 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -943,7 +943,7 @@ class Storage: return dm_chan - async def guild_from_channel(self, channel_id: int): + async def guild_from_channel(self, channel_id: int) -> int: """Get the guild id coming from a channel id.""" return await self.db.fetchval(""" SELECT guild_id diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index e3301d1..8b74293 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -23,6 +23,8 @@ from dataclasses import fields from logbook import Logger +from litecord.permissions import get_permissions +from litecord.enums import ChannelType, VOICE_CHANNELS from litecord.voice.state import VoiceState @@ -43,53 +45,95 @@ class VoiceManager: def __init__(self, app): self.app = app + # double dict, first key is guild/channel id, second key is user id self.states = defaultdict(dict) # TODO: hold voice server LVSP connections # TODO: map channel ids to voice servers + async def can_join(self, user_id: int, channel_id: int) -> int: + """Return if a user can join a channel.""" + + channel = await self.app.storage.get_channel(channel_id) + ctype = ChannelType(channel['type']) + + if ctype not in VOICE_CHANNELS: + return + + states = await self.app.voice.state_count(channel_id) + + # get_permissions returns ALL_PERMISSIONS when + # the channel isn't from a guild + perms = await get_permissions( + user_id, channel_id, storage=self.app.storage + ) + + # hacky user_limit but should work, as channels not + # in guilds won't have that field. + is_full = states >= channel.get('user_limit', 100) + is_bot = (await self.app.storage.get_user(user_id))['bot'] + is_manager = perms.bits.manage_channels + + # if the channel is full AND: + # - user is not a bot + # - user is not manage channels + # then it fails + if not is_bot and not is_manager and is_full: + return + + # all good + return True + async def state_count(self, channel_id: int) -> int: """Get the current amount of voice states in a channel.""" return len(self.states[channel_id]) async def fetch_states(self, channel_id: int) -> Dict[int, VoiceState]: """Fetch the states of the given channel.""" - # NOTE: maybe we *could* optimize by just returning a reference to the - # states dict instead of calling dict()... + # since the state key is (user_id, guild_id | channel_id), we need + # to determine which kind of search we want to do. + guild_id = await self.app.storage.guild_from_channel(channel_id) - # however I'm really worried about state inconsistencies caused - # by this, so i'll just use dict(). - return dict(self.states[channel_id]) + # if there isn't a guild for the channel, it is a dm or group dm. + # those are simple to handle. + if not guild_id: + return dict(self.states[channel_id]) + + # guild states hold a dict mapping user ids to guild states, + # same as channels, thats the structure. + guild_states = self.states[guild_id] + res = {} + + # iterate over all users with states and add the channel matches + # into res + for user_id, state in guild_states.items(): + if state.channel_id == channel_id: + res[user_id] = state + + return res async def get_state(self, voice_key: VoiceKey) -> VoiceState: """Get a single VoiceState for a user in a channel. Returns None if no VoiceState is found.""" - channel_id, user_id = voice_key + user_id, sec_key_id = voice_key try: - return self.states[channel_id][user_id] + return self.states[sec_key_id][user_id] except KeyError: return None async def del_state(self, voice_key: VoiceKey): """Delete a given voice state.""" - chan_id, user_id = voice_key + user_id, sec_key_id = voice_key try: # TODO: tell that to the voice server of the channel. - self.states[chan_id].pop(user_id) + self.states[sec_key_id].pop(user_id) except KeyError: pass - async def update_state(self, voice_key: VoiceKey, prop: dict): + async def update_state(self, state: VoiceState, prop: dict): """Update a state in a channel""" - chan_id, user_id = voice_key - - try: - state = self.states[chan_id][user_id] - except KeyError: - return - # construct a new state based on the old one + properties new_state_dict = dict(state.as_json) @@ -103,7 +147,7 @@ class VoiceManager: new_state = _construct_state(new_state_dict) # TODO: dispatch to voice server - self.states[chan_id][user_id] = new_state + self.states[state.key][state.user_id] = new_state async def move_channels(self, old_voice_key: VoiceKey, channel_id: int): """Move a user between channels.""" @@ -113,3 +157,26 @@ class VoiceManager: async def create_state(self, voice_key: VoiceKey, channel_id: int, data: dict): pass + + async def leave_all(self, user_id: int) -> int: + """Leave all voice channels.""" + + # iterate over every state finding matches + + # NOTE: we copy the current states dict since we're modifying + # on iteration. this is SLOW. + + # TODO: better solution instead of copying, maybe we can generate + # a list of tasks to run that actually do the deletion by themselves + # instead of us generating a delete. then only start running them later + # on. + for sec_key_id, states in dict(self.states): + for state in states: + if state.user_id != user_id: + continue + + await self.del_state((user_id, sec_key_id)) + + async def leave(self, guild_id: int, user_id: int): + """Make a user leave a channel IN A GUILD.""" + await self.del_state((guild_id, user_id)) diff --git a/litecord/voice/state.py b/litecord/voice/state.py index adb3c20..d5e8732 100644 --- a/litecord/voice/state.py +++ b/litecord/voice/state.py @@ -23,6 +23,7 @@ from dataclasses import dataclass, asdict @dataclass class VoiceState: """Represents a voice state.""" + guild_id: int channel_id: int user_id: int session_id: str @@ -32,6 +33,11 @@ class VoiceState: self_mute: bool suppressed_by: int + @property + def key(self): + """Get the second part of a key identifying a state.""" + return self.channel_id if self.guild_id is None else self.guild_id + @property def as_json(self): """Return JSON-serializable dict.""" diff --git a/litecord/voice/utils.py b/litecord/voice/utils.py new file mode 100644 index 0000000..e69de29 From 3a6073468e837fa090ee3f38cb75073432ab8003 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 00:05:33 -0300 Subject: [PATCH 20/55] rm litecord/voice/utils.py --- litecord/voice/utils.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 litecord/voice/utils.py diff --git a/litecord/voice/utils.py b/litecord/voice/utils.py deleted file mode 100644 index e69de29..0000000 From 307feacc7819f69a23f12c0ce896a6787e9e4cd4 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 00:13:32 -0300 Subject: [PATCH 21/55] gateway.websocket, voice.manager: quickfixes --- litecord/gateway/websocket.py | 4 ++-- litecord/voice/manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index e999875..9a4ddee 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -637,12 +637,12 @@ class GatewayWebsocket: # if its null and null, disconnect the user from any voice # TODO: maybe just leave from DMs? idk... if channel_id is None and guild_id is None: - await self.ext.voice.leave_all(self.state.user_id) + return await self.ext.voice.leave_all(self.state.user_id) # if guild is not none but channel is, we are leaving # a guild's channel if channel_id is None: - await self.ext.voice.leave(guild_id, self.state.user_id) + return await self.ext.voice.leave(guild_id, self.state.user_id) # fetch an existing state given user and guild OR user and channel chan_type = ChannelType( diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 8b74293..988e2e5 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -170,7 +170,7 @@ class VoiceManager: # a list of tasks to run that actually do the deletion by themselves # instead of us generating a delete. then only start running them later # on. - for sec_key_id, states in dict(self.states): + for sec_key_id, states in dict(self.states).items(): for state in states: if state.user_id != user_id: continue From 62d1252975bcdeb592974b68e0c8e8ef414c361a Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 02:40:39 -0300 Subject: [PATCH 22/55] lvsp: remove VST REQUEST in favour of a generic INFO message - lvsp: add sequence numbers - lvsp: add health scoring on heartbeats and ready - lvsp: change info table to allow for generic actions --- docs/lvsp.md | 100 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 36 deletions(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index 744af0e..22af56c 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -28,7 +28,6 @@ LVSP runs over a *long-lived* websocket with TLS. The encoding is JSON. | 4 | HEARTBEAT | client | | 5 | HEARTBEAT\_ACK | server | | 6 | INFO | client / server | -| 7 | VST\_REQUEST | client | ## Message structure @@ -41,6 +40,9 @@ snowflake type: A string encoding a Discord Snowflake. | --: | :-- | :-- | | op | integer, opcode | operator code | | d | map[string, any] | message data | +| s | Optional[int] | sequence number | + + - The `s` field is explained in the `RESUME` message. ## High level overview @@ -87,12 +89,27 @@ The server will resend its data, then send a READY message. | token | string | same value from IDENTIFY.token | | seq | integer | last sequence number to resume from | +### Sequence numbers + +Sequence numbers are used to resume a failed connection back and make the +voice server replay its missing events to the client. + +They are **positive** integers, **starting from 0.** There is no default +upper limit. A "long int" type in languages will probably be enough for most +use cases. + +Replayable messages MUST have sequence numbers embedded into the message +itself with a `s` field. The field lives at the root of the message, alongside +`op` and `d`. + ## READY message -**TODO:** does READY need any information? + - The `health` field is described with more detail in the `HEARTBEAT_ACK` + message. | field | type | description | | --: | :-- | :-- | +| `health` | Health | server health | ## HEARTBEAT message @@ -101,8 +118,6 @@ Sent by the client as a keepalive / health monitoring method. The server MUST reply with a HEARTBEAT\_ACK message back in a reasonable time period. -**TODO:** specify sequence numbers in INFO messages - | field | type | description | | --: | :-- | :-- | | s | integer | sequence number | @@ -111,60 +126,73 @@ time period. Sent by the server in reply to a HEARTBEAT message coming from the client. -**TODO:** add sequence numbers to ACK +The `health` field determines how well is the server's overall health. It is a +float going from 0 to 1, where 0 is the worst health possible, and 1 is the +best health possible. + +Servers SHOULD use the same algorithm to determine health, it CAN be based off: + - Machine resource usage (RAM, CPU, etc), however they're too general and can + be unreliable. + - Total users connected. + - Total bandwidth used in some X amount of time. + +Among others. | field | type | description | | --: | :-- | :-- | | s | integer | sequence number | +| health | float | server health | ## INFO message -Sent by either client or server on creation of update of a given object ( -such as a channel's bitrate setting or a user joining a channel). +Sent by either client or a server to send information between eachother. +The INFO message is extensible in which many request / response scenarios +are laid on. | field | type | description | | --: | :-- | :-- | | type | InfoType | info type | -| info | Union[ChannelInfo, VoiceInfo] | info object | +| data | Any | info data, varies depending on InfoType | -### IntoType Enum +### InfoType Enum | value | name | description | | --: | :-- | :-- | -| 0 | CHANNEL | channel information | -| 1 | VST | Voice State | +| 0 | CHANNEL\_REQ | channel assignment request | +| 1 | CHANNEL\_ASSIGN | channel assignment reply | +| 2 | CHANNEL\_UPDATE | channel update | +| 3 | CHANNEL\_DESTROY | channel destroy | +| 4 | VST\_CREATE | voice state create request | +| 5 | VST\_UPDATE | voice state update | +| 6 | VST\_LEAVE | voice state leave | -### ChannelInfo object +**TODO:** finish all infos -| field | type | description | -| --: | :-- | :-- | -| id | snowflake | channel id | -| bitrate | integer | channel bitrate | +### CHANNEL\_REQ -### VoiceInfo object +Request a channel to be created inside the voice server. -| field | type | description | -| --: | :-- | :-- | -| user\_id | snowflake | user id | -| channel\_id | snowflake | channel id | -| session\_id | string | session id | -| deaf | boolean | deaf status | -| mute | boolean | mute status | -| self\_deaf | boolean | self-deaf status | -| self\_mute | boolean | self-mute status | -| suppress | boolean | supress status | +The Server MUST reply back with a CHANNEL\_ASSIGN when resources are +allocated for the channel. -## VST\_REQUEST message +**TODO:** fields -Sent by the client to request the creation of a voice state in the voice server. +### CHANNEL\_ASSIGN -**TODO:** verify correctness of this behavior. +Sent by the Server to signal the successful creation of a voice channel. -**TODO:** add logic on client connection +**TODO:** fields -The server SHALL send an INFO message containing the respective VoiceInfo data. +### CHANNEL\_UPDATE -| field | type | description | -| --: | :-- | :-- | -| user\_id | snowflake | user id for the voice state | -| channel\_id | snowflake | channel id for the voice state | +Sent by the client to signal an update to the properties of a channel, +such as its bitrate. + +**TODO:** fields + +### CHANNEL\_DESTROY + +Sent by the client to signal the destruction of a voice channel. Be it +a channel being deleted, or all members in it leaving. + +**TODO:** fields From 287368ad1cae067299b0800003939a3d5cede642 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 04:30:41 -0300 Subject: [PATCH 23/55] litecord.voice: add LVSPManager, LVSPConnection --- litecord/voice/lvsp_conn.py | 23 +++++++++++++++++++++++ litecord/voice/lvsp_manager.py | 27 +++++++++++++++++++++++++++ litecord/voice/manager.py | 3 ++- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 litecord/voice/lvsp_conn.py create mode 100644 litecord/voice/lvsp_manager.py diff --git a/litecord/voice/lvsp_conn.py b/litecord/voice/lvsp_conn.py new file mode 100644 index 0000000..f700d2c --- /dev/null +++ b/litecord/voice/lvsp_conn.py @@ -0,0 +1,23 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + + +class LVSPConnection: + def __init__(self, app): + self.app = app diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py new file mode 100644 index 0000000..19e5098 --- /dev/null +++ b/litecord/voice/lvsp_manager.py @@ -0,0 +1,27 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +class LVSPManager: + """Manager class for Litecord Voice Server Protocol (LVSP) connections. + + Spawns :class:`LVSPConnection` as needed, etc. + """ + def __init__(self, app, voice): + self.app = app + self.voice = voice diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 988e2e5..0d3c687 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -26,6 +26,7 @@ from logbook import Logger from litecord.permissions import get_permissions from litecord.enums import ChannelType, VOICE_CHANNELS from litecord.voice.state import VoiceState +from litecord.voice.lvsp_manager import LVSPManager VoiceKey = Tuple[int, int] @@ -47,8 +48,8 @@ class VoiceManager: # double dict, first key is guild/channel id, second key is user id self.states = defaultdict(dict) + self.lvsp = LVSPManager(app, self) - # TODO: hold voice server LVSP connections # TODO: map channel ids to voice servers async def can_join(self, user_id: int, channel_id: int) -> int: From 287678331d4ec8b5532466c007bbf146b09a0e00 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 18:11:11 -0300 Subject: [PATCH 24/55] add voice_regions, voice_servers, and guild.region foreign key --- docs/lvsp.md | 2 +- .../scripts/11_voice_regions_servers.sql | 37 +++++++++++++++++++ schema.sql | 36 +++++++++++++++++- 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 manage/cmd/migration/scripts/11_voice_regions_servers.sql diff --git a/docs/lvsp.md b/docs/lvsp.md index 22af56c..de30b8a 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -126,7 +126,7 @@ time period. Sent by the server in reply to a HEARTBEAT message coming from the client. -The `health` field determines how well is the server's overall health. It is a +The `health` field is a measure of the servers's overall health. It is a float going from 0 to 1, where 0 is the worst health possible, and 1 is the best health possible. diff --git a/manage/cmd/migration/scripts/11_voice_regions_servers.sql b/manage/cmd/migration/scripts/11_voice_regions_servers.sql new file mode 100644 index 0000000..398c91a --- /dev/null +++ b/manage/cmd/migration/scripts/11_voice_regions_servers.sql @@ -0,0 +1,37 @@ +-- voice region data +-- NOTE: do NOT remove any rows. use deprectated=true and +-- DELETE FROM voice_servers instead. +CREATE TABLE IF NOT EXISTS voice_regions ( + -- always lowercase + id text PRIMARY KEY, + + -- "Russia", "Brazil", "Antartica", etc + name text NOT NULL, + + -- we don't have the concept of vip guilds yet, but better + -- future proof. + vip boolean DEFAULT FALSE, + + deprecated boolean DEFAULT FALSE, + + -- we don't have the concept of custom regions too. we don't have the + -- concept of official guilds either, but i'm keeping this in + custom boolean DEFAULT FALSE +); + +-- voice server pool. when someone wants to connect to voice, we choose +-- a server that is in the same region the guild is too, and choose the one +-- with the best health value +CREATE TABLE IF NOT EXISTS voice_servers ( + -- hostname is a reachable url, e.g "brazil2.example.com" + hostname text PRIMARY KEY, + region_id text REFERENCES voice_regions (id), + + -- health values are more thoroughly defined in the LVSP documentation + last_health float default 0.5 +); + + +ALTER TABLE guilds DROP COLUMN IF EXISTS region; +ALTER TABLE guilds ADD COLUMN + region text REFERENCES voice_regions (id); diff --git a/schema.sql b/schema.sql index 5eab831..269ff91 100644 --- a/schema.sql +++ b/schema.sql @@ -318,6 +318,40 @@ CREATE TABLE IF NOT EXISTS user_read_state ( PRIMARY KEY (user_id, channel_id) ); + +-- voice region data +-- NOTE: do NOT remove any rows. use deprectated=true and +-- DELETE FROM voice_servers instead. +CREATE TABLE IF NOT EXISTS voice_regions ( + -- always lowercase + id text PRIMARY KEY, + + -- "Russia", "Brazil", "Antartica", etc + name text NOT NULL, + + -- we don't have the concept of vip guilds yet, but better + -- future proof. + vip boolean DEFAULT FALSE, + + deprecated boolean DEFAULT FALSE, + + -- we don't have the concept of custom regions too. we don't have the + -- concept of official guilds either, but i'm keeping this in + custom boolean DEFAULT FALSE +); + +-- voice server pool. when someone wants to connect to voice, we choose +-- a server that is in the same region the guild is too, and choose the one +-- with the best health value +CREATE TABLE IF NOT EXISTS voice_servers ( + -- hostname is a reachable url, e.g "brazil2.example.com" + hostname text PRIMARY KEY, + region_id text REFERENCES voice_regions (id), + + -- health values are more thoroughly defined in the LVSP documentation + last_health float default 0.5 +); + CREATE TABLE IF NOT EXISTS guilds ( id bigint PRIMARY KEY NOT NULL, @@ -326,7 +360,7 @@ CREATE TABLE IF NOT EXISTS guilds ( splash text DEFAULT NULL, owner_id bigint NOT NULL REFERENCES users (id), - region text NOT NULL, + region text NOT NULL REFERENCES voice_regions (id), /* default no afk channel afk channel is voice-only. From 9d853f0bda2234233aaad8daee236401f3e5eec5 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 18:40:35 -0300 Subject: [PATCH 25/55] lvsp_manager: add basic region spawn code --- litecord/voice/lvsp_manager.py | 55 ++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py index 19e5098..8186a9f 100644 --- a/litecord/voice/lvsp_manager.py +++ b/litecord/voice/lvsp_manager.py @@ -17,6 +17,14 @@ along with this program. If not, see . """ +from collections import defaultdict + +from logbook import Logger + +from litecord.voice.lvsp_conn import LVSPConnection + +log = Logger(__name__) + class LVSPManager: """Manager class for Litecord Voice Server Protocol (LVSP) connections. @@ -25,3 +33,50 @@ class LVSPManager: def __init__(self, app, voice): self.app = app self.voice = voice + + self.servers = defaultdict(dict) + self.app.loop.create_task(self._spawn()) + + async def _spawn(self): + """Spawn LVSPConnection for each region.""" + + regions = await self.app.db.fetchval(""" + SELECT id + FROM voice_regions + WHERE deprecated = false + """) + + for region in regions: + self.app.loop.create_task( + self._spawn_region(region) + ) + + async def _spawn_region(self, region: str): + """Spawn a region. Involves fetching all the hostnames + for the regions and spawning a LVSPConnection for each.""" + servers = await self.app.db.fetch(""" + SELECT hostname + FROM voice_servers + WHERE region_id = $1 + """, region) + + if not servers: + log.warning('region {} does not have servers', region) + return + + servers = [r['hostname'] for r in servers] + + for hostname in servers: + conn = LVSPConnection(self, region, hostname) + self.servers[region][hostname] = conn + + self.app.loop.create_task( + conn.run() + ) + + async def del_conn(self, conn): + """Delete a connection from the connection pool.""" + try: + self.servers[conn.region].pop(conn.hostname) + except KeyError: + pass From 42cd0ae12cdde3ecb7e816394d6ef34315016b56 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 18:41:27 -0300 Subject: [PATCH 26/55] lvsp_manager: handle no region case --- litecord/voice/lvsp_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py index 8186a9f..08ffcd4 100644 --- a/litecord/voice/lvsp_manager.py +++ b/litecord/voice/lvsp_manager.py @@ -46,6 +46,10 @@ class LVSPManager: WHERE deprecated = false """) + if not regions: + log.warning('no regions are setup') + return + for region in regions: self.app.loop.create_task( self._spawn_region(region) From 7fdb74370e5756c443caff77b011156aeb598593 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 18:57:54 -0300 Subject: [PATCH 27/55] add better implementation for GET /voice/region this takes account of: - the majority region selected in all the guilds the user owns. if that fails: - the majority region on all the guilds the user is in. if that fails: - random region - storage: add Storage.all_voice_regions --- litecord/blueprints/voice.py | 88 ++++++++++++++++++++++++++++++++++-- litecord/storage.py | 9 ++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/litecord/blueprints/voice.py b/litecord/blueprints/voice.py index 85e4121..621a73c 100644 --- a/litecord/blueprints/voice.py +++ b/litecord/blueprints/voice.py @@ -17,13 +17,93 @@ along with this program. If not, see . """ -from quart import Blueprint, jsonify +from collections import Counter +from random import choice + +from quart import Blueprint, jsonify, current_app as app + +from litecord.blueprints.auth import token_check bp = Blueprint('voice', __name__) +def _majority_region_count(regions: list) -> str: + """Return the first most common element in a given list.""" + counter = Counter(regions) + common = counter.most_common(1) + region, _count = common[0] + + return region + + +async def _choose_random_region() -> str: + """Give a random voice region.""" + regions = await app.db.fetchval(""" + SELECT id + FROM voice_regions + """) + + regions = [r['id'] for r in regions] + + if not regions: + return None + + return choice(regions) + + +async def _majority_region_any(user_id) -> str: + """Calculate the most likely region to make the user happy, but + this is based on the guilds the user is IN, instead of the guilds + the user owns.""" + guilds = await app.storage.get_user_guilds(user_id) + + if not guilds: + return await _choose_random_region() + + res = [] + + for guild_id in guilds: + region = await app.db.fetchval(""" + SELECT region + FROM guilds + WHERE id = $1 + """, guild_id) + + res.append(region) + + most_common = _majority_region_count(res) + + if most_common is None: + return await _choose_random_region() + + return most_common + + +async def majority_region(user_id) -> str: + """Given a user ID, give the most likely region for the user to be + happy with.""" + regions = await app.db.fetch(""" + SELECT region + FROM guilds + WHERE owner_id = $1 + """, user_id) + + if not regions: + return await _majority_region_any(user_id) + + regions = [r['region'] for r in regions] + return _majority_region_count(regions) + + @bp.route('/regions', methods=['GET']) async def voice_regions(): - return jsonify([ - {'name': 'Brazil', 'deprecated': False, 'id': 'Brazil', 'optimal': True, 'vip': True} - ]) + """Return voice regions.""" + user_id = await token_check() + + best_region = await majority_region(user_id) + regions = await app.storage.get_all_regions() + + for region in regions: + region['optimal'] = region['id'] == best_region + + return jsonify(regions) diff --git a/litecord/storage.py b/litecord/storage.py index 0910a65..962bb57 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -1015,3 +1015,12 @@ class Storage: """, role_id) return [r['id'] for r in rows] + + async def all_voice_regions(self) -> List[Dict[str, Any]]: + """Return a list of all voice regions.""" + rows = await self.db.fetch(""" + SELECT id, name, vip, deprecated, custom + FROM voice_regions + """) + + return list(map(dict, rows)) From b71aec5aa11e83e676a089b7d0660e87e761ea8f Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 3 Mar 2019 20:16:17 -0300 Subject: [PATCH 28/55] add basic LVSP client implementation --- litecord/voice/lvsp_conn.py | 127 ++++++++++++++++++++++++++++++++- litecord/voice/lvsp_opcodes.py | 28 ++++++++ 2 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 litecord/voice/lvsp_opcodes.py diff --git a/litecord/voice/lvsp_conn.py b/litecord/voice/lvsp_conn.py index f700d2c..366188b 100644 --- a/litecord/voice/lvsp_conn.py +++ b/litecord/voice/lvsp_conn.py @@ -17,7 +17,130 @@ along with this program. If not, see . """ +import json +import asyncio + +import websockets +from logbook import Logger + +from litecord.voice.lvsp_opcodes import OPCodes as OP + +log = Logger(__name__) + class LVSPConnection: - def __init__(self, app): - self.app = app + """Represents a single LVSP connection.""" + def __init__(self, lvsp, region: str, hostname: str): + self.lvsp = lvsp + self.app = lvsp.app + + self.region = region + self.hostname = hostname + + self.conn = None + + self._hb_task = None + self._hb_interval = None + + @property + def _log_id(self): + return f'region={self.region} hostname={self.hostname}' + + async def send(self, payload): + """Send a payload down the websocket.""" + msg = json.dumps(payload) + await self.conn.send(msg) + + async def recv(self): + """Receive a payload.""" + msg = await self.conn.recv() + msg = json.dumps(msg) + return msg + + async def send_op(self, opcode: int, data: dict): + """Send a message with an OP code included""" + await self.send({ + 'op': opcode, + 'd': data + }) + + async def _heartbeater(self, hb_interval: int): + try: + await asyncio.sleep(hb_interval) + + # TODO: add self._seq + await self.send_op(OP.heartbeat, { + 's': 0 + }) + + # give the server 300 milliseconds to reply. + await asyncio.sleep(300) + await self.conn.close(4000, 'heartbeat timeout') + except asyncio.CancelledError: + pass + + def _start_hb(self): + self._hb_task = self.app.loop.create_task( + self._heartbeater(self._hb_interval) + ) + + def _stop_hb(self): + self._hb_task.cancel() + + async def _handle_0(self, msg): + """Handle HELLO message.""" + data = msg['d'] + + # nonce = data['nonce'] + self._hb_interval = data['heartbeat_interval'] + + # TODO: send identify + + async def _update_health(self, new_health: float): + """Update the health value of a given voice server.""" + await self.app.db.execute(""" + UPDATE voice_servers + SET health = $1 + WHERE hostname = $2 + """, new_health, self.hostname) + + async def _handle_3(self, msg): + """Handle READY message. + + We only start heartbeating after READY. + """ + await self._update_health(msg['health']) + self._start_hb() + + async def _handle_5(self, msg): + """Handle HEARTBEAT_ACK.""" + self._stop_hb() + await self._update_health(msg['health']) + self._start_hb() + + async def _loop(self): + while True: + msg = await self.recv() + + try: + opcode = msg['op'] + handler = getattr(self, f'_handle_{opcode}') + await handler(msg) + except (KeyError, AttributeError): + # TODO: error codes in LVSP + raise Exception('invalid op code') + + async def run(self): + """Start the websocket.""" + self.conn = await websockets.connect(f'wss://{self.hostname}') + + try: + await self._loop() + except websockets.exceptions.ConnectionClosed as err: + log.warning('conn close, {}, err={}', self._log_id, err) + # except WebsocketClose as err: + # log.warning('ws close, state={} err={}', self.state, err) + # await self.conn.close(code=err.code, reason=err.reason) + except Exception as err: + log.exception('An exception has occoured. {}', self._log_id) + await self.conn.close(code=4000, reason=repr(err)) diff --git a/litecord/voice/lvsp_opcodes.py b/litecord/voice/lvsp_opcodes.py new file mode 100644 index 0000000..4b19249 --- /dev/null +++ b/litecord/voice/lvsp_opcodes.py @@ -0,0 +1,28 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +class OPCodes: + """LVSP OP codes.""" + hello = 0 + identify = 1 + resume = 2 + ready = 3 + heartbeat = 4 + heartbeat_ack = 5 + info = 6 From cb029b36e335df8a1cdfb742b4ec7d11ed0cd748 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 00:37:56 -0300 Subject: [PATCH 29/55] add admin api (w/ voice endpoints) - add docs/admin_api.md - add litecord.voice_schemas - auth: add admin_check - voice.manager: add VoiceManager.voice_server_list --- docs/admin_api.md | 45 ++++++++++++ litecord/admin_schemas.py | 31 +++++++++ litecord/auth.py | 18 +++++ litecord/blueprints/admin_api/__init__.py | 22 ++++++ litecord/blueprints/admin_api/voice.py | 83 +++++++++++++++++++++++ litecord/voice/manager.py | 10 +++ run.py | 10 ++- 7 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 docs/admin_api.md create mode 100644 litecord/admin_schemas.py create mode 100644 litecord/blueprints/admin_api/__init__.py create mode 100644 litecord/blueprints/admin_api/voice.py diff --git a/docs/admin_api.md b/docs/admin_api.md new file mode 100644 index 0000000..3c4649c --- /dev/null +++ b/docs/admin_api.md @@ -0,0 +1,45 @@ +# Litecord Admin API + +the base path is `/api/v6/admin`. + +## GET `/voice/regions/` + +Return a list of voice server objects for the region. + +Returns empty list if the region does not exist. + +| field | type | description | +| --: | :-- | :-- | +| hostname | string | the hostname of the voice server | +| last\_health | float | the health of the voice server | + +## PUT `/voice/regions` + +Create a voice region. + +Receives JSON body as input, returns a list of voice region objects as output. + +| field | type | description | +| --: | :-- | :-- | +| id | string | id of the voice region, "brazil", "us-east", "eu-west", etc | +| name | string | name of the voice region | +| vip | Optional[bool] | if voice region is vip-only, default false | +| deprecated | Optional[bool] | if voice region is deprecated, default false | +| custom | Optional[bool] | if voice region is custom-only, default false | + +## PUT `/voice/regions//server` + +Create a voice server for a region. + +Returns empty body with 204 status code on success. + +| field | type | description | +| --: | :-- | :-- | +| hostname | string | the hostname of the voice server | + +## PUT `/voice/regions//deprecate` + +Mark a voice region as deprecated. Disables any voice actions on guilds that are +using the voice region. + +Returns empty body with 204 status code on success. diff --git a/litecord/admin_schemas.py b/litecord/admin_schemas.py new file mode 100644 index 0000000..3896dc9 --- /dev/null +++ b/litecord/admin_schemas.py @@ -0,0 +1,31 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +VOICE_SERVER = { + 'hostname': {'type': 'string', 'maxlength': 255, 'required': True} +} + +VOICE_REGION = { + 'id': {'type': 'string', 'maxlength': 255, 'required': True}, + 'name': {'type': 'string', 'maxlength': 255, 'required': True}, + + 'vip': {'type': 'boolean', 'default': False}, + 'deprecated': {'type': 'boolean', 'default': False}, + 'custom': {'type': 'boolean', 'default': False}, +} diff --git a/litecord/auth.py b/litecord/auth.py index 8294d7e..715865f 100644 --- a/litecord/auth.py +++ b/litecord/auth.py @@ -29,6 +29,7 @@ from quart import request, current_app as app from litecord.errors import Forbidden, Unauthorized, BadRequest from litecord.snowflake import get_snowflake +from litecord.enums import UserFlags log = Logger(__name__) @@ -100,6 +101,23 @@ async def token_check(): return user_id +async def admin_check(): + """Check if the user is an admin.""" + user_id = await token_check() + + flags = await app.db.fetchval(""" + SELECT flags + FROM users + WHERE id = $1 + """, user_id) + + flags = UserFlags.from_int(flags) + if not flags.is_staff: + raise Unauthorized('you are not staff') + + return user_id + + async def hash_data(data: str, loop=None) -> str: """Hash information with bcrypt.""" loop = loop or app.loop diff --git a/litecord/blueprints/admin_api/__init__.py b/litecord/blueprints/admin_api/__init__.py new file mode 100644 index 0000000..d209cf9 --- /dev/null +++ b/litecord/blueprints/admin_api/__init__.py @@ -0,0 +1,22 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +from .voice import bp as voice + +__all__ = ['voice'] diff --git a/litecord/blueprints/admin_api/voice.py b/litecord/blueprints/admin_api/voice.py new file mode 100644 index 0000000..e602972 --- /dev/null +++ b/litecord/blueprints/admin_api/voice.py @@ -0,0 +1,83 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +from quart import Blueprint, jsonify, current_app as app, request + +from litecord.auth import admin_check +from litecord.schemas import validate +from litecord.admin_schemas import VOICE_SERVER, VOICE_REGION + +bp = Blueprint('voice_admin', __name__) + + +@bp.route('/regions/', methods=['GET']) +async def get_region_servers(region): + """Return a list of all servers for a region.""" + _user_id = await admin_check() + servers = await app.voice.voice_server_list(region) + return jsonify(servers) + + +@bp.route('/regions', methods=['PUT']) +async def insert_new_region(): + """Create a voice region.""" + _user_id = await admin_check() + j = validate(await request.get_json(), VOICE_REGION) + + j['id'] = j['id'].lower() + + await app.db.execute(""" + INSERT INTO voice_regions (id, name, vip, deprecated, custom) + VALUES ($1, $2, $3, $4, $5) + """, j['id'], j['name'], j['vip'], j['deprecated'], j['custom']) + + return jsonify( + await app.storage.all_voice_regions() + ) + + +@bp.route('/regions//servers', methods=['PUT']) +async def put_region_server(region): + """Insert a voice server to a region""" + _user_id = await admin_check() + j = validate(await request.get_json(), VOICE_SERVER) + + await app.db.execute(""" + INSERT INTO voice_servers (hostname, region) + VALUES ($1, $2) + """, j['hostname'], region) + + return '', 204 + + +@bp.route('/regions//deprecate', methods=['PUT']) +async def deprecate_region(region): + """Deprecate a voice region.""" + _user_id = await admin_check() + + # TODO: write this + await app.voice.disable_region(region) + + await app.db.execute(""" + UPDATE voice_regions + SET deprecated = true + WHERE id = $1 + """, region) + + return '', 204 diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 0d3c687..535b6b2 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -181,3 +181,13 @@ class VoiceManager: async def leave(self, guild_id: int, user_id: int): """Make a user leave a channel IN A GUILD.""" await self.del_state((guild_id, user_id)) + + async def voice_server_list(self, region: str): + """Get a list of voice server objects""" + rows = await self.app.db.fetch(""" + SELECT hostname, last_health + FROM voice_servers + WHERE region_id = $1 + """, region) + + return list(map(dict, rows)) diff --git a/run.py b/run.py index c467aaa..ed3cdb9 100644 --- a/run.py +++ b/run.py @@ -54,8 +54,10 @@ from litecord.blueprints.user import ( user_settings, user_billing, fake_store ) -from litecord.blueprints.user.billing_job import ( - payment_job +from litecord.blueprints.user.billing_job import payment_job + +from litecord.blueprints.admin_api import ( + voice as voice_admin ) from litecord.ratelimits.handler import ratelimit_handler @@ -137,7 +139,9 @@ def set_blueprints(app_): icons: -1, attachments: -1, nodeinfo: -1, - static: -1 + static: -1, + + voice_admin: '/admin/voice' } for bp, suffix in bps.items(): From 547353e8a597841de053ea8566939bd324bb3ac7 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 00:48:32 -0300 Subject: [PATCH 30/55] admin_api.voice: remove unused vars --- litecord/blueprints/admin_api/voice.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litecord/blueprints/admin_api/voice.py b/litecord/blueprints/admin_api/voice.py index e602972..4d26488 100644 --- a/litecord/blueprints/admin_api/voice.py +++ b/litecord/blueprints/admin_api/voice.py @@ -29,7 +29,7 @@ bp = Blueprint('voice_admin', __name__) @bp.route('/regions/', methods=['GET']) async def get_region_servers(region): """Return a list of all servers for a region.""" - _user_id = await admin_check() + await admin_check() servers = await app.voice.voice_server_list(region) return jsonify(servers) @@ -37,7 +37,7 @@ async def get_region_servers(region): @bp.route('/regions', methods=['PUT']) async def insert_new_region(): """Create a voice region.""" - _user_id = await admin_check() + await admin_check() j = validate(await request.get_json(), VOICE_REGION) j['id'] = j['id'].lower() @@ -55,7 +55,7 @@ async def insert_new_region(): @bp.route('/regions//servers', methods=['PUT']) async def put_region_server(region): """Insert a voice server to a region""" - _user_id = await admin_check() + await admin_check() j = validate(await request.get_json(), VOICE_SERVER) await app.db.execute(""" @@ -69,7 +69,7 @@ async def put_region_server(region): @bp.route('/regions//deprecate', methods=['PUT']) async def deprecate_region(region): """Deprecate a voice region.""" - _user_id = await admin_check() + await admin_check() # TODO: write this await app.voice.disable_region(region) From 4a792966f1331244699e5b9933df8db0e81008ac Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 01:21:06 -0300 Subject: [PATCH 31/55] voice: fix bugs --- litecord/blueprints/voice.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litecord/blueprints/voice.py b/litecord/blueprints/voice.py index 621a73c..4d788cd 100644 --- a/litecord/blueprints/voice.py +++ b/litecord/blueprints/voice.py @@ -38,7 +38,7 @@ def _majority_region_count(regions: list) -> str: async def _choose_random_region() -> str: """Give a random voice region.""" - regions = await app.db.fetchval(""" + regions = await app.db.fetch(""" SELECT id FROM voice_regions """) @@ -55,7 +55,7 @@ async def _majority_region_any(user_id) -> str: """Calculate the most likely region to make the user happy, but this is based on the guilds the user is IN, instead of the guilds the user owns.""" - guilds = await app.storage.get_user_guilds(user_id) + guilds = await app.user_storage.get_user_guilds(user_id) if not guilds: return await _choose_random_region() @@ -101,7 +101,7 @@ async def voice_regions(): user_id = await token_check() best_region = await majority_region(user_id) - regions = await app.storage.get_all_regions() + regions = await app.storage.all_voice_regions() for region in regions: region['optimal'] = region['id'] == best_region From 2d7dc05453fbb1e46c4841abd50bf4fe17b75902 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 01:22:18 -0300 Subject: [PATCH 32/55] lvsp_manager: fix bug when fetching regions --- litecord/voice/lvsp_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py index 08ffcd4..2182859 100644 --- a/litecord/voice/lvsp_manager.py +++ b/litecord/voice/lvsp_manager.py @@ -40,12 +40,14 @@ class LVSPManager: async def _spawn(self): """Spawn LVSPConnection for each region.""" - regions = await self.app.db.fetchval(""" + regions = await self.app.db.fetch(""" SELECT id FROM voice_regions WHERE deprecated = false """) + regions = [r['id'] for r in regions] + if not regions: log.warning('no regions are setup') return From 9d80cb5564798f40936ac49a480ac243b882238f Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 01:26:53 -0300 Subject: [PATCH 33/55] admin_api.voice: handle UniqueViolationError --- litecord/blueprints/admin_api/voice.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/litecord/blueprints/admin_api/voice.py b/litecord/blueprints/admin_api/voice.py index 4d26488..b6e123a 100644 --- a/litecord/blueprints/admin_api/voice.py +++ b/litecord/blueprints/admin_api/voice.py @@ -17,11 +17,13 @@ along with this program. If not, see . """ +import asyncpg from quart import Blueprint, jsonify, current_app as app, request from litecord.auth import admin_check from litecord.schemas import validate from litecord.admin_schemas import VOICE_SERVER, VOICE_REGION +from litecord.errors import BadRequest bp = Blueprint('voice_admin', __name__) @@ -58,10 +60,13 @@ async def put_region_server(region): await admin_check() j = validate(await request.get_json(), VOICE_SERVER) - await app.db.execute(""" - INSERT INTO voice_servers (hostname, region) - VALUES ($1, $2) - """, j['hostname'], region) + try: + await app.db.execute(""" + INSERT INTO voice_servers (hostname, region_id) + VALUES ($1, $2) + """, j['hostname'], region) + except asyncpg.UniqueViolationError: + raise BadRequest('voice server already exists with given hostname') return '', 204 From 336f3a6eafcd3ed26735bead843de5f7ae3e765a Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 01:29:14 -0300 Subject: [PATCH 34/55] lvsp_conn: handle errors when connecting to voice server --- litecord/voice/lvsp_conn.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/litecord/voice/lvsp_conn.py b/litecord/voice/lvsp_conn.py index 366188b..744c9a6 100644 --- a/litecord/voice/lvsp_conn.py +++ b/litecord/voice/lvsp_conn.py @@ -130,11 +130,22 @@ class LVSPConnection: # TODO: error codes in LVSP raise Exception('invalid op code') + async def start(self): + """Try to start a websocket connection.""" + try: + self.conn = await websockets.connect(f'wss://{self.hostname}') + except Exception as e: + log.exception('failed to start lvsp conn to {}', self.hostname) + async def run(self): """Start the websocket.""" - self.conn = await websockets.connect(f'wss://{self.hostname}') + await self.start() try: + if not self.conn: + log.error('failed to start lvsp connection, stopping') + return + await self._loop() except websockets.exceptions.ConnectionClosed as err: log.warning('conn close, {}, err={}', self._log_id, err) From ad7f93b40a3c1df34f6b8695fbd09fadf200e636 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 03:00:46 -0300 Subject: [PATCH 35/55] add guild.region = null consistency checker as soon as a voice region is added, we should move all guilds to it. or if there are any guilds with a NULL region, we select one at random. region being NULL causes the client to be unable to change it, so.. - channels: fix update handlers for guild channels --- litecord/blueprints/admin_api/voice.py | 47 ++++++++++++++++++++++++-- litecord/blueprints/channels.py | 4 +-- run.py | 3 ++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/litecord/blueprints/admin_api/voice.py b/litecord/blueprints/admin_api/voice.py index b6e123a..e700b27 100644 --- a/litecord/blueprints/admin_api/voice.py +++ b/litecord/blueprints/admin_api/voice.py @@ -19,12 +19,14 @@ along with this program. If not, see . import asyncpg from quart import Blueprint, jsonify, current_app as app, request +from logbook import Logger from litecord.auth import admin_check from litecord.schemas import validate from litecord.admin_schemas import VOICE_SERVER, VOICE_REGION from litecord.errors import BadRequest +log = Logger(__name__) bp = Blueprint('voice_admin', __name__) @@ -49,9 +51,20 @@ async def insert_new_region(): VALUES ($1, $2, $3, $4, $5) """, j['id'], j['name'], j['vip'], j['deprecated'], j['custom']) - return jsonify( - await app.storage.all_voice_regions() - ) + regions = await app.storage.all_voice_regions() + region_count = len(regions) + + # if region count is 1, this is the first region to be created, + # so we should update all guilds to that region + if region_count == 1: + res = await app.db.execute(""" + UPDATE guilds + SET region = $1 + """, j['id']) + + log.info('updating guilds to first voice region: {}', res) + + return jsonify(regions) @bp.route('/regions//servers', methods=['PUT']) @@ -86,3 +99,31 @@ async def deprecate_region(region): """, region) return '', 204 + + +async def guild_region_check(app_): + """Check all guilds for voice region inconsistencies. + + Since the voice migration caused all guilds.region columns + to become NULL, we need to remove such NULLs if we have more + than one region setup. + """ + + regions = await app_.storage.all_voice_regions() + + if not regions: + log.info('region check: no regions to move guilds to') + return + + res = await app_.db.execute(""" + UPDATE guilds + SET region = ( + SELECT id + FROM voice_regions + OFFSET floor(random()*$1) + LIMIT 1 + ) + WHERE region = NULL + """, len(regions)) + + log.info('region check: updating guild.region=null: {!r}', res) diff --git a/litecord/blueprints/channels.py b/litecord/blueprints/channels.py index 2922bfb..f900ef6 100644 --- a/litecord/blueprints/channels.py +++ b/litecord/blueprints/channels.py @@ -393,7 +393,7 @@ async def _common_guild_chan(channel_id, j: dict): """, j[field], channel_id) -async def _update_text_channel(channel_id: int, j: dict): +async def _update_text_channel(channel_id: int, j: dict, _user_id: int): # first do the specific ones related to guild_text_channels for field in [field for field in j.keys() if field in ('topic', 'rate_limit_per_user')]: @@ -406,7 +406,7 @@ async def _update_text_channel(channel_id: int, j: dict): await _common_guild_chan(channel_id, j) -async def _update_voice_channel(channel_id: int, j: dict): +async def _update_voice_channel(channel_id: int, j: dict, _user_id: int): # first do the specific ones in guild_voice_channels for field in [field for field in j.keys() if field in ('bitrate', 'user_limit')]: diff --git a/run.py b/run.py index ed3cdb9..8dcfc51 100644 --- a/run.py +++ b/run.py @@ -60,6 +60,8 @@ from litecord.blueprints.admin_api import ( voice as voice_admin ) +from litecord.blueprints.admin_api.voice import guild_region_check + from litecord.ratelimits.handler import ratelimit_handler from litecord.ratelimits.main import RatelimitManager @@ -298,6 +300,7 @@ async def post_app_start(app_): # we'll need to start a billing job app_.sched.spawn(payment_job(app_)) app_.sched.spawn(api_index(app_)) + app_.sched.spawn(guild_region_check(app_)) def start_websocket(host, port, ws_handler) -> asyncio.Future: From 0759a520469fc073004efefe973abc8bd73259d3 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 03:16:28 -0300 Subject: [PATCH 36/55] pubsub.lazy_guild: s/pop/remove on remove_channel --- litecord/pubsub/lazy_guild.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litecord/pubsub/lazy_guild.py b/litecord/pubsub/lazy_guild.py index 3e9b7b3..da87367 100644 --- a/litecord/pubsub/lazy_guild.py +++ b/litecord/pubsub/lazy_guild.py @@ -1443,10 +1443,10 @@ class LazyGuildDispatcher(Dispatcher): # remove it from guild map as well guild_id = gml.guild_id - self.guild_map[guild_id].pop(channel_id) + self.guild_map[guild_id].remove(channel_id) gml.close() - except KeyError: + except (KeyError, ValueError): pass async def chan_update(self, channel_id: int): From 627d87b4a795424aec8d7ed3ea2514dc1371840c Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 03:20:34 -0300 Subject: [PATCH 37/55] lvsp_conn: remove unused var --- litecord/voice/lvsp_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litecord/voice/lvsp_conn.py b/litecord/voice/lvsp_conn.py index 744c9a6..a350737 100644 --- a/litecord/voice/lvsp_conn.py +++ b/litecord/voice/lvsp_conn.py @@ -134,7 +134,7 @@ class LVSPConnection: """Try to start a websocket connection.""" try: self.conn = await websockets.connect(f'wss://{self.hostname}') - except Exception as e: + except Exception: log.exception('failed to start lvsp conn to {}', self.hostname) async def run(self): From 506bd8afbe5b3fc780e6d855bc02f1504a899e5a Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 04:30:27 -0300 Subject: [PATCH 38/55] pipenv: add mypy --- Pipfile | 1 + Pipfile.lock | 332 ++++++++++++++++++++++++++++----------------------- 2 files changed, 185 insertions(+), 148 deletions(-) diff --git a/Pipfile b/Pipfile index 9c1a0f5..53bf257 100644 --- a/Pipfile +++ b/Pipfile @@ -19,6 +19,7 @@ aiohttp = "==3.4.4" pytest = "==3.10.1" pytest-asyncio = "==0.9.0" pyflakes = "*" +mypy = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 29c671a..8697992 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b546ad1edfe79457cb4da95e19fd17506b7adabe6a43acbc0906fb12cfda68b2" + "sha256": "4332c948a4bf656d0b95f4753eed6a3793dcb5ee9cd3538a443d7094a584e6cd" }, "pipfile-spec": 6, "requires": { @@ -82,10 +82,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "bcrypt": { "hashes": [ @@ -142,40 +142,36 @@ }, "cffi": { "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + "sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f", + "sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11", + "sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d", + "sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891", + "sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf", + "sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c", + "sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed", + "sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b", + "sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a", + "sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585", + "sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea", + "sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f", + "sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33", + "sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145", + "sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a", + "sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3", + "sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f", + "sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd", + "sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804", + "sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d", + "sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92", + "sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f", + "sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84", + "sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb", + "sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7", + "sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7", + "sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35", + "sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889" ], - "version": "==1.11.5" + "version": "==1.12.2" }, "chardet": { "hashes": [ @@ -213,10 +209,10 @@ }, "h2": { "hashes": [ - "sha256:4be613e35caad5680dc48f98f3bf4e7338c7c429e6375a5137be7fbe45219981", - "sha256:b2962f883fa392a23cbfcc4ad03c335bcc661be0cf9627657b589f0df2206e64" + "sha256:c8f387e0e4878904d4978cd688a3195f6b169d49b1ffa572a3d347d7adc5e09f", + "sha256:fd07e865a3272ac6ef195d8904de92dc7b38dc28297ec39cfa22716b6d62e6eb" ], - "version": "==3.0.1" + "version": "==3.1.0" }, "hpack": { "hashes": [ @@ -227,17 +223,17 @@ }, "hypercorn": { "hashes": [ - "sha256:3931144309c40341a46a2d054ac550bbd012a1f1a803774b5d6a3add90f52259", - "sha256:4df03fbc101efb4faf0b0883863ff7e620f94310e309311ceafaadb38ee1fa36" + "sha256:97bad5887ff543e2dff0a584d1a084e702789a26df5c8fb027ac1efab32274c5", + "sha256:b90799a1bc84f00ee999071e259f194087881b85c7240994ba9d86c4ceff3305" ], - "version": "==0.4.2" + "version": "==0.5.3" }, "hyperframe": { "hashes": [ - "sha256:87567c9eb1540de1e7f48805adf00e87856409342fdebd0cd20cf5d381c38b69", - "sha256:a25944539db36d6a2e47689e7915dcee562b3f8d10c6cdfa0d53c91ed692fb04" + "sha256:5187962cb16dcc078f23cb5a4b110098d546c3f41ff2d4038a9896893bbd0b40", + "sha256:a9f5c17f2cc3c719b917c4f33ed1c61bd1f8dfac4b1bd23b7c80b3400971b41f" ], - "version": "==5.1.0" + "version": "==5.2.0" }, "idna": { "hashes": [ @@ -273,36 +269,36 @@ }, "markupsafe": { "hashes": [ - "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", - "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", - "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", - "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", - "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", - "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", - "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", - "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", - "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", - "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", - "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", - "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", - "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", - "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", - "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", - "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", - "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", - "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", - "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", - "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", - "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", - "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", - "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", - "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", - "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", - "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", - "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", - "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" ], - "version": "==1.1.0" + "version": "==1.1.1" }, "multidict": { "hashes": [ @@ -340,39 +336,39 @@ }, "pillow": { "hashes": [ - "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360", - "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7", - "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e", - "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93", - "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c", - "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8", - "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b", - "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0", - "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd", - "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb", - "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581", - "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64", - "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace", - "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956", - "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60", - "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16", - "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37", - "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae", - "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a", - "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073", - "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8", - "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb", - "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409", - "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f", - "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2", - "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4", - "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74", - "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e", - "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1", - "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888" + "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e", + "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7", + "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a", + "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3", + "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1", + "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1", + "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7", + "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1", + "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3", + "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055", + "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf", + "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f", + "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f", + "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239", + "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe", + "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c", + "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697", + "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494", + "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356", + "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6", + "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000", + "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f", + "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c", + "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca", + "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8", + "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3", + "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad", + "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9", + "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc", + "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e" ], "index": "pypi", - "version": "==5.3.0" + "version": "==5.4.1" }, "pycparser": { "hashes": [ @@ -393,10 +389,10 @@ }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" }, "sortedcontainers": { "hashes": [ @@ -407,11 +403,11 @@ }, "typing-extensions": { "hashes": [ - "sha256:2a6c6e78e291a4b6cbd0bbfd30edc0baaa366de962129506ec8fe06bdec66457", - "sha256:51e7b7f3dcabf9ad22eed61490f3b8d23d9922af400fe6656cb08e66656b701f", - "sha256:55401f6ed58ade5638eb566615c150ba13624e2f0c1eedd080fc3c1b6cb76f1d" + "sha256:07b2c978670896022a43c4b915df8958bec4a6b84add7f2c87b2b728bda3ba64", + "sha256:f3f0e67e1d42de47b5c67c32c9b26641642e9170fe7e292991793705cd5fef7c", + "sha256:fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71" ], - "version": "==3.6.6" + "version": "==3.7.2" }, "websockets": { "hashes": [ @@ -442,70 +438,86 @@ }, "wsproto": { "hashes": [ - "sha256:1fcb726d448f1b9bcbea884e26621af5ddd01d2d502941a024f4c727828b6009", - "sha256:6a51cf18d9de612892b9c1d38a8c1bdadec0cfe15de61cd5c0f09174bf0c7e82" + "sha256:c013342d7a9180486713c6c986872e4fe24e18a21ccbece314939d8b58312e0e", + "sha256:fd6020d825022247053400306448e161d8740bdd52e328e5553cd9eee089f705" ], - "version": "==0.12.0" + "version": "==0.13.0" }, "yarl": { "hashes": [ - "sha256:2556b779125621b311844a072e0ed367e8409a18fa12cbd68eb1258d187820f9", - "sha256:4aec0769f1799a9d4496827292c02a7b1f75c0bab56ab2b60dd94ebb57cbd5ee", - "sha256:55369d95afaacf2fa6b49c84d18b51f1704a6560c432a0f9a1aeb23f7b971308", - "sha256:6c098b85442c8fe3303e708bbb775afd0f6b29f77612e8892627bcab4b939357", - "sha256:9182cd6f93412d32e009020a44d6d170d2093646464a88aeec2aef50592f8c78", - "sha256:c8cbc21bbfa1dd7d5386d48cc814fe3d35b80f60299cdde9279046f399c3b0d8", - "sha256:db6f70a4b09cde813a4807843abaaa60f3b15fb4a2a06f9ae9c311472662daa1", - "sha256:f17495e6fe3d377e3faac68121caef6f974fcb9e046bc075bcff40d8e5cc69a4", - "sha256:f85900b9cca0c67767bb61b2b9bd53208aaa7373dae633dbe25d179b4bf38aa7" + "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9", + "sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f", + "sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb", + "sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320", + "sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842", + "sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0", + "sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829", + "sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310", + "sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4", + "sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8", + "sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1" ], - "version": "==1.2.6" + "version": "==1.3.0" } }, "develop": { "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.1" + "version": "==1.3.0" }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", + "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" ], - "version": "==4.3.0" + "version": "==6.0.0" + }, + "mypy": { + "hashes": [ + "sha256:308c274eb8482fbf16006f549137ddc0d69e5a589465e37b99c4564414363ca7", + "sha256:e80fd6af34614a0e898a57f14296d0dacb584648f0339c2e000ddbf0f4cc2f8d" + ], + "index": "pypi", + "version": "==0.670" + }, + "mypy-extensions": { + "hashes": [ + "sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812", + "sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e" + ], + "version": "==0.4.1" }, "pluggy": { "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", + "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" ], - "version": "==0.8.0" + "version": "==0.9.0" }, "py": { "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" ], - "version": "==1.7.0" + "version": "==1.8.0" }, "pyflakes": { "hashes": [ - "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", - "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], "index": "pypi", - "version": "==2.0.0" + "version": "==2.1.1" }, "pytest": { "hashes": [ @@ -525,10 +537,34 @@ }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" + }, + "typed-ast": { + "hashes": [ + "sha256:035a54ede6ce1380599b2ce57844c6554666522e376bd111eb940fbc7c3dad23", + "sha256:037c35f2741ce3a9ac0d55abfcd119133cbd821fffa4461397718287092d9d15", + "sha256:049feae7e9f180b64efacbdc36b3af64a00393a47be22fa9cb6794e68d4e73d3", + "sha256:19228f7940beafc1ba21a6e8e070e0b0bfd1457902a3a81709762b8b9039b88d", + "sha256:2ea681e91e3550a30c2265d2916f40a5f5d89b59469a20f3bad7d07adee0f7a6", + "sha256:3a6b0a78af298d82323660df5497bcea0f0a4a25a0b003afd0ce5af049bd1f60", + "sha256:5385da8f3b801014504df0852bf83524599df890387a3c2b17b7caa3d78b1773", + "sha256:606d8afa07eef77280c2bf84335e24390055b478392e1975f96286d99d0cb424", + "sha256:69245b5b23bbf7fb242c9f8f08493e9ecd7711f063259aefffaeb90595d62287", + "sha256:6f6d839ab09830d59b7fa8fb6917023d8cb5498ee1f1dbd82d37db78eb76bc99", + "sha256:730888475f5ac0e37c1de4bd05eeb799fdb742697867f524dc8a4cd74bcecc23", + "sha256:9819b5162ffc121b9e334923c685b0d0826154e41dfe70b2ede2ce29034c71d8", + "sha256:9e60ef9426efab601dd9aa120e4ff560f4461cf8442e9c0a2b92548d52800699", + "sha256:af5fbdde0690c7da68e841d7fc2632345d570768ea7406a9434446d7b33b0ee1", + "sha256:b64efdbdf3bbb1377562c179f167f3bf301251411eb5ac77dec6b7d32bcda463", + "sha256:bac5f444c118aeb456fac1b0b5d14c6a71ea2a42069b09c176f75e9bd4c186f6", + "sha256:bda9068aafb73859491e13b99b682bd299c1b5fd50644d697533775828a28ee0", + "sha256:d659517ca116e6750101a1326107d3479028c5191f0ecee3c7203c50f5b915b0", + "sha256:eddd3fb1f3e0f82e5915a899285a39ee34ce18fd25d89582bc89fc9fb16cd2c6" + ], + "version": "==1.3.1" } } } From d91030a2c15fcacf50a42755170401d9f93ddd57 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 05:09:04 -0300 Subject: [PATCH 39/55] typings, episode 1 (i installed mypy and its beautiful) --- litecord/blueprints/channel/messages.py | 5 +-- litecord/blueprints/channels.py | 5 +-- litecord/blueprints/guild/members.py | 2 +- litecord/blueprints/guild/roles.py | 7 +++-- litecord/blueprints/voice.py | 7 +++-- litecord/gateway/websocket.py | 6 ++-- litecord/pubsub/channel.py | 6 ++-- litecord/pubsub/dispatcher.py | 7 ++--- litecord/pubsub/lazy_guild.py | 31 ++++++++++--------- litecord/schemas.py | 2 +- litecord/storage.py | 41 +++++++++++++------------ litecord/utils.py | 4 +-- 12 files changed, 65 insertions(+), 58 deletions(-) diff --git a/litecord/blueprints/channel/messages.py b/litecord/blueprints/channel/messages.py index fd8a060..41850e2 100644 --- a/litecord/blueprints/channel/messages.py +++ b/litecord/blueprints/channel/messages.py @@ -217,7 +217,8 @@ async def _guild_text_mentions(payload: dict, guild_id: int, # for the users that have a state # in the channel. if mentions_here: - uids = [] + uids = set() + await app.db.execute(""" UPDATE user_read_state SET mention_count = mention_count + 1 @@ -229,7 +230,7 @@ async def _guild_text_mentions(payload: dict, guild_id: int, # that might not have read permissions # to the channel. if mentions_everyone: - uids = [] + uids = set() member_ids = await app.storage.get_member_ids(guild_id) diff --git a/litecord/blueprints/channels.py b/litecord/blueprints/channels.py index f900ef6..5cfff5d 100644 --- a/litecord/blueprints/channels.py +++ b/litecord/blueprints/channels.py @@ -18,6 +18,7 @@ along with this program. If not, see . """ import time +from typing import List from quart import Blueprint, request, current_app as app, jsonify from logbook import Logger @@ -262,7 +263,7 @@ async def _update_pos(channel_id, pos: int): """, pos, channel_id) -async def _mass_chan_update(guild_id, channel_ids: int): +async def _mass_chan_update(guild_id, channel_ids: List[int]): for channel_id in channel_ids: chan = await app.storage.get_channel(channel_id) await app.dispatcher.dispatch( @@ -337,7 +338,7 @@ async def _update_channel_common(channel_id, guild_id: int, j: dict): if 'position' in j: channel_data = await app.storage.get_channel_data(guild_id) - chans = [None * len(channel_data)] + chans = [None] * len(channel_data) for chandata in channel_data: chans.insert(chandata['position'], int(chandata['id'])) diff --git a/litecord/blueprints/guild/members.py b/litecord/blueprints/guild/members.py index 8dd9204..3558168 100644 --- a/litecord/blueprints/guild/members.py +++ b/litecord/blueprints/guild/members.py @@ -68,7 +68,7 @@ async def get_members(guild_id): async def _update_member_roles(guild_id: int, member_id: int, - wanted_roles: list): + wanted_roles: set): """Update the roles a member has.""" # first, fetch all current roles diff --git a/litecord/blueprints/guild/roles.py b/litecord/blueprints/guild/roles.py index bd419cb..2a06c73 100644 --- a/litecord/blueprints/guild/roles.py +++ b/litecord/blueprints/guild/roles.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import List, Dict +from typing import List, Dict, Tuple from quart import Blueprint, request, current_app as app, jsonify from logbook import Logger @@ -184,10 +184,11 @@ async def _role_pairs_update(guild_id: int, pairs: list): await _role_update_dispatch(role_1, guild_id) await _role_update_dispatch(role_2, guild_id) +PairList = List[Tuple[Tuple[int, int], Tuple[int, int]]] def gen_pairs(list_of_changes: List[Dict[str, int]], current_state: Dict[int, int], - blacklist: List[int] = None) -> List[tuple]: + blacklist: List[int] = None) -> PairList: """Generate a list of pairs that, when applied to the database, will generate the desired state given in list_of_changes. @@ -262,7 +263,7 @@ def gen_pairs(list_of_changes: List[Dict[str, int]], # if its being swapped to leave space, add it # to the pairs list - if new_pos_2: + if element_2 and new_pos_2: pairs.append( ((element_1, new_pos_1), (element_2, new_pos_2)) ) diff --git a/litecord/blueprints/voice.py b/litecord/blueprints/voice.py index 4d788cd..a06eec1 100644 --- a/litecord/blueprints/voice.py +++ b/litecord/blueprints/voice.py @@ -17,6 +17,7 @@ along with this program. If not, see . """ +from typing import Optional from collections import Counter from random import choice @@ -36,7 +37,7 @@ def _majority_region_count(regions: list) -> str: return region -async def _choose_random_region() -> str: +async def _choose_random_region() -> Optional[str]: """Give a random voice region.""" regions = await app.db.fetch(""" SELECT id @@ -51,7 +52,7 @@ async def _choose_random_region() -> str: return choice(regions) -async def _majority_region_any(user_id) -> str: +async def _majority_region_any(user_id) -> Optional[str]: """Calculate the most likely region to make the user happy, but this is based on the guilds the user is IN, instead of the guilds the user owns.""" @@ -79,7 +80,7 @@ async def _majority_region_any(user_id) -> str: return most_common -async def majority_region(user_id) -> str: +async def majority_region(user_id: int) -> Optional[str]: """Given a user ID, give the most likely region for the user to be happy with.""" regions = await app.db.fetch(""" diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index 9a4ddee..e6fa7d4 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -235,7 +235,7 @@ class GatewayWebsocket: 's': None }) - def _check_ratelimit(self, key: str, ratelimit_key: str): + def _check_ratelimit(self, key: str, ratelimit_key): ratelimit = self.ext.ratelimiter.get_ratelimit(f'_ws.{key}') bucket = ratelimit.get_bucket(ratelimit_key) return bucket.update_rate_limit() @@ -292,7 +292,7 @@ class GatewayWebsocket: await self.send(payload) - async def _make_guild_list(self) -> List[int]: + async def _make_guild_list(self) -> List[Dict[str, Any]]: user_id = self.state.user_id guild_ids = await self._guild_ids() @@ -772,7 +772,7 @@ class GatewayWebsocket: await self._resume(range(seq, state.seq)) - async def _req_guild_members(self, guild_id: str, user_ids: List[int], + async def _req_guild_members(self, guild_id, user_ids: List[int], query: str, limit: int): try: guild_id = int(guild_id) diff --git a/litecord/pubsub/channel.py b/litecord/pubsub/channel.py index 2b32a1d..b931bec 100644 --- a/litecord/pubsub/channel.py +++ b/litecord/pubsub/channel.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import Any +from typing import Any, List from logbook import Logger @@ -54,13 +54,13 @@ class ChannelDispatcher(DispatcherWithState): VAL_TYPE = int async def dispatch(self, channel_id, - event: str, data: Any): + event: str, data: Any) -> List[str]: """Dispatch an event to a channel.""" # get everyone who is subscribed # and store the number of states we dispatched the event to user_ids = self.state[channel_id] dispatched = 0 - sessions = [] + sessions: List[str] = [] # making a copy of user_ids since # we'll modify it later on. diff --git a/litecord/pubsub/dispatcher.py b/litecord/pubsub/dispatcher.py index 4b60019..dd03ef2 100644 --- a/litecord/pubsub/dispatcher.py +++ b/litecord/pubsub/dispatcher.py @@ -17,9 +17,7 @@ along with this program. If not, see . """ -""" -litecord.pubsub.dispatcher: main dispatcher class -""" +from typing import List from collections import defaultdict from logbook import Logger @@ -82,7 +80,8 @@ class Dispatcher: """ raise NotImplementedError - async def _dispatch_states(self, states: list, event: str, data) -> int: + async def _dispatch_states(self, states: list, event: str, + data) -> List[str]: """Dispatch an event to a list of states.""" res = [] diff --git a/litecord/pubsub/lazy_guild.py b/litecord/pubsub/lazy_guild.py index da87367..93c15aa 100644 --- a/litecord/pubsub/lazy_guild.py +++ b/litecord/pubsub/lazy_guild.py @@ -28,7 +28,7 @@ lazy guilds: import asyncio from collections import defaultdict -from typing import Any, List, Dict, Union +from typing import Any, List, Dict, Union, Optional, Iterable, Tuple from dataclasses import dataclass, asdict, field from logbook import Logger @@ -39,7 +39,7 @@ from litecord.permissions import ( ) from litecord.utils import index_by_func from litecord.utils import mmh3 - +from litecord.gateway.state import GatewayState log = Logger(__name__) @@ -113,7 +113,7 @@ class MemberList: yield group, self.data[group.gid] @property - def iter_non_empty(self) -> tuple: + def iter_non_empty(self) -> Generator[Tuple[GroupInfo, List[int]]]: """Only iterate through non-empty groups. Note that while the offline group can be empty, it is always @@ -359,7 +359,7 @@ class GuildMemberList: # then the final perms for that role if # any overwrite exists in the channel final_perms = overwrite_find_mix( - role_perms, self.list.overwrites, group.gid) + role_perms, self.list.overwrites, int(group.gid)) # update the group's permissions # with the mixed ones @@ -423,7 +423,7 @@ class GuildMemberList: async def _get_group_for_member(self, member_id: int, roles: List[Union[str, int]], - status: str) -> GroupID: + status: str) -> Optional[GroupID]: """Return a fitting group ID for the member.""" member_roles = list(map(int, roles)) @@ -463,15 +463,15 @@ class GuildMemberList: self.list.members[member_id] = member self.list.data[group_id].append(member_id) - def _display_name(self, member_id: int) -> str: + def _display_name(self, member_id: int) -> Optional[str]: """Get the display name for a given member. This is more efficient than the old function (not method) of same name, as we dont need to pass nickname information to it. """ - member = self.list.members.get(member_id) - - if not member_id: + try: + member = self.list.members[member_id] + except KeyError: return None username = member['user']['username'] @@ -578,7 +578,7 @@ class GuildMemberList: if not self.state: self._set_empty_list() - def _get_state(self, session_id: str): + def _get_state(self, session_id: str) -> Optional[GatewayState]: """Get the state for a session id. Wrapper for :meth:`StateManager.fetch_raw` @@ -625,7 +625,8 @@ class GuildMemberList: return dispatched - async def _resync(self, session_ids: int, item_index: int) -> List[str]: + async def _resync(self, session_ids: List[int], + item_index: int) -> List[str]: """Send a SYNC event to all states that are subscribed to an item. Returns @@ -729,7 +730,7 @@ class GuildMemberList: # send SYNCs to the state that requested await self._dispatch_sess([session_id], ops) - def _get_item_index(self, user_id: Union[str, int]) -> int: + def _get_item_index(self, user_id: Union[str, int]) -> Optional[int]: """Get the item index a user is on.""" # NOTE: this is inefficient user_id = int(user_id) @@ -749,7 +750,7 @@ class GuildMemberList: return None - def _get_group_item_index(self, group_id: GroupID) -> int: + def _get_group_item_index(self, group_id: GroupID) -> Optional[int]: """Get the item index a group is on.""" index = 0 @@ -773,7 +774,7 @@ class GuildMemberList: return False - def _get_subs(self, item_index: int) -> filter: + def _get_subs(self, item_index: int) -> Iterable[str]: """Get the list of subscribed states to a given item.""" return filter( lambda sess_id: self._is_subbed(item_index, sess_id), @@ -1141,7 +1142,7 @@ class GuildMemberList: # when bots come along. self.list.data[new_group.gid] = [] - def _get_role_as_group_idx(self, role_id: int) -> int: + def _get_role_as_group_idx(self, role_id: int) -> Optional[int]: """Get a group index representing the given role id. Returns diff --git a/litecord/schemas.py b/litecord/schemas.py index 8b4fd28..3683334 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -147,7 +147,7 @@ class LitecordValidator(Validator): def validate(reqjson: Union[Dict, List], schema: Dict, - raise_err: bool = True) -> Union[Dict, List]: + raise_err: bool = True) -> Dict: """Validate a given document (user-input) and give the correct document as a result. """ diff --git a/litecord/storage.py b/litecord/storage.py index 962bb57..2b63e82 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional from logbook import Logger @@ -77,7 +77,7 @@ class Storage: self.db = app.db self.presence = None - async def fetchrow_with_json(self, query: str, *args): + async def fetchrow_with_json(self, query: str, *args) -> Any: """Fetch a single row with JSON/JSONB support.""" # the pool by itself doesn't have # set_type_codec, so we must set it manually @@ -86,19 +86,19 @@ class Storage: await pg_set_json(con) return await con.fetchrow(query, *args) - async def fetch_with_json(self, query: str, *args): + async def fetch_with_json(self, query: str, *args) -> List[Any]: """Fetch many rows with JSON/JSONB support.""" async with self.db.acquire() as con: await pg_set_json(con) return await con.fetch(query, *args) - async def execute_with_json(self, query: str, *args): + async def execute_with_json(self, query: str, *args) -> str: """Execute a SQL statement with JSON/JSONB support.""" async with self.db.acquire() as con: await pg_set_json(con) return await con.execute(query, *args) - async def get_user(self, user_id, secure=False) -> Dict[str, Any]: + async def get_user(self, user_id, secure=False) -> Optional[Dict[str, Any]]: """Get a single user payload.""" user_id = int(user_id) @@ -115,7 +115,7 @@ class Storage: """, user_id) if not user_row: - return + return None duser = dict(user_row) @@ -141,7 +141,7 @@ class Storage: """Search a user""" if len(discriminator) < 4: # how do we do this in f-strings again..? - discriminator = '%04d' % discriminator + discriminator = '%04d' % int(discriminator) return await self.db.fetchval(""" SELECT id FROM users @@ -219,12 +219,12 @@ class Storage: } async def get_member_data_one(self, guild_id: int, - member_id: int) -> Dict[str, Any]: + member_id: int) -> Optional[Dict[str, Any]]: """Get data about one member in a guild.""" basic = await self._member_basic(guild_id, member_id) if not basic: - return + return None return await self._member_dict(basic, guild_id, member_id) @@ -376,7 +376,7 @@ class Storage: return [r['member_id'] for r in user_ids] async def _gdm_recipients(self, channel_id: int, - reference_id: int = None) -> List[int]: + reference_id: int = None) -> List[Dict]: """Get the list of users that are recipients of the given Group DM.""" recipients = await self.gdm_recipient_ids(channel_id) @@ -392,7 +392,8 @@ class Storage: return res - async def get_channel(self, channel_id: int, **kwargs) -> Dict[str, Any]: + async def get_channel(self, channel_id: int, + **kwargs) -> Optional[Dict[str, Any]]: """Fetch a single channel's information.""" chan_type = await self.get_chan_type(channel_id) ctype = ChannelType(chan_type) @@ -501,7 +502,7 @@ class Storage: return channels async def get_role(self, role_id: int, - guild_id: int = None) -> Dict[str, Any]: + guild_id: int = None) -> Optional[Dict[str, Any]]: """get a single role's information.""" guild_field = 'AND guild_id = $2' if guild_id else '' @@ -519,7 +520,7 @@ class Storage: """, *args) if not row: - return + return None return dict(row) @@ -769,7 +770,8 @@ class Storage: res.pop('author_id') - async def get_message(self, message_id: int, user_id=None) -> Dict: + async def get_message(self, message_id: int, + user_id: Optional[int] = None) -> Optional[Dict]: """Get a single message's payload.""" row = await self.fetchrow_with_json(""" SELECT id::text, channel_id::text, author_id, webhook_id, content, @@ -780,7 +782,7 @@ class Storage: """, message_id) if not row: - return + return None res = dict(row) res['nonce'] = str(res['nonce']) @@ -915,7 +917,8 @@ class Storage: 'approximate_member_count': len(mids), } - async def get_invite_metadata(self, invite_code: str) -> Dict[str, Any]: + async def get_invite_metadata(self, + invite_code: str) -> Optional[Dict[str, Any]]: """Fetch invite metadata (max_age and friends).""" invite = await self.db.fetchrow(""" SELECT code, inviter, created_at, uses, @@ -925,7 +928,7 @@ class Storage: """, invite_code) if invite is None: - return + return None dinv = dict_(invite) inviter = await self.get_user(invite['inviter']) @@ -966,7 +969,7 @@ class Storage: return parties[0] - async def get_emoji(self, emoji_id: int) -> Dict: + async def get_emoji(self, emoji_id: int) -> Optional[Dict[str, Any]]: """Get a single emoji.""" row = await self.db.fetchrow(""" SELECT id::text, name, animated, managed, @@ -976,7 +979,7 @@ class Storage: """, emoji_id) if not row: - return + return None drow = dict(row) diff --git a/litecord/utils.py b/litecord/utils.py index 5647a37..1dca864 100644 --- a/litecord/utils.py +++ b/litecord/utils.py @@ -20,7 +20,7 @@ along with this program. If not, see . import asyncio import json from logbook import Logger -from typing import Any +from typing import Any, Iterable from quart.json import JSONEncoder log = Logger(__name__) @@ -160,7 +160,7 @@ async def pg_set_json(con): ) -def yield_chunks(input_list: list, chunk_size: int): +def yield_chunks(input_list: Iterable, chunk_size: int): """Yield successive n-sized chunks from l. Taken from https://stackoverflow.com/a/312464. From bbf80f72b13a270a0cab4098dbae8334ca0480c4 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 05:24:40 -0300 Subject: [PATCH 40/55] fix tests and an ugly circular import --- litecord/blueprints/invites.py | 2 +- litecord/gateway/__init__.py | 22 ---------------------- litecord/pubsub/lazy_guild.py | 4 ++-- run.py | 2 +- 4 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 litecord/gateway/__init__.py diff --git a/litecord/blueprints/invites.py b/litecord/blueprints/invites.py index f97bd65..f6fb585 100644 --- a/litecord/blueprints/invites.py +++ b/litecord/blueprints/invites.py @@ -91,7 +91,7 @@ async def invite_precheck_gdm(user_id: int, channel_id: int): async def _inv_check_age(inv: dict): - if inv['max_age'] is 0: + if inv['max_age'] != 0: return now = datetime.datetime.utcnow() diff --git a/litecord/gateway/__init__.py b/litecord/gateway/__init__.py deleted file mode 100644 index 3b62d06..0000000 --- a/litecord/gateway/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -""" - -Litecord -Copyright (C) 2018-2019 Luna Mendes - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -""" - -__all__ = ['websocket_handler'] - -from .gateway import websocket_handler diff --git a/litecord/pubsub/lazy_guild.py b/litecord/pubsub/lazy_guild.py index 93c15aa..df5c0dc 100644 --- a/litecord/pubsub/lazy_guild.py +++ b/litecord/pubsub/lazy_guild.py @@ -28,7 +28,7 @@ lazy guilds: import asyncio from collections import defaultdict -from typing import Any, List, Dict, Union, Optional, Iterable, Tuple +from typing import Any, List, Dict, Union, Optional, Iterable, Iterator, Tuple from dataclasses import dataclass, asdict, field from logbook import Logger @@ -113,7 +113,7 @@ class MemberList: yield group, self.data[group.gid] @property - def iter_non_empty(self) -> Generator[Tuple[GroupInfo, List[int]]]: + def iter_non_empty(self) -> Iterator[Tuple[GroupInfo, List[int]]]: """Only iterate through non-empty groups. Note that while the offline group can be empty, it is always diff --git a/run.py b/run.py index 8dcfc51..398896d 100644 --- a/run.py +++ b/run.py @@ -75,7 +75,7 @@ from litecord.images import IconManager from litecord.jobs import JobManager from litecord.voice.manager import VoiceManager -from litecord.gateway import websocket_handler +from litecord.gateway.gateway import websocket_handler from litecord.utils import LitecordJSONEncoder From 68c6442375ad622137cff433fae1a7d7ee81e89d Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 15:12:38 -0300 Subject: [PATCH 41/55] storage: fix more type hints, handle None on more places --- litecord/storage.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/litecord/storage.py b/litecord/storage.py index 2b63e82..d105aca 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -60,11 +60,9 @@ def bool_(val): return maybe(int, val) -def _filter_recipients(recipients: List[Dict[str, Any]], user_id: int): +def _filter_recipients(recipients: List[Dict[str, Any]], user_id: str): """Filter recipients in a list of recipients, removing the one that is reundant (ourselves).""" - user_id = str(user_id) - return list(filter( lambda recipient: recipient['id'] != user_id, recipients)) @@ -148,7 +146,7 @@ class Storage: WHERE username = $1 AND discriminator = $2 """, username, discriminator) - async def get_guild(self, guild_id: int, user_id=None) -> Dict: + async def get_guild(self, guild_id: int, user_id=None) -> Optional[Dict]: """Get gulid payload.""" row = await self.db.fetchrow(""" SELECT id::text, owner_id::text, name, icon, splash, @@ -163,7 +161,7 @@ class Storage: """, guild_id) if not row: - return + return None drow = dict(row) @@ -180,7 +178,7 @@ class Storage: """, guild_id, member_id) async def get_member_role_ids(self, guild_id: int, - member_id: int) -> List[int]: + member_id: int) -> List[str]: """Get a list of role IDs that are on a member.""" roles = await self.db.fetch(""" SELECT role_id::text @@ -322,6 +320,7 @@ class Storage: return {**row, **dict(vrow)} log.warning('unknown channel type: {}', chan_type) + return row async def get_chan_type(self, channel_id: int) -> int: """Get the channel type integer, given channel ID.""" @@ -386,9 +385,12 @@ class Storage: if user_id == reference_id: continue - res.append( - await self.get_user(user_id) - ) + user = await self.get_user(user_id) + + if user is None: + continue + + res.append(user) return res @@ -600,13 +602,17 @@ class Storage: 'voice_states': await self.guild_voice_states(guild_id), }} - async def get_guild_full(self, guild_id: int, - user_id: int, large_count: int = 250) -> Dict: + async def get_guild_full(self, guild_id: int, user_id: int, + large_count: int = 250) -> Optional[Dict]: """Get full information on a guild. This is a very expensive operation. """ guild = await self.get_guild(guild_id, user_id) + + if guild is None: + return None + extra = await self.get_guild_extra(guild_id, user_id, large_count) return {**guild, **extra} @@ -856,7 +862,7 @@ class Storage: return res - async def get_invite(self, invite_code: str) -> dict: + async def get_invite(self, invite_code: str) -> Optional[Dict]: """Fetch invite information given its code.""" invite = await self.db.fetchrow(""" SELECT code, guild_id, channel_id @@ -885,6 +891,10 @@ class Storage: dinv['guild'] = {} chan = await self.get_channel(invite['channel_id']) + + if chan is None: + return None + dinv['channel'] = { 'id': chan['id'], 'name': chan['name'], @@ -936,12 +946,13 @@ class Storage: return dinv - async def get_dm(self, dm_id: int, user_id: int = None): + async def get_dm(self, dm_id: int, user_id: int = None) -> Optional[Dict]: + """Get a DM channel.""" dm_chan = await self.get_channel(dm_id) - if user_id: + if user_id and dm_chan: dm_chan['recipients'] = _filter_recipients( - dm_chan['recipients'], user_id + dm_chan['recipients'], str(user_id) ) return dm_chan From ed3c436b6dfd777b1fa759563ec36a04aa273f2a Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 15:19:05 -0300 Subject: [PATCH 42/55] channels, guild.roles, user.billing: typing fixes --- litecord/blueprints/channels.py | 7 +++++-- litecord/blueprints/guild/roles.py | 5 ++++- litecord/blueprints/user/billing.py | 2 -- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/litecord/blueprints/channels.py b/litecord/blueprints/channels.py index 5cfff5d..2690440 100644 --- a/litecord/blueprints/channels.py +++ b/litecord/blueprints/channels.py @@ -18,7 +18,7 @@ along with this program. If not, see . """ import time -from typing import List +from typing import List, Optional from quart import Blueprint, request, current_app as app, jsonify from logbook import Logger @@ -263,8 +263,11 @@ async def _update_pos(channel_id, pos: int): """, pos, channel_id) -async def _mass_chan_update(guild_id, channel_ids: List[int]): +async def _mass_chan_update(guild_id, channel_ids: List[Optional[int]]): for channel_id in channel_ids: + if channel_id is None: + continue + chan = await app.storage.get_channel(channel_id) await app.dispatcher.dispatch( 'guild', guild_id, 'CHANNEL_UPDATE', chan) diff --git a/litecord/blueprints/guild/roles.py b/litecord/blueprints/guild/roles.py index 2a06c73..5dfcbe7 100644 --- a/litecord/blueprints/guild/roles.py +++ b/litecord/blueprints/guild/roles.py @@ -257,13 +257,16 @@ def gen_pairs(list_of_changes: List[Dict[str, int]], # position we want to change to element_2 = current_state.get(new_pos_1) + if element_2 is None: + continue + # if there is, is that existing channel being # swapped to another position? new_pos_2 = preferred_state.get(element_2) # if its being swapped to leave space, add it # to the pairs list - if element_2 and new_pos_2: + if new_pos_2 is not None: pairs.append( ((element_1, new_pos_1), (element_2, new_pos_2)) ) diff --git a/litecord/blueprints/user/billing.py b/litecord/blueprints/user/billing.py index eb4de46..f37744e 100644 --- a/litecord/blueprints/user/billing.py +++ b/litecord/blueprints/user/billing.py @@ -148,8 +148,6 @@ async def get_payment_source(user_id: int, source_id: int, db=None) -> dict: if not db: db = app.db - source = {} - source_type = await db.fetchval(""" SELECT source_type FROM user_payment_sources From 9dab5b20ae685f9257ee66d8ffd6251c0f4f841b Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 15:48:51 -0300 Subject: [PATCH 43/55] typing, episode 2 --- litecord/blueprints/channel/dm_checks.py | 8 ++--- litecord/embed/sanitizer.py | 14 ++++----- litecord/images.py | 38 ++++++++++++++++++------ litecord/permissions.py | 9 ++++-- litecord/presence.py | 2 +- litecord/pubsub/channel.py | 2 +- litecord/pubsub/lazy_guild.py | 22 +++++++++----- litecord/utils.py | 9 +++--- 8 files changed, 68 insertions(+), 36 deletions(-) diff --git a/litecord/blueprints/channel/dm_checks.py b/litecord/blueprints/channel/dm_checks.py index 743815d..1d0851a 100644 --- a/litecord/blueprints/channel/dm_checks.py +++ b/litecord/blueprints/channel/dm_checks.py @@ -58,11 +58,11 @@ async def dm_pre_check(user_id: int, channel_id: int, peer_id: int): user_settings = await app.user_storage.get_user_settings(user_id) peer_settings = await app.user_storage.get_user_settings(peer_id) - restricted_user = [int(v) for v in user_settings['restricted_guilds']] - restricted_peer = [int(v) for v in peer_settings['restricted_guilds']] + restricted_user_ = [int(v) for v in user_settings['restricted_guilds']] + restricted_peer_ = [int(v) for v in peer_settings['restricted_guilds']] - restricted_user = set(restricted_user) - restricted_peer = set(restricted_peer) + restricted_user = set(restricted_user_) + restricted_peer = set(restricted_peer_) mutual_guilds -= restricted_user mutual_guilds -= restricted_peer diff --git a/litecord/embed/sanitizer.py b/litecord/embed/sanitizer.py index 4b0b0b9..ef912c4 100644 --- a/litecord/embed/sanitizer.py +++ b/litecord/embed/sanitizer.py @@ -22,7 +22,7 @@ litecord.embed.sanitizer sanitize embeds by giving common values such as type: rich """ -from typing import Dict, Any +from typing import Dict, Any, Optional, Union, List from logbook import Logger from quart import current_app as app @@ -44,7 +44,7 @@ def sanitize_embed(embed: Embed) -> Embed: }} -def path_exists(embed: Embed, components: str): +def path_exists(embed: Embed, components_in: Union[List[str], str]): """Tell if a given path exists in an embed (or any dictionary). The components string is formatted like this: @@ -54,10 +54,10 @@ def path_exists(embed: Embed, components: str): """ # get the list of components given - if isinstance(components, str): - components = components.split('.') + if isinstance(components_in, str): + components = components_in.split('.') else: - components = list(components) + components = list(components_in) # if there are no components, we reached the end of recursion # and can return true @@ -96,7 +96,7 @@ def proxify(url, *, config=None) -> str: ) -async def fetch_metadata(url, *, config=None, session=None) -> dict: +async def fetch_metadata(url, *, config=None, session=None) -> Optional[Dict]: """Fetch metadata for a url.""" if session is None: @@ -123,7 +123,7 @@ async def fetch_metadata(url, *, config=None, session=None) -> dict: log.warning('failed to generate meta for {!r}: {} {!r}', url, resp.status, body) - return + return None return await resp.json() diff --git a/litecord/images.py b/litecord/images.py index 854e2b9..51ed855 100644 --- a/litecord/images.py +++ b/litecord/images.py @@ -22,6 +22,7 @@ import mimetypes import asyncio import base64 import tempfile +from typing import Optional from dataclasses import dataclass from hashlib import sha256 @@ -67,22 +68,33 @@ def get_mime(ext: str): @dataclass class Icon: """Main icon class""" - key: str - icon_hash: str - mime: str + key: Optional[str] + icon_hash: Optional[str] + mime: Optional[str] @property - def as_path(self) -> str: + def as_path(self) -> Optional[str]: """Return a filesystem path for the given icon.""" + if self.mime is None: + return None + ext = get_ext(self.mime) return str(IMAGE_FOLDER / f'{self.key}_{self.icon_hash}.{ext}') @property - def as_pathlib(self) -> str: + def as_pathlib(self) -> Optional[Path]: + """Get a Path instance of this icon.""" + if self.as_path is None: + return None + return Path(self.as_path) @property - def extension(self) -> str: + def extension(self) -> Optional[str]: + """Get the extension of this icon.""" + if self.mime is None: + return None + return get_ext(self.mime) @@ -91,7 +103,7 @@ class ImageError(Exception): pass -def to_raw(data_type: str, data: str) -> bytes: +def to_raw(data_type: str, data: str) -> Optional[bytes]: """Given a data type in the data URI and data, give the raw bytes being encoded.""" if data_type == 'base64': @@ -176,7 +188,7 @@ def _gen_update_sql(scope: str) -> str: """ -def _invalid(kwargs: dict): +def _invalid(kwargs: dict) -> Optional[Icon]: """Send an invalid value.""" if not kwargs.get('always_icon', False): return None @@ -272,7 +284,8 @@ class IconManager: return Icon(icon.key, icon.icon_hash, target_mime) - async def generic_get(self, scope, key, icon_hash, **kwargs) -> Icon: + async def generic_get(self, scope, key, icon_hash, + **kwargs) -> Optional[Icon]: """Get any icon.""" if icon_hash is None: return None @@ -302,10 +315,17 @@ class IconManager: icon = Icon(icon_row['key'], icon_row['hash'], icon_row['mime']) + # ensure we aren't messing with NULLs everywhere. + if icon.as_pathlib is None: + return None + if not icon.as_pathlib.exists(): await self.delete(icon) return None + if icon.extension is None: + return None + if 'ext' in kwargs and kwargs['ext'] != icon.extension: return await self._convert_ext(icon, kwargs['ext']) diff --git a/litecord/permissions.py b/litecord/permissions.py index 589b976..4e2a4b6 100644 --- a/litecord/permissions.py +++ b/litecord/permissions.py @@ -198,7 +198,8 @@ async def role_permissions(guild_id: int, role_id: int, async def compute_overwrites(base_perms: Permissions, user_id, channel_id: int, - guild_id: int = None, storage=None): + guild_id: Optional[int] = None, + storage=None): """Compute the permissions in the context of a channel.""" if not storage: storage = app.storage @@ -211,8 +212,12 @@ async def compute_overwrites(base_perms: Permissions, # list of overwrites overwrites = await storage.chan_overwrites(channel_id) + # if the channel isn't a guild, we should just return + # ALL_PERMISSIONS. the old approach was calling guild_from_channel + # again, but it is already passed by get_permissions(), so its + # redundant. if not guild_id: - guild_id = await storage.guild_from_channel(channel_id) + return ALL_PERMISSIONS # make it a map for better usage overwrites = {int(o['id']): o for o in overwrites} diff --git a/litecord/presence.py b/litecord/presence.py index a8c267e..bf7bad9 100644 --- a/litecord/presence.py +++ b/litecord/presence.py @@ -127,7 +127,7 @@ class PresenceManager: # shards that are in lazy guilds with 'everyone' # enabled - in_lazy = [] + in_lazy: List[str] = [] for member_list in lists: session_ids = await member_list.pres_update( diff --git a/litecord/pubsub/channel.py b/litecord/pubsub/channel.py index b931bec..15e4010 100644 --- a/litecord/pubsub/channel.py +++ b/litecord/pubsub/channel.py @@ -84,7 +84,7 @@ class ChannelDispatcher(DispatcherWithState): await self.unsub(channel_id, user_id) continue - cur_sess = 0 + cur_sess = [] if event in ('CHANNEL_CREATE', 'CHANNEL_UPDATE') \ and data.get('type') == ChannelType.GROUP_DM.value: diff --git a/litecord/pubsub/lazy_guild.py b/litecord/pubsub/lazy_guild.py index df5c0dc..2cc9527 100644 --- a/litecord/pubsub/lazy_guild.py +++ b/litecord/pubsub/lazy_guild.py @@ -28,7 +28,9 @@ lazy guilds: import asyncio from collections import defaultdict -from typing import Any, List, Dict, Union, Optional, Iterable, Iterator, Tuple +from typing import ( + Any, List, Dict, Union, Optional, Iterable, Iterator, Tuple, Set +) from dataclasses import dataclass, asdict, field from logbook import Logger @@ -265,7 +267,7 @@ class GuildMemberList: #: store the states that are subscribed to the list. # type is {session_id: set[list]} - self.state = defaultdict(set) + self.state: Dict[str, Set[List[int, int]]] = defaultdict(set) self._list_lock = asyncio.Lock() @@ -589,7 +591,7 @@ class GuildMemberList: except KeyError: return None - async def _dispatch_sess(self, session_ids: List[str], + async def _dispatch_sess(self, session_ids: Iterable[str], operations: List[Operation]): """Dispatch a GUILD_MEMBER_LIST_UPDATE to the given session ids.""" @@ -613,11 +615,12 @@ class GuildMemberList: } states = map(self._get_state, session_ids) - states = filter(lambda state: state is not None, states) - dispatched = [] for state in states: + if state is None: + continue + await state.ws.dispatch( 'GUILD_MEMBER_LIST_UPDATE', payload) @@ -625,7 +628,7 @@ class GuildMemberList: return dispatched - async def _resync(self, session_ids: List[int], + async def _resync(self, session_ids: List[str], item_index: int) -> List[str]: """Send a SYNC event to all states that are subscribed to an item. @@ -661,7 +664,7 @@ class GuildMemberList: return result - async def _resync_by_item(self, item_index: int): + async def _resync_by_item(self, item_index: Optional[int]): """Resync but only giving the item index.""" if item_index is None: return [] @@ -1339,7 +1342,10 @@ class GuildMemberList: log.debug('there are {} session ids to resync (for item {})', len(sess_ids_resync), role_item_index) - return await self._resync(sess_ids_resync, role_item_index) + if role_item_index is not None: + return await self._resync(sess_ids_resync, role_item_index) + + return [] async def chan_update(self): """Called then a channel's data has been updated.""" diff --git a/litecord/utils.py b/litecord/utils.py index 1dca864..0145825 100644 --- a/litecord/utils.py +++ b/litecord/utils.py @@ -19,8 +19,9 @@ along with this program. If not, see . import asyncio import json +from typing import Any, Iterable, Optional, Indexable + from logbook import Logger -from typing import Any, Iterable from quart.json import JSONEncoder log = Logger(__name__) @@ -51,7 +52,7 @@ def dict_get(mapping, key, default): return mapping.get(key) or default -def index_by_func(function, indexable: iter) -> int: +def index_by_func(function, indexable: Indexable) -> Optional[int]: """Search in an idexable and return the index number for an iterm that has func(item) = True.""" for index, item in enumerate(indexable): @@ -66,7 +67,7 @@ def _u(val): return val % 0x100000000 -def mmh3(key: str, seed: int = 0): +def mmh3(inp_str: str, seed: int = 0): """MurMurHash3 implementation. This seems to match Discord's JavaScript implementaiton. @@ -74,7 +75,7 @@ def mmh3(key: str, seed: int = 0): Based off https://github.com/garycourt/murmurhash-js/blob/master/murmurhash3_gc.js """ - key = [ord(c) for c in key] + key = [ord(c) for c in inp_str] remainder = len(key) & 3 bytecount = len(key) - remainder From e2af6b63701d16c5554bf1b75c9b4371f0299c3e Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 20:46:14 -0300 Subject: [PATCH 44/55] voice: more voice goodies - lvsp manager: change internal structure of lvsp conns - voice.manager: add incomplete impl for creating a channel --- litecord/gateway/websocket.py | 6 +-- litecord/permissions.py | 1 + litecord/utils.py | 8 ++-- litecord/voice/lvsp_conn.py | 3 ++ litecord/voice/lvsp_manager.py | 57 ++++++++++++++++++++++++++-- litecord/voice/manager.py | 69 ++++++++++++++++++++++++++++++++-- 6 files changed, 130 insertions(+), 14 deletions(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index e6fa7d4..d17cf27 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -670,7 +670,7 @@ class GatewayWebsocket: voice_state = await self.ext.voice.get_state(voice_key) if voice_state is None: - await self.ext.voice.create_state(voice_key) + return await self.ext.voice.create_state(voice_key) same_guild = guild_id == voice_state.guild_id same_channel = channel_id == voice_state.channel_id @@ -678,10 +678,10 @@ class GatewayWebsocket: prop = await self._vsu_get_prop(voice_state, data) if same_guild and same_channel: - await self.ext.voice.update_state(voice_state, prop) + return await self.ext.voice.update_state(voice_state, prop) if same_guild and not same_channel: - await self.ext.voice.move_state(voice_state, channel_id) + return await self.ext.voice.move_state(voice_state, channel_id) async def _handle_5(self, payload: Dict[str, Any]): """Handle OP 5 Voice Server Ping. diff --git a/litecord/permissions.py b/litecord/permissions.py index 4e2a4b6..4b062e8 100644 --- a/litecord/permissions.py +++ b/litecord/permissions.py @@ -18,6 +18,7 @@ along with this program. If not, see . """ import ctypes +from typing import Optional from quart import current_app as app diff --git a/litecord/utils.py b/litecord/utils.py index 0145825..513fafa 100644 --- a/litecord/utils.py +++ b/litecord/utils.py @@ -19,7 +19,7 @@ along with this program. If not, see . import asyncio import json -from typing import Any, Iterable, Optional, Indexable +from typing import Any, Iterable, Optional, Sequence from logbook import Logger from quart.json import JSONEncoder @@ -27,7 +27,7 @@ from quart.json import JSONEncoder log = Logger(__name__) -async def async_map(function, iterable) -> list: +async def async_map(function, iterable: Iterable) -> list: """Map a coroutine to an iterable.""" res = [] @@ -52,7 +52,7 @@ def dict_get(mapping, key, default): return mapping.get(key) or default -def index_by_func(function, indexable: Indexable) -> Optional[int]: +def index_by_func(function, indexable: Sequence[Any]) -> Optional[int]: """Search in an idexable and return the index number for an iterm that has func(item) = True.""" for index, item in enumerate(indexable): @@ -161,7 +161,7 @@ async def pg_set_json(con): ) -def yield_chunks(input_list: Iterable, chunk_size: int): +def yield_chunks(input_list: Sequence[Any], chunk_size: int): """Yield successive n-sized chunks from l. Taken from https://stackoverflow.com/a/312464. diff --git a/litecord/voice/lvsp_conn.py b/litecord/voice/lvsp_conn.py index a350737..143ba97 100644 --- a/litecord/voice/lvsp_conn.py +++ b/litecord/voice/lvsp_conn.py @@ -38,6 +38,7 @@ class LVSPConnection: self.hostname = hostname self.conn = None + self.health = 0.5 self._hb_task = None self._hb_interval = None @@ -98,6 +99,8 @@ class LVSPConnection: async def _update_health(self, new_health: float): """Update the health value of a given voice server.""" + self.health = new_health + await self.app.db.execute(""" UPDATE voice_servers SET health = $1 diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py index 2182859..a40ff83 100644 --- a/litecord/voice/lvsp_manager.py +++ b/litecord/voice/lvsp_manager.py @@ -17,6 +17,7 @@ along with this program. If not, see . """ +from typing import Optional from collections import defaultdict from logbook import Logger @@ -34,7 +35,15 @@ class LVSPManager: self.app = app self.voice = voice - self.servers = defaultdict(dict) + # map servers to LVSPConnection + self.conns = {} + + # maps regions to server hostnames + self.servers = defaultdict(list) + + # maps guilds to server hostnames + self.guild_servers = {} + self.app.loop.create_task(self._spawn()) async def _spawn(self): @@ -71,10 +80,11 @@ class LVSPManager: return servers = [r['hostname'] for r in servers] + self.servers[region] = servers for hostname in servers: conn = LVSPConnection(self, region, hostname) - self.servers[region][hostname] = conn + self.conns[hostname] = conn self.app.loop.create_task( conn.run() @@ -83,6 +93,47 @@ class LVSPManager: async def del_conn(self, conn): """Delete a connection from the connection pool.""" try: - self.servers[conn.region].pop(conn.hostname) + self.servers[conn.region].remove(conn.hostname) except KeyError: pass + + try: + self.conns.pop(conn.hostname) + except KeyError: + pass + + async def guild_region(self, guild_id: int) -> Optional[str]: + """Return the voice region of a guild.""" + return await self.app.db.fetchval(""" + SELECT region + FROM guilds + WHERE id = $1 + """, guild_id) + + def get_health(self, hostname: str) -> float: + """Get voice server health, given hostname.""" + try: + conn = self.conns[hostname] + except KeyError: + return -1 + + return conn.health + + async def get_server(self, guild_id: int) -> str: + """Get a voice server for the given guild, assigns + one if there isn't any.""" + + try: + hostname = self.guild_servers[guild_id] + except KeyError: + region = await self.guild_region(guild_id) + + # sort connected servers by health + sorted_servers = sorted( + self.servers[region], + self.get_health, + ) + + hostname = sorted_servers[0] + + return hostname diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 535b6b2..a408909 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -153,11 +153,72 @@ class VoiceManager: async def move_channels(self, old_voice_key: VoiceKey, channel_id: int): """Move a user between channels.""" await self.del_state(old_voice_key) - await self.create_state(old_voice_key, channel_id, {}) + await self.create_state(old_voice_key, {'channel_id': channel_id}) - async def create_state(self, voice_key: VoiceKey, channel_id: int, - data: dict): - pass + async def _create_ctx_guild(self, guild_id, channel_id): + # get a voice server + server = await self.lvsp.get_server(guild_id) + conn = self.lvsp.get_conn(server) + chan = await self.app.storage.get_channel(channel_id) + + # TODO: this, but properly + # TODO: when the server sends a reply to CHAN_REQ, we need to update + # LVSPManager.guild_servers. + await conn.send_info('CHAN_REQ', { + 'guild_id': str(guild_id), + 'channel_id': str(channel_id), + + 'channel_properties': { + 'bitrate': chan['bitrate'] + } + }) + + async def _start_voice_guild(self, voice_key: VoiceKey, data: dict): + """Start a voice context in a guild.""" + user_id, guild_id = voice_key + channel_id = int(data['channel_id']) + + existing_states = self.states[voice_key] + channel_exists = any( + state.channel_id == channel_id for state in existing_states) + + if not channel_exists: + await self._create_ctx_guild(guild_id, channel_id) + + async def create_state(self, voice_key: VoiceKey, data: dict): + """Creates (or tries to create) a voice state. + + Depending on the VoiceKey given, it will use the guild's voice + region or assign one based on the starter of a call, or the owner of + a Group DM. + + Once a region is assigned, it'll choose the best voice server + and send a request to it. + """ + + # TODO: handle CALL events. + + # compare if this voice key is for a guild or a channel + _uid, id2 = voice_key + guild = await self.app.storage.get_guild(id2) + + # if guild not found, then we are dealing with a dm or group dm + if not guild: + ctype = await self.app.storage.get_chan_type(id2) + ctype = ChannelType(ctype) + + if ctype == ChannelType.GROUP_DM: + # await self._start_voice_dm() + pass + elif ctype == ChannelType.DM: + # await self._start_voice_gdm() + pass + + return + + # if guild found, then data.channel_id exists, and we treat it + # as a guild + # await self._start_voice_guild() async def leave_all(self, user_id: int) -> int: """Leave all voice channels.""" From 00100f9abb89309f9db9ef60e498382535d4cb1e Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 21:32:04 -0300 Subject: [PATCH 45/55] voice.manager: add VST_REQ call --- litecord/voice/manager.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index a408909..bd4f48f 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import Tuple, Dict +from typing import Tuple, Dict, List from collections import defaultdict from dataclasses import fields @@ -155,16 +155,20 @@ class VoiceManager: await self.del_state(old_voice_key) await self.create_state(old_voice_key, {'channel_id': channel_id}) + async def _lvsp_info_guild(self, guild_id, info_type, info_data): + hostname = await self.lvsp.get_server(guild_id) + conn = self.lvsp.get_conn(hostname) + + # TODO: impl send_info + await conn.send_info(info_type, info_data) + async def _create_ctx_guild(self, guild_id, channel_id): - # get a voice server - server = await self.lvsp.get_server(guild_id) - conn = self.lvsp.get_conn(server) chan = await self.app.storage.get_channel(channel_id) # TODO: this, but properly # TODO: when the server sends a reply to CHAN_REQ, we need to update # LVSPManager.guild_servers. - await conn.send_info('CHAN_REQ', { + await self._lvsp_info_guild(guild_id, 'CHAN_REQ', { 'guild_id': str(guild_id), 'channel_id': str(channel_id), @@ -180,11 +184,18 @@ class VoiceManager: existing_states = self.states[voice_key] channel_exists = any( - state.channel_id == channel_id for state in existing_states) + state.channel_id == channel_id for state in existing_states + ) if not channel_exists: await self._create_ctx_guild(guild_id, channel_id) + await self._lvsp_info_guild(guild_id, 'VST_REQ', { + 'user_id': str(user_id), + 'guild_id': str(guild_id), + 'channel_id': str(channel_id), + }) + async def create_state(self, voice_key: VoiceKey, data: dict): """Creates (or tries to create) a voice state. @@ -208,17 +219,17 @@ class VoiceManager: ctype = ChannelType(ctype) if ctype == ChannelType.GROUP_DM: - # await self._start_voice_dm() + # await self._start_voice_dm(voice_key) pass elif ctype == ChannelType.DM: - # await self._start_voice_gdm() + # await self._start_voice_gdm(voice_key) pass return # if guild found, then data.channel_id exists, and we treat it # as a guild - # await self._start_voice_guild() + await self._start_voice_guild(voice_key, data) async def leave_all(self, user_id: int) -> int: """Leave all voice channels.""" @@ -243,7 +254,7 @@ class VoiceManager: """Make a user leave a channel IN A GUILD.""" await self.del_state((guild_id, user_id)) - async def voice_server_list(self, region: str): + async def voice_server_list(self, region: str) -> List[dict]: """Get a list of voice server objects""" rows = await self.app.db.fetch(""" SELECT hostname, last_health From 579a71dd9b1344a2f32d689db0b42a440bc7dfe2 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 4 Mar 2019 22:46:09 -0300 Subject: [PATCH 46/55] lvsp_conn: add basic INFO handling for channel assign - lvsp: finish defining data for channel info messages - lvsp_conn: add send_info() - lvsp_opcodes: add InfoTable, InfoReverse - voice.manager: handle channels without bitrate --- docs/lvsp.md | 21 +++++++++++++---- litecord/voice/lvsp_conn.py | 42 +++++++++++++++++++++++++++++++++- litecord/voice/lvsp_manager.py | 14 ++++++++---- litecord/voice/lvsp_opcodes.py | 13 +++++++++++ litecord/voice/manager.py | 11 +++------ 5 files changed, 83 insertions(+), 18 deletions(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index de30b8a..a022f25 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -175,24 +175,37 @@ Request a channel to be created inside the voice server. The Server MUST reply back with a CHANNEL\_ASSIGN when resources are allocated for the channel. -**TODO:** fields +| field | type | description | +| --: | :-- | :-- | +| channel\_id | snowflake | channel id | +| guild\_id | Optional[snowflake] | guild id, not provided if dm / group dm | +| channel\_properties | ChannelProperties | channel properties | + +#### ChannelProperties + +| field | type | description | +| --: | :-- | :-- | +| bitrate | integer | channel bitrate | ### CHANNEL\_ASSIGN Sent by the Server to signal the successful creation of a voice channel. -**TODO:** fields +| field | type | description | +| --: | :-- | :-- | +| channel\_id | snowflake | channel id | +| guild\_id | Optional[snowflake] | guild id, not provided if dm / group dm | ### CHANNEL\_UPDATE Sent by the client to signal an update to the properties of a channel, such as its bitrate. -**TODO:** fields +Same data as CHANNEL\_REQ. ### CHANNEL\_DESTROY Sent by the client to signal the destruction of a voice channel. Be it a channel being deleted, or all members in it leaving. -**TODO:** fields +Same data as CHANNEL\_ASSIGN. diff --git a/litecord/voice/lvsp_conn.py b/litecord/voice/lvsp_conn.py index 143ba97..3902674 100644 --- a/litecord/voice/lvsp_conn.py +++ b/litecord/voice/lvsp_conn.py @@ -19,11 +19,12 @@ along with this program. If not, see . import json import asyncio +from typing import Dict import websockets from logbook import Logger -from litecord.voice.lvsp_opcodes import OPCodes as OP +from litecord.voice.lvsp_opcodes import OPCodes as OP, InfoTable, InfoReverse log = Logger(__name__) @@ -65,6 +66,16 @@ class LVSPConnection: 'd': data }) + async def send_info(self, info_type: str, info_data: Dict): + """Send an INFO message down the websocket.""" + await self.send({ + 'op': OP.info, + 'd': { + 'type': InfoTable[info_type.upper()], + 'data': info_data + } + }) + async def _heartbeater(self, hb_interval: int): try: await asyncio.sleep(hb_interval) @@ -121,6 +132,35 @@ class LVSPConnection: await self._update_health(msg['health']) self._start_hb() + async def _handle_6(self, msg): + """Handle INFO messages.""" + info = msg['d'] + info_type_str = InfoReverse[info['type']].lower() + + try: + info_handler = getattr(self, f'_handle_info_{info_type_str}') + except AttributeError: + return + + await info_handler(info['data']) + + async def _handle_info_channel_assign(self, data: dict): + """called by the server once we got a channel assign.""" + try: + channel_id = data['channel_id'] + channel_id = int(channel_id) + except (TypeError, ValueError): + return + + try: + guild_id = data['guild_id'] + guild_id = int(guild_id) + except (TypeError, ValueError): + guild_id = None + + main_key = guild_id if guild_id is not None else channel_id + await self.lvsp.assign(main_key, self.hostname) + async def _loop(self): while True: msg = await self.recv() diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py index a40ff83..7436941 100644 --- a/litecord/voice/lvsp_manager.py +++ b/litecord/voice/lvsp_manager.py @@ -41,8 +41,8 @@ class LVSPManager: # maps regions to server hostnames self.servers = defaultdict(list) - # maps guilds to server hostnames - self.guild_servers = {} + # maps Union[GuildID, DMId, GroupDMId] to server hostnames + self.assign = {} self.app.loop.create_task(self._spawn()) @@ -119,12 +119,12 @@ class LVSPManager: return conn.health - async def get_server(self, guild_id: int) -> str: + async def get_guild_server(self, guild_id: int) -> str: """Get a voice server for the given guild, assigns - one if there isn't any.""" + one if there isn't any""" try: - hostname = self.guild_servers[guild_id] + hostname = self.assign[guild_id] except KeyError: region = await self.guild_region(guild_id) @@ -137,3 +137,7 @@ class LVSPManager: hostname = sorted_servers[0] return hostname + + async def assign_conn(self, key: int, hostname: str): + """Assign a connection to a given key in the assign map""" + self.assign[key] = hostname diff --git a/litecord/voice/lvsp_opcodes.py b/litecord/voice/lvsp_opcodes.py index 4b19249..4cd0b3c 100644 --- a/litecord/voice/lvsp_opcodes.py +++ b/litecord/voice/lvsp_opcodes.py @@ -26,3 +26,16 @@ class OPCodes: heartbeat = 4 heartbeat_ack = 5 info = 6 + + +InfoTable = { + 'CHANNEL_REQ': 0, + 'CHANNEL_ASSIGN': 1, + 'CHANNEL_UPDATE': 2, + 'CHANNEL_DESTROY': 3, + 'VST_CREATE': 4, + 'VST_UPDATE': 5, + 'VST_LEAVE': 6, +} + +InfoReverse = {v: k for k, v in InfoTable.items()} diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index bd4f48f..063ebce 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -156,24 +156,19 @@ class VoiceManager: await self.create_state(old_voice_key, {'channel_id': channel_id}) async def _lvsp_info_guild(self, guild_id, info_type, info_data): - hostname = await self.lvsp.get_server(guild_id) + hostname = await self.lvsp.get_guild_server(guild_id) conn = self.lvsp.get_conn(hostname) - - # TODO: impl send_info await conn.send_info(info_type, info_data) async def _create_ctx_guild(self, guild_id, channel_id): chan = await self.app.storage.get_channel(channel_id) - # TODO: this, but properly - # TODO: when the server sends a reply to CHAN_REQ, we need to update - # LVSPManager.guild_servers. - await self._lvsp_info_guild(guild_id, 'CHAN_REQ', { + await self._lvsp_info_guild(guild_id, 'CHANNEL_REQ', { 'guild_id': str(guild_id), 'channel_id': str(channel_id), 'channel_properties': { - 'bitrate': chan['bitrate'] + 'bitrate': chan.get('bitrate', 96) } }) From d21b3c8f2a30d69a1fdd59b1d090a10316e2aea1 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 5 Mar 2019 00:59:37 -0300 Subject: [PATCH 47/55] lvsp: add "common logic scenarios" --- docs/lvsp.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/lvsp.md b/docs/lvsp.md index a022f25..c59c073 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -209,3 +209,41 @@ Sent by the client to signal the destruction of a voice channel. Be it a channel being deleted, or all members in it leaving. Same data as CHANNEL\_ASSIGN. + +## Common logic scenarios + +### User joins an unitialized voice channel + +Since the channel is unitialized, both logic on initialization AND +user join is here. + + - Client will send a CHANNEL\_REQ. + - Client MAY send a VST\_CREATE right after as well. + - The Server MUST process CHANNEL\_REQ first, so the Server can keep + a lock on channel operations while it is initialized. + - Reply with CHANNEL\_ASSIGN once initialization is done. + - Process VST\_CREATE **TODO** + +### Updating a voice channel + + - Client sends CHANNEL\_UPDATE. + - Server DOES NOT reply. + +### Destroying a voice channel + + - Client sends CHANNEL\_DESTROY. + - Server MUST disconnect any users currently connected with its + voice websocket. + +### User joining an (initialized) voice channel + + - Client sends VST\_CREATE + - **TODO** + +### User moves a channel + +**TODO** + +### User leaves a channel + +**TODO** From 30d9520935ce9c0c59f27bc50615cdea43210a95 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 5 Mar 2019 01:18:53 -0300 Subject: [PATCH 48/55] lvsp: remove TODO --- docs/lvsp.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index c59c073..9023af8 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -6,15 +6,6 @@ Voice Websocket Discord and Voice UDP connections. LVSP runs over a *long-lived* websocket with TLS. The encoding is JSON. -**TODO:** common logic scenarios: - - initializing a voice channel - - updating a voice channel - - destroying a voice channel - - user joining to a voice channel - - user joining to a voice channel (while also initializing it, e.g - first member in the channel) - - user leaving a voice channel - ## OP code table "client" is litecord. "server" is the voice server. From 899f19b24ca5c1d4ea0f9c9ec27d37c7e3f18959 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 5 Mar 2019 01:35:17 -0300 Subject: [PATCH 49/55] gateway.websocket: fix some errors right now requesting a voice channel won't return ANY voice events back to the client, causing a visual/behavioral "lock" on the official client, since there isn't any available voice servers. --- litecord/gateway/websocket.py | 2 +- litecord/voice/lvsp_manager.py | 9 ++++++--- litecord/voice/manager.py | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index d17cf27..c2977b7 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -670,7 +670,7 @@ class GatewayWebsocket: voice_state = await self.ext.voice.get_state(voice_key) if voice_state is None: - return await self.ext.voice.create_state(voice_key) + return await self.ext.voice.create_state(voice_key, data) same_guild = guild_id == voice_state.guild_id same_channel = channel_id == voice_state.channel_id diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py index 7436941..78484e0 100644 --- a/litecord/voice/lvsp_manager.py +++ b/litecord/voice/lvsp_manager.py @@ -119,7 +119,7 @@ class LVSPManager: return conn.health - async def get_guild_server(self, guild_id: int) -> str: + async def get_guild_server(self, guild_id: int) -> Optional[str]: """Get a voice server for the given guild, assigns one if there isn't any""" @@ -131,10 +131,13 @@ class LVSPManager: # sort connected servers by health sorted_servers = sorted( self.servers[region], - self.get_health, + key=self.get_health ) - hostname = sorted_servers[0] + try: + hostname = sorted_servers[0] + except IndexError: + return None return hostname diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 063ebce..10d6ee1 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -157,6 +157,10 @@ class VoiceManager: async def _lvsp_info_guild(self, guild_id, info_type, info_data): hostname = await self.lvsp.get_guild_server(guild_id) + if hostname is None: + log.error('no voice server for guild id {}', guild_id) + return + conn = self.lvsp.get_conn(hostname) await conn.send_info(info_type, info_data) From 7b4aaabcbfbc87510adb4f54cbe7e60b7ebf64a3 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 5 Mar 2019 16:30:38 -0300 Subject: [PATCH 50/55] storage: better fix for message.webhook_id the old fix only works for the return value in message create, where as it should be in Storage.get_message instead so that the bug doesn't happen to the other routes --- litecord/blueprints/channel/messages.py | 3 --- litecord/storage.py | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/litecord/blueprints/channel/messages.py b/litecord/blueprints/channel/messages.py index dfba47b..41850e2 100644 --- a/litecord/blueprints/channel/messages.py +++ b/litecord/blueprints/channel/messages.py @@ -408,9 +408,6 @@ async def _create_message(channel_id): await _dm_pre_dispatch(channel_id, user_id) await _dm_pre_dispatch(channel_id, guild_id) - if payload['webhook_id'] == None: - payload.pop('webhook_id', None) - await app.dispatcher.dispatch('channel', channel_id, 'MESSAGE_CREATE', payload) diff --git a/litecord/storage.py b/litecord/storage.py index d23eacb..74a6be5 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -763,12 +763,13 @@ class Storage: return res - async def _inject_author(self, res): + async def _inject_author(self, res: dict): """Inject a pseudo-user object when the message is made by a webhook.""" author_id, webhook_id = res['author_id'], res['webhook_id'] if author_id is not None: res['author'] = await self.get_user(res['author_id']) + res.pop('webhook_id') elif webhook_id is not None: res['author'] = { 'id': webhook_id, From a846c57ae8d4de21612c0fbfaae148ebba368cef Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 5 Mar 2019 18:31:33 -0300 Subject: [PATCH 51/55] storage: inject message.member field required some changes around how we fetch data from the members table, but it works --- litecord/storage.py | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/litecord/storage.py b/litecord/storage.py index 74a6be5..1530a65 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -171,12 +171,34 @@ class Storage: return drow async def _member_basic(self, guild_id: int, member_id: int): - return await self.db.fetchrow(""" - SELECT user_id, nickname, joined_at, deafened, muted + row = await self.db.fetchrow(""" + SELECT user_id, nickname, joined_at, + deafened AS deaf, muted AS mute FROM members WHERE guild_id = $1 and user_id = $2 """, guild_id, member_id) + if row is None: + return None + + row = dict(row) + row['joined_at'] = timestamp_(row['joined_at']) + return row + + async def _member_basic_with_roles(self, guild_id: int, + member_id: int): + basic = await self._member_basic(guild_id, member_id) + + if basic is None: + return None + + basic = dict(basic) + roles = await self.get_member_role_ids(guild_id, member_id) + + return {**basic, **{ + 'roles': roles + }} + async def get_member_role_ids(self, guild_id: int, member_id: int) -> List[str]: """Get a list of role IDs that are on a member.""" @@ -203,6 +225,7 @@ class Storage: async def _member_dict(self, row, guild_id, member_id) -> Dict[str, Any]: roles = await self.get_member_role_ids(guild_id, member_id) + return { 'user': await self.get_user(member_id), 'nick': row['nickname'], @@ -211,9 +234,9 @@ class Storage: # the user since it is known that everyone has # that role. 'roles': roles, - 'joined_at': timestamp_(row['joined_at']), - 'deaf': row['deafened'], - 'mute': row['muted'], + 'joined_at': row['joined_at'], + 'deaf': row['deaf'], + 'mute': row['mute'], } async def get_member_data_one(self, guild_id: int, @@ -244,7 +267,8 @@ class Storage: async def get_member_data(self, guild_id: int) -> List[Dict[str, Any]]: """Get member information on a guild.""" members_basic = await self.db.fetch(""" - SELECT user_id, nickname, joined_at, deafened, muted + SELECT user_id, nickname, joined_at, + deafened AS deaf, muted AS mute FROM members WHERE guild_id = $1 """, guild_id) @@ -849,8 +873,12 @@ class Storage: res['attachments'] = await self.get_attachments(message_id) - # TODO: res['member'] for partial member data - # of the author + # if message is not from a dm, guild_id is None and so, _member_basic + # will just return None + res['member'] = await self._member_basic_with_roles(guild_id, user_id) + + if res['member'] is None: + res.pop('member') pin_id = await self.db.fetchval(""" SELECT message_id From b8f5e01106b7cd9af0ab0f8321969ca135e39184 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 6 Mar 2019 00:44:16 -0300 Subject: [PATCH 52/55] lvsp: add CHANNEL_REQ.token --- docs/lvsp.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index 9023af8..0441131 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -186,6 +186,7 @@ Sent by the Server to signal the successful creation of a voice channel. | --: | :-- | :-- | | channel\_id | snowflake | channel id | | guild\_id | Optional[snowflake] | guild id, not provided if dm / group dm | +| token | string | authentication token | ### CHANNEL\_UPDATE @@ -199,7 +200,7 @@ Same data as CHANNEL\_REQ. Sent by the client to signal the destruction of a voice channel. Be it a channel being deleted, or all members in it leaving. -Same data as CHANNEL\_ASSIGN. +Same data as CHANNEL\_ASSIGN, but without `token`. ## Common logic scenarios From b8c3208fa49e77ad47189794c73aeb715e872fac Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 6 Mar 2019 02:19:28 -0300 Subject: [PATCH 53/55] lvsp: remove CHANNEL_UPDATE bitrate is a client thing. lol - lvsp: remove ChannelProperties, same applies to CHANNEL\_REQ - voice.manager: remove channel\_properties --- docs/lvsp.md | 40 +++++++++++++++++++++------------------ litecord/voice/manager.py | 8 +------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index 0441131..dd3a943 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -151,9 +151,9 @@ are laid on. | --: | :-- | :-- | | 0 | CHANNEL\_REQ | channel assignment request | | 1 | CHANNEL\_ASSIGN | channel assignment reply | -| 2 | CHANNEL\_UPDATE | channel update | -| 3 | CHANNEL\_DESTROY | channel destroy | -| 4 | VST\_CREATE | voice state create request | +| 2 | CHANNEL\_DESTROY | channel destroy | +| 3 | VST\_CREATE | voice state create request | +| 4 | VST\_DONE | voice state created | | 5 | VST\_UPDATE | voice state update | | 6 | VST\_LEAVE | voice state leave | @@ -170,13 +170,6 @@ allocated for the channel. | --: | :-- | :-- | | channel\_id | snowflake | channel id | | guild\_id | Optional[snowflake] | guild id, not provided if dm / group dm | -| channel\_properties | ChannelProperties | channel properties | - -#### ChannelProperties - -| field | type | description | -| --: | :-- | :-- | -| bitrate | integer | channel bitrate | ### CHANNEL\_ASSIGN @@ -188,13 +181,6 @@ Sent by the Server to signal the successful creation of a voice channel. | guild\_id | Optional[snowflake] | guild id, not provided if dm / group dm | | token | string | authentication token | -### CHANNEL\_UPDATE - -Sent by the client to signal an update to the properties of a channel, -such as its bitrate. - -Same data as CHANNEL\_REQ. - ### CHANNEL\_DESTROY Sent by the client to signal the destruction of a voice channel. Be it @@ -202,6 +188,24 @@ a channel being deleted, or all members in it leaving. Same data as CHANNEL\_ASSIGN, but without `token`. +### VST\_CREATE + +**TODO** + +| field | type | description | +| --: | :-- | :-- | +| user\_id | snowflake | user id | +| channel\_id | snowflake | channel id | +| guild\_id | Optional[snowflake] | guild id | + +### VST\_DONE + +**TODO** + +### VST\_DESTROY + +**TODO** + ## Common logic scenarios ### User joins an unitialized voice channel @@ -230,7 +234,7 @@ user join is here. ### User joining an (initialized) voice channel - Client sends VST\_CREATE - - **TODO** + - Server sends VST\_DONE ### User moves a channel diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py index 10d6ee1..b927cd3 100644 --- a/litecord/voice/manager.py +++ b/litecord/voice/manager.py @@ -165,15 +165,9 @@ class VoiceManager: await conn.send_info(info_type, info_data) async def _create_ctx_guild(self, guild_id, channel_id): - chan = await self.app.storage.get_channel(channel_id) - await self._lvsp_info_guild(guild_id, 'CHANNEL_REQ', { 'guild_id': str(guild_id), 'channel_id': str(channel_id), - - 'channel_properties': { - 'bitrate': chan.get('bitrate', 96) - } }) async def _start_voice_guild(self, voice_key: VoiceKey, data: dict): @@ -189,7 +183,7 @@ class VoiceManager: if not channel_exists: await self._create_ctx_guild(guild_id, channel_id) - await self._lvsp_info_guild(guild_id, 'VST_REQ', { + await self._lvsp_info_guild(guild_id, 'VST_CREATE', { 'user_id': str(user_id), 'guild_id': str(guild_id), 'channel_id': str(channel_id), From 76a8038cb43aca43e3bf4d62942492a23cdaca12 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 6 Mar 2019 03:47:40 -0300 Subject: [PATCH 54/55] lvsp: add docs for voice state create and destroy finishes all TODOs. --- docs/lvsp.md | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index dd3a943..482b38c 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -157,8 +157,6 @@ are laid on. | 5 | VST\_UPDATE | voice state update | | 6 | VST\_LEAVE | voice state leave | -**TODO:** finish all infos - ### CHANNEL\_REQ Request a channel to be created inside the voice server. @@ -190,21 +188,32 @@ Same data as CHANNEL\_ASSIGN, but without `token`. ### VST\_CREATE -**TODO** +Sent by the client to create a voice state. | field | type | description | | --: | :-- | :-- | | user\_id | snowflake | user id | | channel\_id | snowflake | channel id | -| guild\_id | Optional[snowflake] | guild id | +| guild\_id | Optional[snowflake] | guild id. not provided if dm / group dm | ### VST\_DONE -**TODO** +Sent by the server to indicate the success of a VST\_CREATE. + +Has the same fields as VST\_CREATE, but with extras: + +| field | type | description | +| --: | :-- | :-- | +| session\_id | string | session id for the voice state | ### VST\_DESTROY -**TODO** +Sent by the client when a user is leaving a channel OR moving between channels +in a guild. More on state transitions later on. + +| field | type | description | +| --: | :-- | :-- | +| session\_id | string | session id for the voice state | ## Common logic scenarios @@ -218,7 +227,7 @@ user join is here. - The Server MUST process CHANNEL\_REQ first, so the Server can keep a lock on channel operations while it is initialized. - Reply with CHANNEL\_ASSIGN once initialization is done. - - Process VST\_CREATE **TODO** + - Process VST\_CREATE ### Updating a voice channel @@ -236,10 +245,12 @@ user join is here. - Client sends VST\_CREATE - Server sends VST\_DONE -### User moves a channel - -**TODO** - ### User leaves a channel -**TODO** + - Client sends VST\_DESTROY with the old fields + +### User moves a channel + + - Client sends VST\_DESTROY with the old fields + - Client sends VST\_CREATE with the new fields + - Server sends VST\_DONE From 1bf59e3106b305f272754991745f38b6e79983ae Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 6 Mar 2019 04:04:00 -0300 Subject: [PATCH 55/55] lvsp: remove sequence numbers thinking about it guaranteeing consistency via sequence numbers isn't really needed when we already have Litecord as a single source of truth. plus, it keeps the protocol simple. --- docs/lvsp.md | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/docs/lvsp.md b/docs/lvsp.md index 482b38c..bbd31ba 100644 --- a/docs/lvsp.md +++ b/docs/lvsp.md @@ -31,9 +31,6 @@ snowflake type: A string encoding a Discord Snowflake. | --: | :-- | :-- | | op | integer, opcode | operator code | | d | map[string, any] | message data | -| s | Optional[int] | sequence number | - - - The `s` field is explained in the `RESUME` message. ## High level overview @@ -42,13 +39,13 @@ snowflake type: A string encoding a Discord Snowflake. - if RESUME, process incoming messages as they were post-ready - receive READY - start HEARTBEAT'ing - - send INFO / VSU\_REQUEST messages as needed + - send INFO messages as needed ## Error codes | code | meaning | | --: | :-- | -| 4000 | general error. Reconnect | +| 4000 | general error. reconnect | | 4001 | authentication failure | | 4002 | decode error, given message failed to decode as json | @@ -69,30 +66,6 @@ Sent by the client to identify itself. | --: | :-- | :-- | | token | string | `HMAC(SHA256, key=[secret shared between server and client]), message=[nonce from HELLO]` | -## RESUME message - -Sent by the client to resume itself from a failed websocket connection. - -The server will resend its data, then send a READY message. - -| field | type | description | -| --: | :-- | :-- | -| token | string | same value from IDENTIFY.token | -| seq | integer | last sequence number to resume from | - -### Sequence numbers - -Sequence numbers are used to resume a failed connection back and make the -voice server replay its missing events to the client. - -They are **positive** integers, **starting from 0.** There is no default -upper limit. A "long int" type in languages will probably be enough for most -use cases. - -Replayable messages MUST have sequence numbers embedded into the message -itself with a `s` field. The field lives at the root of the message, alongside -`op` and `d`. - ## READY message - The `health` field is described with more detail in the `HEARTBEAT_ACK` @@ -109,9 +82,7 @@ Sent by the client as a keepalive / health monitoring method. The server MUST reply with a HEARTBEAT\_ACK message back in a reasonable time period. -| field | type | description | -| --: | :-- | :-- | -| s | integer | sequence number | +There are no other fields in this message. ## HEARTBEAT\_ACK message @@ -131,7 +102,6 @@ Among others. | field | type | description | | --: | :-- | :-- | -| s | integer | sequence number | | health | float | server health | ## INFO message @@ -140,6 +110,8 @@ Sent by either client or a server to send information between eachother. The INFO message is extensible in which many request / response scenarios are laid on. +*This message type MUST be replayable.* + | field | type | description | | --: | :-- | :-- | | type | InfoType | info type |