From ca676f6bfbb4a7aa53f548bdef61f9fbbc8a309a Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 17 Mar 2019 01:33:05 -0300 Subject: [PATCH 01/20] webhooks: add impl to create and fetch webhook list - schemas: add WEBHOOK_CREATE --- litecord/blueprints/webhooks.py | 83 +++++++++++++++++++++++++++++++-- litecord/schemas.py | 8 ++++ 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 06253aa..85e81e3 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -17,19 +17,92 @@ along with this program. If not, see . """ -from quart import Blueprint +from typing import Dict, Any, Optional + +from quart import Blueprint, jsonify, current_app as app, request + +from litecord.auth import token_check +from litecord.blueprints.checks import channel_check, channel_perm_check + +from litecord.schemas import validate, WEBHOOK_CREATE +from litecord.enums import ChannelType +from litecord.snowflake import get_snowflake +from litecord.utils import async_map bp = Blueprint('webhooks', __name__) +async def get_webhook(webhook_id: int) -> Optional[Dict[str, Any]]: + """Get a webhook data""" + row = await app.db.fetchrow(""" + SELECT id::text, guild_id::text, channel_id::text, creator_id + name, avatar, token + FROM webhooks + WHERE id = $1 + """, webhook_id) + + if not row: + return None + + drow = dict(row) + + drow['user'] = await app.storage.get_user(row['creator_id']) + drow.pop('creator_id') + + return drow + + +async def _webhook_check(): + user_id = await token_check() + + await channel_check(user_id, channel_id, ChannelType.GUILD_TEXT) + await channel_perm_check(user_id, channel_id, 'manage_webhooks') + + return user_id + + @bp.route('/channels//webhooks', methods=['POST']) -async def create_webhook(channel_id): - pass +async def create_webhook(channel_id: int): + """Create a webhook given a channel.""" + user_id = await _webhook_check() + + j = validate(await request.get_json(), WEBHOOK_CREATE) + + guild_id = await app.storage.guild_from_channel(channel_id) + + webhook_id = get_snowflake() + token = 'asd' + + await app.db.execute( + """ + INSERT INTO webhooks + (id, guild_id, channel_id, creator_id, name, avatar, token) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + """, + webhook_id, guild_id, channel_id, user_id, + j['name'], j.get('avatar'), token + ) + + return jsonify(await get_webhook(webhook_id)) @bp.route('/channels//webhooks', methods=['GET']) -async def get_channel_webhook(channel_id): - pass +async def get_channel_webhook(channel_id: int): + """Get a list of webhooks in a channel""" + _user_id = await _webhook_check() + + webhook_ids = await app.db.fetch(""" + SELECT id + FROM webhooks + WHERE channel_id = $1 + """, channel_id) + + webhook_ids = [r['id'] for r in webhook_ids] + + return jsonify( + await async_map(get_webhook, webhook_ids) + ) @bp.route('/guilds//webhooks', methods=['GET']) diff --git a/litecord/schemas.py b/litecord/schemas.py index c52cf8f..f3dca79 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -677,3 +677,11 @@ VANITY_URL_PATCH = { # TODO: put proper values in maybe an invite data type 'code': {'type': 'string', 'minlength': 5, 'maxlength': 30} } + +WEBHOOK_CREATE = { + 'name': { + 'type': 'string', 'minlength': 2, 'maxlength': 32, + 'required': True + }, + 'avatar': {'type': 'b64_icon', 'required': False, 'nullable': False} +} From 89009d44c6db1fcacdb9103096e6dc297a5e9376 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 17 Mar 2019 02:00:51 -0300 Subject: [PATCH 02/20] webhooks: fix unused/undefined vars --- litecord/blueprints/webhooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 85e81e3..eb84396 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -52,7 +52,7 @@ async def get_webhook(webhook_id: int) -> Optional[Dict[str, Any]]: return drow -async def _webhook_check(): +async def _webhook_check(channel_id): user_id = await token_check() await channel_check(user_id, channel_id, ChannelType.GUILD_TEXT) @@ -64,7 +64,7 @@ async def _webhook_check(): @bp.route('/channels//webhooks', methods=['POST']) async def create_webhook(channel_id: int): """Create a webhook given a channel.""" - user_id = await _webhook_check() + user_id = await _webhook_check(channel_id) j = validate(await request.get_json(), WEBHOOK_CREATE) @@ -90,7 +90,7 @@ async def create_webhook(channel_id: int): @bp.route('/channels//webhooks', methods=['GET']) async def get_channel_webhook(channel_id: int): """Get a list of webhooks in a channel""" - _user_id = await _webhook_check() + await _webhook_check(channel_id) webhook_ids = await app.db.fetch(""" SELECT id From 5a198de51702a5e6258c40bc43bccd8936b45018 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 17 Mar 2019 03:28:54 -0300 Subject: [PATCH 03/20] webhooks: generate a proper token and insert icon on creation --- litecord/blueprints/webhooks.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index eb84396..6616bba 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -17,6 +17,7 @@ along with this program. If not, see . """ +import secrets from typing import Dict, Any, Optional from quart import Blueprint, jsonify, current_app as app, request @@ -71,7 +72,16 @@ async def create_webhook(channel_id: int): guild_id = await app.storage.guild_from_channel(channel_id) webhook_id = get_snowflake() - token = 'asd' + + # I'd say generating a full fledged token with itsdangerous is + # relatively wasteful since webhooks don't even have a password_hash, + # and we don't make a webhook in the users table either. + token = secrets.token_urlsafe(40) + + webhook_icon = await app.icons.put( + 'user', webhook_id, j.get('avatar'), + always_icon=True, size=(128, 128) + ) await app.db.execute( """ @@ -81,7 +91,7 @@ async def create_webhook(channel_id: int): ($1, $2, $3, $4, $5, $6, $7) """, webhook_id, guild_id, channel_id, user_id, - j['name'], j.get('avatar'), token + j['name'], webhook_icon.icon_hash, token ) return jsonify(await get_webhook(webhook_id)) From eeee05cfe9917f02ca1be962383091aef994df7f Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 17 Mar 2019 03:45:31 -0300 Subject: [PATCH 04/20] webhooks: add more impls --- litecord/blueprints/webhooks.py | 85 ++++++++++++++++++++++++++------- litecord/errors.py | 4 ++ 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 6616bba..b3a573d 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -23,17 +23,21 @@ from typing import Dict, Any, Optional from quart import Blueprint, jsonify, current_app as app, request from litecord.auth import token_check -from litecord.blueprints.checks import channel_check, channel_perm_check +from litecord.blueprints.checks import ( + channel_check, channel_perm_check, guild_check, guild_perm_check +) from litecord.schemas import validate, WEBHOOK_CREATE from litecord.enums import ChannelType from litecord.snowflake import get_snowflake from litecord.utils import async_map +from litecord.errors import WebhookNotFound, Unauthorized bp = Blueprint('webhooks', __name__) -async def get_webhook(webhook_id: int) -> Optional[Dict[str, Any]]: +async def get_webhook(webhook_id: int, *, + secure: bool=True) -> Optional[Dict[str, Any]]: """Get a webhook data""" row = await app.db.fetchrow(""" SELECT id::text, guild_id::text, channel_id::text, creator_id @@ -50,6 +54,10 @@ async def get_webhook(webhook_id: int) -> Optional[Dict[str, Any]]: drow['user'] = await app.storage.get_user(row['creator_id']) drow.pop('creator_id') + if not secure: + drow.pop('user') + drow.pop('guild_id') + return drow @@ -62,6 +70,54 @@ async def _webhook_check(channel_id): return user_id +async def _webhook_check_guild(guild_id): + user_id = await token_check() + + await guild_check(user_id, guild_id) + await guild_perm_check(user_id, guild_id, 'manage_webhooks') + + return user_id + + +async def _webhook_check_fw(webhook_id): + """Make a check from an incoming webhook id (fw = from webhook).""" + guild_id = await app.db.fetchval(""" + SELECT guild_id FROM webhooks + WHERE id = $1 + """, webhook_id) + + if guild_id is None: + raise WebhookNotFound() + + return await _webhook_check_guild(guild_id) + + +async def _webhook_many(where_clause, arg: int): + webhook_ids = await app.db.fetch(f""" + SELECT id + FROM webhooks + {where_clause} + """, arg) + + webhook_ids = [r['id'] for r in webhook_ids] + + return jsonify( + await async_map(get_webhook, webhook_ids) + ) + + +async def webhook_token_check(webhook_id: int, webhook_token: str): + """token_check() equivalent for webhooks.""" + webhook_id_fetch = await app.db.fetchrow(""" + SELECT id + FROM webhooks + WHERE id = $1 AND token = $2 + """, webhook_id, webhook_token) + + if webhook_id_fetch is None: + raise Unauthorized('webhook not found or unauthorized') + + @bp.route('/channels//webhooks', methods=['POST']) async def create_webhook(channel_id: int): """Create a webhook given a channel.""" @@ -101,33 +157,28 @@ async def create_webhook(channel_id: int): async def get_channel_webhook(channel_id: int): """Get a list of webhooks in a channel""" await _webhook_check(channel_id) - - webhook_ids = await app.db.fetch(""" - SELECT id - FROM webhooks - WHERE channel_id = $1 - """, channel_id) - - webhook_ids = [r['id'] for r in webhook_ids] - - return jsonify( - await async_map(get_webhook, webhook_ids) - ) + return await _webhook_many('WHERE channel_id = $1', channel_id) @bp.route('/guilds//webhooks', methods=['GET']) async def get_guild_webhook(guild_id): - pass + """Get all webhooks in a guild""" + await _webhook_check_guild(guild_id) + return await _webhook_many('WHERE guild_id = $1', guild_id) @bp.route('/webhooks/', methods=['GET']) async def get_single_webhook(webhook_id): - pass + """Get a single webhook's information.""" + await _webhook_check_fw(webhook_id) + return await jsonify(await get_webhook(webhook_id)) @bp.route('/webhooks//', methods=['GET']) async def get_tokened_webhook(webhook_id, webhook_token): - pass + """Get a webhook using its token.""" + await webhook_token_check(webhook_id, webhook_token) + return await jsonify(await get_webhook(webhook_id, secure=False)) @bp.route('/webhooks/', methods=['PATCH']) diff --git a/litecord/errors.py b/litecord/errors.py index 64c0ebf..76f937c 100644 --- a/litecord/errors.py +++ b/litecord/errors.py @@ -122,6 +122,10 @@ class MessageNotFound(NotFound): error_code = 10008 +class WebhookNotFound(NotFound): + error_code = 10015 + + class Ratelimited(LitecordError): status_code = 429 From d29fa2e1f33c6fac5e7e4fcb0216f941250bbc06 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 17 Mar 2019 17:35:51 -0300 Subject: [PATCH 05/20] webhooks: add webhook patch impl - schemas: add WEBHOOK_UPDATE --- litecord/blueprints/webhooks.py | 49 ++++++++++++++++++++++++++++++--- litecord/schemas.py | 12 ++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index b3a573d..ead8eb2 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -27,7 +27,7 @@ from litecord.blueprints.checks import ( channel_check, channel_perm_check, guild_check, guild_perm_check ) -from litecord.schemas import validate, WEBHOOK_CREATE +from litecord.schemas import validate, WEBHOOK_CREATE, WEBHOOK_UPDATE from litecord.enums import ChannelType from litecord.snowflake import get_snowflake from litecord.utils import async_map @@ -181,14 +181,55 @@ async def get_tokened_webhook(webhook_id, webhook_token): return await jsonify(await get_webhook(webhook_id, secure=False)) +async def _update_webhook(webhook_id: int, j: dict): + if 'name' in j: + await app.db.execute(""" + UPDATE webhooks + SET name = $1 + WHERE id = $2 + """, j['name'], webhook_id) + + if 'channel_id' in j: + await app.db.execute(""" + UPDATE webhooks + SET channel_id = $1 + WHERE id = $2 + """, j['name'], webhook_id) + + if 'avatar' in j: + new_icon = await app.icons.update( + 'user', webhook_id, j['avatar'], always_icon=True, size=(128, 128) + ) + + await app.db.execute(""" + UPDATE webhooks + SET icon = $1 + WHERE id = $2 + """, new_icon.icon_hash, webhook_id) + + @bp.route('/webhooks/', methods=['PATCH']) -async def modify_webhook(webhook_id): - pass +async def modify_webhook(webhook_id: int): + """Patch a webhook.""" + await _webhook_check_fw(webhook_id) + j = validate(await request.get_json(), WEBHOOK_UPDATE) + + await _update_webhook(webhook_id, j) + return jsonify(await get_webhook(webhook_id)) @bp.route('/webhooks//', methods=['PATCH']) async def modify_webhook_tokened(webhook_id, webhook_token): - pass + """Modify a webhook, using its token.""" + await webhook_token_check(webhook_id, webhook_token) + + # forcefully pop() the channel id out of the schema + # instead of making another, for simplicity's sake + j = validate(await request.get_json(), + WEBHOOK_UPDATE.pop('channel_id')) + + await _update_webhook(webhook_id, j) + return jsonify(await get_webhook(webhook_id, secure=False)) @bp.route('/webhooks/', methods=['DELETE']) diff --git a/litecord/schemas.py b/litecord/schemas.py index f3dca79..10dedf1 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -685,3 +685,15 @@ WEBHOOK_CREATE = { }, 'avatar': {'type': 'b64_icon', 'required': False, 'nullable': False} } + +WEBHOOK_UPDATE = { + 'name': { + 'type': 'string', 'minlength': 2, 'maxlength': 32, + 'required': False + }, + + # TODO: check if its b64_icon or string since the client + # could pass an icon hash instead. + 'avatar': {'type': 'b64_icon', 'required': False, 'nullable': False}, + 'channel_id': {'coerce': int, 'required': False, 'nullable': False} +} From 836063835690df1cf721b6cfbe9ce0abd90ad4fc Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 17 Mar 2019 17:39:59 -0300 Subject: [PATCH 06/20] webhooks: add webhook delete impl --- litecord/blueprints/webhooks.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index ead8eb2..3e7f99d 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -232,14 +232,31 @@ async def modify_webhook_tokened(webhook_id, webhook_token): return jsonify(await get_webhook(webhook_id, secure=False)) +async def delete_webhook(webhook_id: int): + """Delete a webhook.""" + res = await app.db.execute(""" + DELETE FROM webhooks + WHERE id = $1 + """, webhook_id) + + if res.lower() == 'delete 0': + raise WebhookNotFound() + + @bp.route('/webhooks/', methods=['DELETE']) async def del_webhook(webhook_id): - pass + """Delete a webhook.""" + await _webhook_check_fw(webhook_id) + await delete_webhook(webhook_id) + return '', 204 @bp.route('/webhooks//', methods=['DELETE']) async def del_webhook_tokened(webhook_id, webhook_token): - pass + """Delete a webhook, with its token.""" + await webhook_token_check(webhook_id, webhook_token) + await delete_webhook(webhook_id) + return '', 204 @bp.route('/webhooks//', methods=['POST']) From 712f1d9ff20922b4a6606ff79e2bd8cd97012979 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 04:35:44 -0300 Subject: [PATCH 07/20] webhooks: add drafts for execution impl - channel.messages: export msg_create_request, msg_create_check_payload, msg_add_attachment - schemas: add WEBHOOK_MESSAGE_CREATE --- litecord/blueprints/channel/messages.py | 24 +++++--- litecord/blueprints/webhooks.py | 74 ++++++++++++++++++++++--- litecord/schemas.py | 24 ++++++++ 3 files changed, 106 insertions(+), 16 deletions(-) diff --git a/litecord/blueprints/channel/messages.py b/litecord/blueprints/channel/messages.py index 41850e2..2fdd904 100644 --- a/litecord/blueprints/channel/messages.py +++ b/litecord/blueprints/channel/messages.py @@ -249,7 +249,7 @@ async def _guild_text_mentions(payload: dict, guild_id: int, """, user_id, channel_id) -async def _msg_input() -> tuple: +async def msg_create_request() -> tuple: """Extract the json input and any file information the client gave to us in the request. @@ -277,26 +277,34 @@ async def _msg_input() -> tuple: return json_from_form, [v for k, v in files.items()] -def _check_content(payload: dict, files: list): +def msg_create_check_content(payload: dict, files: list, *, use_embeds=False): """Check if there is actually any content being sent to us.""" has_content = bool(payload.get('content', '')) - has_embed = 'embed' in payload has_files = len(files) > 0 + embed_field = 'embeds' if use_embeds else 'embed' + has_embed = embed_field in payload + has_total_content = has_content or has_embed or has_files if not has_total_content: raise BadRequest('No content has been provided.') -async def _add_attachment(message_id: int, channel_id: int, - attachment_file) -> int: +async def msg_add_attachment(message_id: int, channel_id: int, + attachment_file) -> int: """Add an attachment to a message. Parameters ---------- message_id: int The ID of the message getting the attachment. + channel_id: int + The ID of the channel the message belongs to. + + Exists because the attachment URL scheme contains + a channel id. The purpose is unknown, but we are + implementing Discord's behavior. attachment_file: quart.FileStorage quart FileStorage instance of the file. """ @@ -362,10 +370,10 @@ async def _create_message(channel_id): await channel_perm_check(user_id, channel_id, 'send_messages') actual_guild_id = guild_id - payload_json, files = await _msg_input() + payload_json, files = await msg_create_request() j = validate(payload_json, MESSAGE_CREATE) - _check_content(payload_json, files) + msg_create_check_content(payload_json, files) # TODO: check connection to the gateway @@ -399,7 +407,7 @@ async def _create_message(channel_id): # for each file given, we add it as an attachment for pre_attachment in files: - await _add_attachment(message_id, channel_id, pre_attachment) + await msg_add_attachment(message_id, channel_id, pre_attachment) payload = await app.storage.get_message(message_id, user_id) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 3e7f99d..36a686f 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -27,12 +27,21 @@ from litecord.blueprints.checks import ( channel_check, channel_perm_check, guild_check, guild_perm_check ) -from litecord.schemas import validate, WEBHOOK_CREATE, WEBHOOK_UPDATE +from litecord.schemas import ( + validate, WEBHOOK_CREATE, WEBHOOK_UPDATE, WEBHOOK_MESSAGE_CREATE +) from litecord.enums import ChannelType from litecord.snowflake import get_snowflake from litecord.utils import async_map from litecord.errors import WebhookNotFound, Unauthorized +from litecord.blueprints.channel.messages import ( + msg_create_request, msg_create_check_content, msg_add_attachment, + # create_message +) +from litecord.embed.sanitizer import fill_embed +from litecord.embed.messages import process_url_embed + bp = Blueprint('webhooks', __name__) @@ -108,15 +117,17 @@ async def _webhook_many(where_clause, arg: int): async def webhook_token_check(webhook_id: int, webhook_token: str): """token_check() equivalent for webhooks.""" - webhook_id_fetch = await app.db.fetchrow(""" - SELECT id + row = await app.db.fetchrow(""" + SELECT guild_id, channel_id FROM webhooks WHERE id = $1 AND token = $2 """, webhook_id, webhook_token) - if webhook_id_fetch is None: + if row is None: raise Unauthorized('webhook not found or unauthorized') + return row['guild_id'], row['channel_id'] + @bp.route('/channels//webhooks', methods=['POST']) async def create_webhook(channel_id: int): @@ -259,17 +270,64 @@ async def del_webhook_tokened(webhook_id, webhook_token): return '', 204 -@bp.route('/webhooks//', methods=['POST']) -async def execute_webhook(webhook_id, webhook_token): +async def create_message_webhook(guild_id, channel_id, webhook_id, j): + # TODO: impl pass +@bp.route('/webhooks//', methods=['POST']) +async def execute_webhook(webhook_id: int, webhook_token): + """Execute a webhook. Sends a message to the channel the webhook + is tied to.""" + guild_id, channel_id = await webhook_token_check(webhook_id, webhook_token) + + # TODO: ensure channel_id points to guild text channel + + payload_json, files = await msg_create_request() + j = validate(payload_json, WEBHOOK_MESSAGE_CREATE) + + msg_create_check_content(j, files) + + # webhooks don't need permissions. + mentions_everyone = '@everyone' in j['content'] + mentions_here = '@here' in j['content'] + + message_id = await create_message_webhook( + guild_id, channel_id, webhook_id, { + 'content': j.get('content', ''), + 'tts': j.get('tts', False), + + 'everyone_mention': mentions_everyone or mentions_here, + 'embeds': [await fill_embed(e) for e in j['embeds']] + } + ) + + for pre_attachment in files: + await msg_add_attachment(message_id, channel_id, pre_attachment) + + payload = await app.storage.get_message(message_id) + + # spawn embedder in the background, even when we're on a webhook. + app.sched.spawn( + process_url_embed( + app.config, app.storage, app.dispatcher, app.session, + payload + ) + ) + + # TODO: is it really 204? + return '', 204 + @bp.route('/webhooks///slack', methods=['POST']) async def execute_slack_webhook(webhook_id, webhook_token): - pass + """Execute a webhook but expecting Slack data.""" + # TODO: know slack webhooks + await webhook_token_check(webhook_id, webhook_token) @bp.route('/webhooks///github', methods=['POST']) async def execute_github_webhook(webhook_id, webhook_token): - pass + """Execute a webhook but expecting GitHub data.""" + # TODO: know github webhooks + await webhook_token_check(webhook_id, webhook_token) diff --git a/litecord/schemas.py b/litecord/schemas.py index 10dedf1..4d724e2 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -697,3 +697,27 @@ WEBHOOK_UPDATE = { 'avatar': {'type': 'b64_icon', 'required': False, 'nullable': False}, 'channel_id': {'coerce': int, 'required': False, 'nullable': False} } + +WEBHOOK_MESSAGE_CREATE = { + 'content': { + 'type': 'string', + 'minlength': 0, 'maxlength': 2000, 'required': False + }, + 'tts': {'type': 'boolean', 'required': False}, + + 'username': { + 'type': 'string', + 'minlength': 2, 'maxlength': 32, 'required': False + }, + + # TODO: url type, or something... + 'avatar_url': { + 'type': 'url', 'required': False + }, + + 'embeds': { + 'type': list, + 'required': False, + 'schema': {'type': 'dict', 'schema': EMBED_OBJECT} + } +} From e8c158da8167fd464175475c214b677ff6db1e37 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 04:54:23 -0300 Subject: [PATCH 08/20] add 15_drop_nullable_webhook_avatar.sql - webhooks: quickfix to webhook_check --- litecord/blueprints/webhooks.py | 2 +- .../cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql | 2 ++ schema.sql | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 manage/cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 36a686f..9db64fa 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -73,7 +73,7 @@ async def get_webhook(webhook_id: int, *, async def _webhook_check(channel_id): user_id = await token_check() - await channel_check(user_id, channel_id, ChannelType.GUILD_TEXT) + await channel_check(user_id, channel_id, only=ChannelType.GUILD_TEXT) await channel_perm_check(user_id, channel_id, 'manage_webhooks') return user_id diff --git a/manage/cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql b/manage/cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql new file mode 100644 index 0000000..a6e110f --- /dev/null +++ b/manage/cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql @@ -0,0 +1,2 @@ +ALTER TABLE webhooks ALTER COLUMN avatar DROP NOT NULL; +ALTER TABLE webhooks ALTER COLUMN avatar SET DEFAULT NULL; diff --git a/schema.sql b/schema.sql index 32fa564..6437028 100644 --- a/schema.sql +++ b/schema.sql @@ -542,7 +542,7 @@ CREATE TABLE IF NOT EXISTS webhooks ( creator_id bigint REFERENCES users (id), name text NOT NULL, - avatar text NOT NULL, + avatar text DEFAULT NULL, -- Yes, we store the webhook's token -- since they aren't users and there's no /api/login for them. From 379bd75424d3e1ee71ec4deb35c64fa9e5391861 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 04:57:09 -0300 Subject: [PATCH 09/20] webhooks: fix typo on get_webhook's sql --- litecord/blueprints/webhooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 9db64fa..dc3eb40 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -49,7 +49,7 @@ async def get_webhook(webhook_id: int, *, secure: bool=True) -> Optional[Dict[str, Any]]: """Get a webhook data""" row = await app.db.fetchrow(""" - SELECT id::text, guild_id::text, channel_id::text, creator_id + SELECT id::text, guild_id::text, channel_id::text, creator_id, name, avatar, token FROM webhooks WHERE id = $1 From 012567c9d2a2f913589eecbb4271666fc79f3068 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 17:44:30 -0300 Subject: [PATCH 10/20] webhooks: add channel checks when user updates webhook --- litecord/blueprints/webhooks.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index dc3eb40..a604014 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -33,7 +33,7 @@ from litecord.schemas import ( from litecord.enums import ChannelType from litecord.snowflake import get_snowflake from litecord.utils import async_map -from litecord.errors import WebhookNotFound, Unauthorized +from litecord.errors import WebhookNotFound, Unauthorized, ChannelNotFound from litecord.blueprints.channel.messages import ( msg_create_request, msg_create_check_content, msg_add_attachment, @@ -98,7 +98,7 @@ async def _webhook_check_fw(webhook_id): if guild_id is None: raise WebhookNotFound() - return await _webhook_check_guild(guild_id) + return (await _webhook_check_guild(guild_id)), guild_id async def _webhook_many(where_clause, arg: int): @@ -205,7 +205,7 @@ async def _update_webhook(webhook_id: int, j: dict): UPDATE webhooks SET channel_id = $1 WHERE id = $2 - """, j['name'], webhook_id) + """, j['channel_id'], webhook_id) if 'avatar' in j: new_icon = await app.icons.update( @@ -222,9 +222,18 @@ async def _update_webhook(webhook_id: int, j: dict): @bp.route('/webhooks/', methods=['PATCH']) async def modify_webhook(webhook_id: int): """Patch a webhook.""" - await _webhook_check_fw(webhook_id) + guild_id, _user_id = await _webhook_check_fw(webhook_id) j = validate(await request.get_json(), WEBHOOK_UPDATE) + if 'channel_id' in j: + # pre checks + chan = await app.storage.get_channel(j['channel_id']) + + # short-circuiting should ensure chan isn't none + # by the time we do chan['guild_id'] + if chan and chan['guild_id'] != str(guild_id): + raise ChannelNotFound('cant assign webhook to channel') + await _update_webhook(webhook_id, j) return jsonify(await get_webhook(webhook_id)) From 8c43bdcee7c19fe82b20e5d4992e930148bc1999 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 17:50:59 -0300 Subject: [PATCH 11/20] webhooks: fix unpack order - run: add warn calls for expected errors --- litecord/blueprints/webhooks.py | 2 +- run.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index a604014..b333804 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -222,7 +222,7 @@ async def _update_webhook(webhook_id: int, j: dict): @bp.route('/webhooks/', methods=['PATCH']) async def modify_webhook(webhook_id: int): """Patch a webhook.""" - guild_id, _user_id = await _webhook_check_fw(webhook_id) + _user_id, guild_id = await _webhook_check_fw(webhook_id) j = validate(await request.get_json(), WEBHOOK_UPDATE) if 'channel_id' in j: diff --git a/run.py b/run.py index f8bf1c7..ef5e118 100644 --- a/run.py +++ b/run.py @@ -373,6 +373,8 @@ async def handle_litecord_err(err): except AttributeError: pass + log.warning('error: {} {!r}', err.status_code, err.message) + return jsonify({ 'error': True, 'status': err.status_code, From 3866b1f2bc6fdb7b23bdfefe1199c9260e81509c Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 18:00:27 -0300 Subject: [PATCH 12/20] webhooks: add WEBHOOKS_UPDATE event dispatch --- litecord/blueprints/webhooks.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index b333804..61670d0 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -129,6 +129,14 @@ async def webhook_token_check(webhook_id: int, webhook_token: str): return row['guild_id'], row['channel_id'] +async def _dispatch_webhook_update(guild_id: int, channel_id): + await app.dispatcher.dispatch('guild', guild_id, 'WEBHOOKS_UPDATE', { + 'guild_id': str(guild_id), + 'channel_id': str(channel_id) + }) + + + @bp.route('/channels//webhooks', methods=['POST']) async def create_webhook(channel_id: int): """Create a webhook given a channel.""" @@ -161,6 +169,7 @@ async def create_webhook(channel_id: int): j['name'], webhook_icon.icon_hash, token ) + await _dispatch_webhook_update(guild_id, channel_id) return jsonify(await get_webhook(webhook_id)) @@ -235,13 +244,20 @@ async def modify_webhook(webhook_id: int): raise ChannelNotFound('cant assign webhook to channel') await _update_webhook(webhook_id, j) - return jsonify(await get_webhook(webhook_id)) + + webhook = await get_webhook(webhook_id) + + # we don't need to cast channel_id to int since that isn't + # used in the dispatcher call + await _dispatch_webhook_update(guild_id, webhook['channel_id']) + return jsonify(webhook) @bp.route('/webhooks//', methods=['PATCH']) async def modify_webhook_tokened(webhook_id, webhook_token): """Modify a webhook, using its token.""" - await webhook_token_check(webhook_id, webhook_token) + guild_id, channel_id = await webhook_token_check( + webhook_id, webhook_token) # forcefully pop() the channel id out of the schema # instead of making another, for simplicity's sake @@ -249,11 +265,14 @@ async def modify_webhook_tokened(webhook_id, webhook_token): WEBHOOK_UPDATE.pop('channel_id')) await _update_webhook(webhook_id, j) + await _dispatch_webhook_update(guild_id, channel_id) return jsonify(await get_webhook(webhook_id, secure=False)) async def delete_webhook(webhook_id: int): """Delete a webhook.""" + webhook = await get_webhook(webhook_id) + res = await app.db.execute(""" DELETE FROM webhooks WHERE id = $1 @@ -262,6 +281,12 @@ async def delete_webhook(webhook_id: int): if res.lower() == 'delete 0': raise WebhookNotFound() + # only casting the guild id since that's whats used + # on the dispatcher call. + await _dispatch_webhook_update( + int(webhook['guild_id']), webhook['channel_id'] + ) + @bp.route('/webhooks/', methods=['DELETE']) async def del_webhook(webhook_id): From 02c6741d743d6a4918a6eb605c8c832e78ce88ed Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 22:47:59 -0300 Subject: [PATCH 13/20] webhooks: "finish" execution impl it can do messages. but many things are still missing. - channel.messages: export msg_guild_text_mentions - storage: revamp Storage._inject_author - schema.sql: remove messages.webhook_id, add message_webhook_info table - add 16_messages_webhooks.sql migration --- litecord/blueprints/channel/messages.py | 9 ++-- litecord/blueprints/webhooks.py | 52 +++++++++++++++++-- litecord/schemas.py | 5 +- litecord/storage.py | 30 +++++++---- manage/cmd/migration/16_messages_webhooks.sql | 17 ++++++ schema.sql | 17 +++--- 6 files changed, 103 insertions(+), 27 deletions(-) create mode 100644 manage/cmd/migration/16_messages_webhooks.sql diff --git a/litecord/blueprints/channel/messages.py b/litecord/blueprints/channel/messages.py index 2fdd904..2e15825 100644 --- a/litecord/blueprints/channel/messages.py +++ b/litecord/blueprints/channel/messages.py @@ -194,8 +194,9 @@ async def create_message(channel_id: int, actual_guild_id: int, return message_id -async def _guild_text_mentions(payload: dict, guild_id: int, - mentions_everyone: bool, mentions_here: bool): +async def msg_guild_text_mentions(payload: dict, guild_id: int, + mentions_everyone: bool, mentions_here: bool): + """Calculates mention data side-effects.""" channel_id = int(payload['channel_id']) # calculate the user ids we'll bump the mention count for @@ -435,8 +436,8 @@ async def _create_message(channel_id): """, message_id, channel_id, user_id) if ctype == ChannelType.GUILD_TEXT: - await _guild_text_mentions(payload, guild_id, - mentions_everyone, mentions_here) + await msg_guild_text_mentions( + payload, guild_id, mentions_everyone, mentions_here) return jsonify(payload) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 61670d0..30b9c23 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -37,10 +37,12 @@ from litecord.errors import WebhookNotFound, Unauthorized, ChannelNotFound from litecord.blueprints.channel.messages import ( msg_create_request, msg_create_check_content, msg_add_attachment, - # create_message + msg_guild_text_mentions ) from litecord.embed.sanitizer import fill_embed from litecord.embed.messages import process_url_embed +from litecord.utils import pg_set_json +from litecord.enums import MessageType bp = Blueprint('webhooks', __name__) @@ -304,9 +306,34 @@ async def del_webhook_tokened(webhook_id, webhook_token): return '', 204 -async def create_message_webhook(guild_id, channel_id, webhook_id, j): - # TODO: impl - pass +async def create_message_webhook(guild_id, channel_id, webhook_id, data): + """Create a message, but for webhooks only.""" + message_id = get_snowflake() + + async with app.db.acquire() as conn: + await pg_set_json(conn) + + await conn.execute( + """ + INSERT INTO messages (id, channel_id, guild_id, webhook_id, + content, tts, mention_everyone, message_type, embeds) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + """, + message_id, + channel_id, + guild_id, + webhook_id, + data['content'], + + data['tts'], + data['everyone_mention'], + + MessageType.DEFAULT.value, + data.get('embeds', []) + ) + + return message_id + @bp.route('/webhooks//', methods=['POST']) @@ -318,6 +345,12 @@ async def execute_webhook(webhook_id: int, webhook_token): # TODO: ensure channel_id points to guild text channel payload_json, files = await msg_create_request() + + # NOTE: we really pop here instead of adding a kwarg + # to msg_create_request just because of webhooks. + # nonce isn't allowed on WEBHOOK_MESSAGE_CREATE + payload_json.pop('nonce') + j = validate(payload_json, WEBHOOK_MESSAGE_CREATE) msg_create_check_content(j, files) @@ -326,13 +359,15 @@ async def execute_webhook(webhook_id: int, webhook_token): mentions_everyone = '@everyone' in j['content'] mentions_here = '@here' in j['content'] + given_embeds = j.get('embeds', []) + message_id = await create_message_webhook( guild_id, channel_id, webhook_id, { 'content': j.get('content', ''), 'tts': j.get('tts', False), 'everyone_mention': mentions_everyone or mentions_here, - 'embeds': [await fill_embed(e) for e in j['embeds']] + 'embeds': await async_map(fill_embed, given_embeds) } ) @@ -341,6 +376,9 @@ async def execute_webhook(webhook_id: int, webhook_token): payload = await app.storage.get_message(message_id) + await app.dispatcher.dispatch('channel', channel_id, + 'MESSAGE_CREATE', payload) + # spawn embedder in the background, even when we're on a webhook. app.sched.spawn( process_url_embed( @@ -349,6 +387,10 @@ async def execute_webhook(webhook_id: int, webhook_token): ) ) + # we can assume its a guild text channel, so just call it + await msg_guild_text_mentions( + payload, guild_id, mentions_everyone, mentions_here) + # TODO: is it really 204? return '', 204 diff --git a/litecord/schemas.py b/litecord/schemas.py index 4d724e2..fd1553e 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -712,11 +712,12 @@ WEBHOOK_MESSAGE_CREATE = { # TODO: url type, or something... 'avatar_url': { - 'type': 'url', 'required': False + # 'type': 'url', 'required': False + 'type': 'string', 'required': False }, 'embeds': { - 'type': list, + 'type': 'list', 'required': False, 'schema': {'type': 'dict', 'schema': EMBED_OBJECT} } diff --git a/litecord/storage.py b/litecord/storage.py index 72812a7..c42977d 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -796,18 +796,28 @@ class Storage: return res async def _inject_author(self, res: dict): - """Inject a pseudo-user object when the message is made by a webhook.""" - author_id, webhook_id = res['author_id'], res['webhook_id'] + """Inject a pseudo-user object when the message is + made by a webhook.""" + author_id = res['author_id'] - if author_id is not None: + # if author_id is None, we fetch webhook info + # from the message_webhook_info table. + if author_id is None: + wb_info = await self.db.fetchrow(""" + SELECT webhook_id, name, avatar + FROM message_webhook_info + WHERE message_id = $1 + """, int(res['id'])) + + res['author'] = { + 'id': str(wb_info['id']), + 'bot': True, + 'username': wb_info['name'], + 'avatar': wb_info['avatar'] + } + else: res['author'] = await self.get_user(res['author_id']) res.pop('webhook_id') - elif webhook_id is not None: - res['author'] = { - 'id': webhook_id, - 'username': 'a', - 'avatar': None - } res.pop('author_id') @@ -815,7 +825,7 @@ class Storage: user_id: Optional[int] = None) -> Optional[Dict]: """Get a single message's payload.""" row = await self.fetchrow_with_json(""" - SELECT id::text, channel_id::text, author_id, webhook_id, content, + SELECT id::text, channel_id::text, author_id, content, created_at AS timestamp, edited_at AS edited_timestamp, tts, mention_everyone, nonce, message_type, embeds FROM messages diff --git a/manage/cmd/migration/16_messages_webhooks.sql b/manage/cmd/migration/16_messages_webhooks.sql new file mode 100644 index 0000000..91aaad2 --- /dev/null +++ b/manage/cmd/migration/16_messages_webhooks.sql @@ -0,0 +1,17 @@ +-- this is a tricky one. blame discord + +-- first, remove all messages made by webhooks +DELETE FROM messages WHERE author_id is null; + +-- delete the row, removing the fkey. no connection anymore. +ALTER TABLE messages DROP COLUMN webhook_id; + +-- add a message_webhook_info table. more on that in Storage._inject_author +CREATE TABLE IF NOT EXISTS message_webhook_info ( + message_id bigint REFERENCES messages (id) PRIMARY KEY, + + webhook_id bigint, + name text DEFAULT '', + avatar text DEFAULT NULL +); + diff --git a/schema.sql b/schema.sql index 6437028..6b93dd7 100644 --- a/schema.sql +++ b/schema.sql @@ -643,13 +643,9 @@ CREATE TABLE IF NOT EXISTS messages ( -- this is good for search. guild_id bigint REFERENCES guilds (id) ON DELETE CASCADE, - -- those are mutually exclusive, only one of them - -- can NOT be NULL at a time. - - -- if author is NULL -> message from webhook - -- if webhook is NULL -> message from author + -- if author is NULL -> message from webhook -> + -- fetch from message_webhook_info author_id bigint REFERENCES users (id), - webhook_id bigint REFERENCES webhooks (id), content text, @@ -666,6 +662,15 @@ CREATE TABLE IF NOT EXISTS messages ( message_type int NOT NULL ); + +CREATE TABLE IF NOT EXISTS message_webhook_info ( + message_id bigint REFERENCES messages (id) PRIMARY KEY, + + webhook_id bigint, + name text DEFAULT '', + avatar text DEFAULT NULL +); + CREATE TABLE IF NOT EXISTS message_reactions ( message_id bigint REFERENCES messages (id), user_id bigint REFERENCES users (id), From dab830066bcf67beded6c70a611a452a97297f1d Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 23:00:16 -0300 Subject: [PATCH 14/20] 16_messages_webhooks.sql: fix comments - storage: add explanation for wb_info --- litecord/storage.py | 4 ++++ manage/cmd/migration/{ => scripts}/16_messages_webhooks.sql | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) rename manage/cmd/migration/{ => scripts}/16_messages_webhooks.sql (76%) diff --git a/litecord/storage.py b/litecord/storage.py index c42977d..d4dbae3 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -803,6 +803,10 @@ class Storage: # if author_id is None, we fetch webhook info # from the message_webhook_info table. if author_id is None: + # webhook information in a message when made by a webhook + # is copied from the webhook table, or inserted by the webhook + # itself. this causes a complete disconnect from the messages + # table into the webhooks table. wb_info = await self.db.fetchrow(""" SELECT webhook_id, name, avatar FROM message_webhook_info diff --git a/manage/cmd/migration/16_messages_webhooks.sql b/manage/cmd/migration/scripts/16_messages_webhooks.sql similarity index 76% rename from manage/cmd/migration/16_messages_webhooks.sql rename to manage/cmd/migration/scripts/16_messages_webhooks.sql index 91aaad2..fc8ba66 100644 --- a/manage/cmd/migration/16_messages_webhooks.sql +++ b/manage/cmd/migration/scripts/16_messages_webhooks.sql @@ -1,9 +1,9 @@ -- this is a tricky one. blame discord --- first, remove all messages made by webhooks +-- first, remove all messages made by webhooks (safety check) DELETE FROM messages WHERE author_id is null; --- delete the row, removing the fkey. no connection anymore. +-- delete the column, removing the fkey. no connection anymore. ALTER TABLE messages DROP COLUMN webhook_id; -- add a message_webhook_info table. more on that in Storage._inject_author From 507393678e4cf57bfdee1bd19fa8f7f9c255479a Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 23:03:35 -0300 Subject: [PATCH 15/20] litecord.storage: inject discriminator on webhook author --- litecord/storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litecord/storage.py b/litecord/storage.py index d4dbae3..31c836f 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -816,6 +816,7 @@ class Storage: res['author'] = { 'id': str(wb_info['id']), 'bot': True, + 'discriminator': '0000', 'username': wb_info['name'], 'avatar': wb_info['avatar'] } From 1d9c9f7b852e37a6a82b85fba2dcf69a8d9af795 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 18 Mar 2019 23:08:10 -0300 Subject: [PATCH 16/20] schemas: change avatar_url in webhook exec to EmbedURL coerce - embed.schemas: add EmbedURL.scheme --- litecord/embed/schemas.py | 8 +++++--- litecord/schemas.py | 6 ++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/litecord/embed/schemas.py b/litecord/embed/schemas.py index 61452c6..63d8246 100644 --- a/litecord/embed/schemas.py +++ b/litecord/embed/schemas.py @@ -31,16 +31,18 @@ class EmbedURL: if parsed.scheme not in ('http', 'https', 'attachment'): raise ValueError('Invalid URL scheme') + self.scheme = parsed.scheme self.raw_url = url self.parsed = parsed @property - def url(self): - """Return the URL.""" + def url(self) -> str: + """Return the unparsed URL.""" return urllib.parse.urlunparse(self.parsed) @property - def to_json(self): + def to_json(self) -> str: + """'json' version of the url.""" return self.url diff --git a/litecord/schemas.py b/litecord/schemas.py index fd1553e..980a7ff 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -31,7 +31,7 @@ from .enums import ( MessageNotifications, ChannelType, VerificationLevel ) -from litecord.embed.schemas import EMBED_OBJECT +from litecord.embed.schemas import EMBED_OBJECT, EmbedURL log = Logger(__name__) @@ -710,10 +710,8 @@ WEBHOOK_MESSAGE_CREATE = { 'minlength': 2, 'maxlength': 32, 'required': False }, - # TODO: url type, or something... 'avatar_url': { - # 'type': 'url', 'required': False - 'type': 'string', 'required': False + 'coerce': EmbedURL, 'required': False }, 'embeds': { From 404783534c23a8ddb14e92f58791297cd94efbf4 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 19 Mar 2019 00:04:42 -0300 Subject: [PATCH 17/20] webhooks: add draft for avatar_url on webhook execute this is a bit messy due to us having to call mediaproxy for the given url, then store it on icons manager. - embed.messages: add is_media_url - embed.sanitizer: add fetch_raw_img - embed.schemas: add EmbedURL.to_md_path --- litecord/blueprints/icons.py | 1 + litecord/blueprints/webhooks.py | 49 +++++++++++++++++++++++++++++--- litecord/embed/messages.py | 14 +++++---- litecord/embed/sanitizer.py | 50 +++++++++++++++++++++++++-------- litecord/embed/schemas.py | 6 ++++ 5 files changed, 100 insertions(+), 20 deletions(-) diff --git a/litecord/blueprints/icons.py b/litecord/blueprints/icons.py index 7434bad..1525f2b 100644 --- a/litecord/blueprints/icons.py +++ b/litecord/blueprints/icons.py @@ -27,6 +27,7 @@ async def send_icon(scope, key, icon_hash, **kwargs): """Send an icon.""" icon = await app.icons.generic_get( scope, key, icon_hash, **kwargs) + if not icon: return '', 404 diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 30b9c23..d17c87e 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -18,6 +18,7 @@ along with this program. If not, see . """ import secrets +import base64 from typing import Dict, Any, Optional from quart import Blueprint, jsonify, current_app as app, request @@ -33,14 +34,16 @@ from litecord.schemas import ( from litecord.enums import ChannelType from litecord.snowflake import get_snowflake from litecord.utils import async_map -from litecord.errors import WebhookNotFound, Unauthorized, ChannelNotFound +from litecord.errors import ( + WebhookNotFound, Unauthorized, ChannelNotFound, BadRequest +) from litecord.blueprints.channel.messages import ( msg_create_request, msg_create_check_content, msg_add_attachment, msg_guild_text_mentions ) -from litecord.embed.sanitizer import fill_embed -from litecord.embed.messages import process_url_embed +from litecord.embed.sanitizer import fill_embed, fetch_raw_img +from litecord.embed.messages import process_url_embed, is_media_url from litecord.utils import pg_set_json from litecord.enums import MessageType @@ -335,6 +338,32 @@ async def create_message_webhook(guild_id, channel_id, webhook_id, data): return message_id +async def _create_avatar(webhook_id: int, avatar_url): + """Create an avatar for a webhook out of an avatar URL, + given when executing the webhook. + + Litecord will query that URL via mediaproxy and store the data + via IconManager. + """ + if avatar_url.scheme not in ('http', 'https'): + raise BadRequest('invalid avatar url scheme') + + if not is_media_url(avatar_url): + raise BadRequest('url is not media url') + + resp, raw = await fetch_raw_img(avatar_url) + raw_b64 = base64.b64encode(raw) + + mime = resp.headers['content-type'] + b64_data = f'data:{mime};base64,{raw_b64}' + + icon = await app.icons.put( + 'user', webhook_id, b64_data, + always_icon=True, size=(128, 128) + ) + + return icon.icon_hash + @bp.route('/webhooks//', methods=['POST']) async def execute_webhook(webhook_id: int, webhook_token): @@ -361,13 +390,25 @@ async def execute_webhook(webhook_id: int, webhook_token): given_embeds = j.get('embeds', []) + webhook = await get_webhook(webhook_id) + avatar = webhook['avatar'] + + if 'avatar_url' in j and j['avatar_url'] is not None: + avatar = await _create_avatar(webhook_id, j['avatar_url']) + message_id = await create_message_webhook( guild_id, channel_id, webhook_id, { 'content': j.get('content', ''), 'tts': j.get('tts', False), 'everyone_mention': mentions_everyone or mentions_here, - 'embeds': await async_map(fill_embed, given_embeds) + 'embeds': await async_map(fill_embed, given_embeds), + + 'info': { + 'id': webhook_id, + 'name': j.get('name', webhook['name']), + 'avatar': avatar + } } ) diff --git a/litecord/embed/messages.py b/litecord/embed/messages.py index f13b60e..70bf6f9 100644 --- a/litecord/embed/messages.py +++ b/litecord/embed/messages.py @@ -85,6 +85,14 @@ async def _update_and_dispatch(payload, new_embeds, storage, dispatcher): 'channel', channel_id, 'MESSAGE_UPDATE', update_payload) +def is_media_url(url: str) -> bool: + parsed = urllib.parse.urlparse(url) + path = Path(parsed.path) + extension = path.suffix.lstrip('.') + + return extension in MEDIA_EXTENSIONS: + + async def insert_mp_embed(parsed, config, session): """Insert mediaproxy embed.""" embed = await fetch_embed(parsed, config=config, session=session) @@ -125,11 +133,7 @@ async def process_url_embed(config, storage, dispatcher, new_embeds = [] for url in urls: - parsed = urllib.parse.urlparse(url) - path = Path(parsed.path) - extension = path.suffix.lstrip('.') - - if extension in MEDIA_EXTENSIONS: + if is_media_url(url): embed = await insert_media_meta(url, config, session) else: embed = await insert_mp_embed(parsed, config, session) diff --git a/litecord/embed/sanitizer.py b/litecord/embed/sanitizer.py index ef912c4..40959c0 100644 --- a/litecord/embed/sanitizer.py +++ b/litecord/embed/sanitizer.py @@ -96,26 +96,32 @@ def proxify(url, *, config=None) -> str: ) -async def fetch_metadata(url, *, config=None, session=None) -> Optional[Dict]: - """Fetch metadata for a url.""" +def _mk_cfg_sess(config, session) -> tuple: + if config is None: + config = app.config if session is None: session = app.session - if config is None: - config = app.config + return config, session - if isinstance(url, str): - url = EmbedURL(url) - - parsed = url.parsed - - md_path = f'{parsed.scheme}/{parsed.netloc}{parsed.path}' +def _md_base(config) -> tuple: md_base_url = config['MEDIA_PROXY'] proto = 'https' if config['IS_SSL'] else 'http' - request_url = f'{proto}://{md_base_url}/meta/{md_path}' + return proto, md_base_url + + +async def fetch_metadata(url, *, config=None, session=None) -> Optional[Dict]: + """Fetch metadata for a url.""" + config, session = _mk_cfg_sess(config, session) + + if not isinstance(url, EmbedURL): + url = EmbedURL(url) + + proto, md_base_url = _md_base(config) + request_url = f'{proto}://{md_base_url}/meta/{url.to_md_path}' async with session.get(request_url) as resp: if resp.status != 200: @@ -128,6 +134,28 @@ async def fetch_metadata(url, *, config=None, session=None) -> Optional[Dict]: return await resp.json() +async def fetch_raw_img(url, *, config=None, session=None) -> Optional[tuple]: + """Fetch metadata for a url.""" + config, session = _mk_cfg_sess(config, session) + + if not isinstance(url, EmbedURL): + url = EmbedURL(url) + + proto, md_base_url = _md_base(config) + # NOTE: the img, instead of /meta/. + request_url = f'{proto}://{md_base_url}/img/{url.to_md_path}' + + async with session.get(request_url) as resp: + if resp.status != 200: + body = await resp.text() + + log.warning('failed to get img for {!r}: {} {!r}', + url, resp.status, body) + return None + + return resp, await resp.read() + + async def fetch_embed(parsed, *, config=None, session=None) -> dict: """Fetch an embed""" diff --git a/litecord/embed/schemas.py b/litecord/embed/schemas.py index 63d8246..819e072 100644 --- a/litecord/embed/schemas.py +++ b/litecord/embed/schemas.py @@ -45,6 +45,12 @@ class EmbedURL: """'json' version of the url.""" return self.url + @property + def to_md_path(self) -> str: + """Convert the EmbedURL to a mediaproxy path.""" + parsed = self.parsed + return f'{parsed.scheme}/{parsed.netloc}{parsed.path}' + EMBED_FOOTER = { 'text': { From 13720904bd1cd99649d88c8e15c259770ffaf488 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 19 Mar 2019 00:06:52 -0300 Subject: [PATCH 18/20] embed.messages: fix syntax, add docstring --- litecord/embed/messages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litecord/embed/messages.py b/litecord/embed/messages.py index 70bf6f9..7c4e0c8 100644 --- a/litecord/embed/messages.py +++ b/litecord/embed/messages.py @@ -86,11 +86,12 @@ async def _update_and_dispatch(payload, new_embeds, storage, dispatcher): def is_media_url(url: str) -> bool: + """Return if the given URL is a media url.""" parsed = urllib.parse.urlparse(url) path = Path(parsed.path) extension = path.suffix.lstrip('.') - return extension in MEDIA_EXTENSIONS: + return extension in MEDIA_EXTENSIONS async def insert_mp_embed(parsed, config, session): From bc13e06eee81cc221c5eae7b6e9793a589e140dd Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 19 Mar 2019 00:29:49 -0300 Subject: [PATCH 19/20] webhooks: fixes from table changes - webhooks: fix create avatar base64 - webhooks: fix info.name - embed.messages: handle EmbedURL on is_media_url - storage: add fallback info for when wb_info is none - storage: fix pop of non-existing webhook_id --- litecord/blueprints/webhooks.py | 19 +++++++++++++------ litecord/embed/messages.py | 10 ++++++++-- litecord/storage.py | 16 +++++++++++++--- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index d17c87e..fb91af0 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -318,14 +318,13 @@ async def create_message_webhook(guild_id, channel_id, webhook_id, data): await conn.execute( """ - INSERT INTO messages (id, channel_id, guild_id, webhook_id, + INSERT INTO messages (id, channel_id, guild_id, content, tts, mention_everyone, message_type, embeds) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) """, message_id, channel_id, guild_id, - webhook_id, data['content'], data['tts'], @@ -335,6 +334,15 @@ async def create_message_webhook(guild_id, channel_id, webhook_id, data): data.get('embeds', []) ) + info = data['info'] + + await conn.execute(""" + INSERT INTO message_webhook_info + (message_id, webhook_id, name, avatar) + VALUES + ($1, $2, $3, $4) + """, message_id, webhook_id, info['name'], info['avatar']) + return message_id @@ -352,7 +360,7 @@ async def _create_avatar(webhook_id: int, avatar_url): raise BadRequest('url is not media url') resp, raw = await fetch_raw_img(avatar_url) - raw_b64 = base64.b64encode(raw) + raw_b64 = base64.b64encode(raw).decode() mime = resp.headers['content-type'] b64_data = f'data:{mime};base64,{raw_b64}' @@ -405,8 +413,7 @@ async def execute_webhook(webhook_id: int, webhook_token): 'embeds': await async_map(fill_embed, given_embeds), 'info': { - 'id': webhook_id, - 'name': j.get('name', webhook['name']), + 'name': j.get('username', webhook['name']), 'avatar': avatar } } diff --git a/litecord/embed/messages.py b/litecord/embed/messages.py index 7c4e0c8..e4989d1 100644 --- a/litecord/embed/messages.py +++ b/litecord/embed/messages.py @@ -25,6 +25,7 @@ from pathlib import Path from logbook import Logger from litecord.embed.sanitizer import proxify, fetch_metadata, fetch_embed +from litecord.embed.schemas import EmbedURL log = Logger(__name__) @@ -85,9 +86,14 @@ async def _update_and_dispatch(payload, new_embeds, storage, dispatcher): 'channel', channel_id, 'MESSAGE_UPDATE', update_payload) -def is_media_url(url: str) -> bool: +def is_media_url(url) -> bool: """Return if the given URL is a media url.""" - parsed = urllib.parse.urlparse(url) + + if isinstance(url, EmbedURL): + parsed = url.parsed + else: + parsed = urllib.parse.urlparse(url) + path = Path(parsed.path) extension = path.suffix.lstrip('.') diff --git a/litecord/storage.py b/litecord/storage.py index 31c836f..600e486 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -813,16 +813,26 @@ class Storage: WHERE message_id = $1 """, int(res['id'])) - res['author'] = { - 'id': str(wb_info['id']), + if not wb_info: + log.warning('webhook info not found for msg {}', + res['id']) + + wb_info = wb_info or { + 'id': res['id'], 'bot': True, + 'avatar': None, + 'username': '', 'discriminator': '0000', + } + + res['author'] = { + 'id': str(wb_info['webhook_id']), + 'bot': True, 'username': wb_info['name'], 'avatar': wb_info['avatar'] } else: res['author'] = await self.get_user(res['author_id']) - res.pop('webhook_id') res.pop('author_id') From 4a56500adbfa67d8997378cd6dccd305e71069ea Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 19 Mar 2019 00:44:34 -0300 Subject: [PATCH 20/20] embed.messages: always pass url as EmbedURL - embed.sanitizer: handle non-EmbedURL on fetch_embed --- litecord/embed/messages.py | 4 +++- litecord/embed/sanitizer.py | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/litecord/embed/messages.py b/litecord/embed/messages.py index e4989d1..3025945 100644 --- a/litecord/embed/messages.py +++ b/litecord/embed/messages.py @@ -140,10 +140,12 @@ async def process_url_embed(config, storage, dispatcher, new_embeds = [] for url in urls: + url = EmbedURL(url) + if is_media_url(url): embed = await insert_media_meta(url, config, session) else: - embed = await insert_mp_embed(parsed, config, session) + embed = await insert_mp_embed(url, config, session) if not embed: continue diff --git a/litecord/embed/sanitizer.py b/litecord/embed/sanitizer.py index 40959c0..e3b4515 100644 --- a/litecord/embed/sanitizer.py +++ b/litecord/embed/sanitizer.py @@ -156,14 +156,14 @@ async def fetch_raw_img(url, *, config=None, session=None) -> Optional[tuple]: return resp, await resp.read() -async def fetch_embed(parsed, *, config=None, session=None) -> dict: +async def fetch_embed(url, *, config=None, session=None) -> dict: """Fetch an embed""" + config, session = _mk_cfg_sess(config, session) - if session is None: - session = app.session + if not isinstance(url, EmbedURL): + url = EmbedURL(url) - if config is None: - config = app.config + parsed = url.parsed # TODO: handle query string md_path = f'{parsed.scheme}/{parsed.netloc}{parsed.path}'