From 0ab3f2bfa44e32dbcf32156c366259f6fd4b21ec Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 26 Apr 2019 03:55:53 -0300 Subject: [PATCH 1/7] add 1_webhook_avatars migration --- manage/cmd/migration/scripts/1_webhook_avatars.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 manage/cmd/migration/scripts/1_webhook_avatars.sql diff --git a/manage/cmd/migration/scripts/1_webhook_avatars.sql b/manage/cmd/migration/scripts/1_webhook_avatars.sql new file mode 100644 index 0000000..150efbb --- /dev/null +++ b/manage/cmd/migration/scripts/1_webhook_avatars.sql @@ -0,0 +1,13 @@ +-- webhook_avatars table. check issue 46. +CREATE TABLE IF NOT EXISTS webhook_avatars ( + webhook_id bigint, + + -- this is e.g a sha256 hash of EmbedURL.to_md_url + hash text, + + -- we don't hardcode the mediaproxy url here for obvious reasons. + -- the output of EmbedURL.to_md_url goes here. + md_url_redir text, + + PRIMARY KEY (webhook_id, hash) +); From 5a3740f41b2c4b4bc14542ea46bd3c06b9f3bbd3 Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 26 Apr 2019 04:10:29 -0300 Subject: [PATCH 2/7] add basic redirect for webhook avatars --- litecord/blueprints/icons.py | 25 ++++++++++++++++++++++--- litecord/embed/sanitizer.py | 6 +++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/litecord/blueprints/icons.py b/litecord/blueprints/icons.py index 1525f2b..e5f65b0 100644 --- a/litecord/blueprints/icons.py +++ b/litecord/blueprints/icons.py @@ -18,7 +18,8 @@ along with this program. If not, see . """ from os.path import splitext -from quart import Blueprint, current_app as app, send_file +from quart import Blueprint, current_app as app, send_file, redirect +from litecord.embed.sanitizer import make_md_req_url bp = Blueprint('images', __name__) @@ -54,14 +55,32 @@ async def _get_guild_icon(guild_id: int, icon_file: str): return await send_icon('guild', guild_id, icon_hash, ext=ext) -@bp.route('/embed/avatars/.png') -async def _get_default_user_avatar(discrim: int): +@bp.route('/embed/avatars/.png') +async def _get_default_user_avatar(default_id: int): + # TODO: how do we determine which assets to use for this? + # I don't think we can use discord assets. pass +async def _handle_webhook_avatar(md_url_redir: str): + md_url = make_md_req_url(None, 'img', md_url_redir) + return redirect(md_url) + + @bp.route('/avatars//') async def _get_user_avatar(user_id, avatar_file): avatar_hash, ext = splitext_(avatar_file) + + # first, check if this is a webhook avatar to redir to + md_url_redir = await app.db.fetchval(""" + SELECT md_url_redir + FROM webhook_avatars + WHERE webhook_id = $1 AND hash = $2 + """, user_id, avatar_hash) + + if md_url_redir: + return await _handle_webhook_avatar(md_url_redir) + return await send_icon('user', user_id, avatar_hash, ext=ext) diff --git a/litecord/embed/sanitizer.py b/litecord/embed/sanitizer.py index bc9f761..9ef15d5 100644 --- a/litecord/embed/sanitizer.py +++ b/litecord/embed/sanitizer.py @@ -96,7 +96,7 @@ def _md_base(config) -> tuple: return proto, md_base_url -def _make_md_req_url(config, scope: str, url): +def make_md_req_url(config, scope: str, url): """Make a mediaproxy request URL given the config, scope, and the url to be proxied.""" proto, base_url = _md_base(config) @@ -111,7 +111,7 @@ def proxify(url, *, config=None) -> str: if isinstance(url, str): url = EmbedURL(url) - return _make_md_req_url(config, 'img', url) + return make_md_req_url(config, 'img', url) async def _md_client_req(config, session, scope: str, @@ -151,7 +151,7 @@ async def _md_client_req(config, session, scope: str, if not isinstance(url, EmbedURL): url = EmbedURL(url) - request_url = _make_md_req_url(config, scope, url) + request_url = make_md_req_url(config, scope, url) async with session.get(request_url) as resp: if resp.status == 200: From acc52a0c6123d9ece97c4cc59eb8d3992887ed8f Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 26 Apr 2019 16:48:27 -0300 Subject: [PATCH 3/7] add basic checking of webhook avatar mime --- litecord/blueprints/webhooks.py | 20 +++++++++++++++++--- litecord/images.py | 5 +++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index fb91af0..a812a2a 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -46,6 +46,7 @@ from litecord.embed.sanitizer import fill_embed, fetch_raw_img from litecord.embed.messages import process_url_embed, is_media_url from litecord.utils import pg_set_json from litecord.enums import MessageType +from litecord.images import STATIC_IMAGE_MIMES bp = Blueprint('webhooks', __name__) @@ -346,12 +347,12 @@ async def create_message_webhook(guild_id, channel_id, webhook_id, data): return message_id -async def _create_avatar(webhook_id: int, avatar_url): +async def _create_avatar(webhook_id: int, avatar_url) -> str: """Create an avatar for a webhook out of an avatar URL, given when executing the webhook. - Litecord will query that URL via mediaproxy and store the data - via IconManager. + Litecord will write an URL that redirects to the given avatar_url, + using mediaproxy. """ if avatar_url.scheme not in ('http', 'https'): raise BadRequest('invalid avatar url scheme') @@ -359,12 +360,21 @@ async def _create_avatar(webhook_id: int, avatar_url): if not is_media_url(avatar_url): raise BadRequest('url is not media url') + # we still fetch the URL to check its validity, mimetypes, etc + # but in the end, we will store it under the webhook_avatars table, + # not IconManager. resp, raw = await fetch_raw_img(avatar_url) raw_b64 = base64.b64encode(raw).decode() mime = resp.headers['content-type'] + + # TODO: apng checks are missing (for this and everywhere else) + if mime not in STATIC_IMAGE_MIMES: + raise BadRequest('invalid mime type for given url') + b64_data = f'data:{mime};base64,{raw_b64}' + # TODO: replace this by webhook_avatars icon = await app.icons.put( 'user', webhook_id, b64_data, always_icon=True, size=(128, 128) @@ -399,6 +409,10 @@ async def execute_webhook(webhook_id: int, webhook_token): given_embeds = j.get('embeds', []) webhook = await get_webhook(webhook_id) + + # webhooks have TWO avatars. one is from settings, the other is from + # the json's icon_url. one can be handled gracefully by IconManager, + # but the other can't, at all. avatar = webhook['avatar'] if 'avatar_url' in j and j['avatar_url'] is not None: diff --git a/litecord/images.py b/litecord/images.py index ec37469..0ce0595 100644 --- a/litecord/images.py +++ b/litecord/images.py @@ -50,6 +50,11 @@ MIMES = { 'webp': 'image/webp', } +STATIC_IMAGE_MIMES = [ + 'image/png', + 'image/jpeg', + 'image/webp' +] def get_ext(mime: str) -> str: if mime in EXTENSIONS: From addabb53efeecb7cafd563e729b932a6c7a4a04e Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 1 May 2019 18:51:57 -0300 Subject: [PATCH 4/7] add draft for webhook_avatars usage --- litecord/blueprints/webhooks.py | 30 ++++++++++++++++++++++-------- litecord/embed/schemas.py | 5 +++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index a812a2a..3669972 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -19,6 +19,7 @@ along with this program. If not, see . import secrets import base64 +import hashlib from typing import Dict, Any, Optional from quart import Blueprint, jsonify, current_app as app, request @@ -42,8 +43,9 @@ from litecord.blueprints.channel.messages import ( msg_create_request, msg_create_check_content, msg_add_attachment, msg_guild_text_mentions ) -from litecord.embed.sanitizer import fill_embed, fetch_raw_img +from litecord.embed.sanitizer import fill_embed, fetch_raw_img, proxify from litecord.embed.messages import process_url_embed, is_media_url +from litecord.embed.schemas import EmbedURL from litecord.utils import pg_set_json from litecord.enums import MessageType from litecord.images import STATIC_IMAGE_MIMES @@ -347,6 +349,18 @@ async def create_message_webhook(guild_id, channel_id, webhook_id, data): return message_id +async def _webhook_avy_redir(webhook_id: int, avatar_url): + eu_avatar_url = EmbedURL.from_parsed(avatar_url) + url_hash = hashlib.sha256(eu_avatar_url.to_md_path).hexdigest() + + await app.db.execute(""" + INSERT INTO webhook_avatars (webhook_id, hash, md_url_redir) + VALUES ($1, $2, $3) + """, webhook_id, url_hash, proxify(eu_avatar_url)) + + return url_hash + + async def _create_avatar(webhook_id: int, avatar_url) -> str: """Create an avatar for a webhook out of an avatar URL, given when executing the webhook. @@ -364,7 +378,7 @@ async def _create_avatar(webhook_id: int, avatar_url) -> str: # but in the end, we will store it under the webhook_avatars table, # not IconManager. resp, raw = await fetch_raw_img(avatar_url) - raw_b64 = base64.b64encode(raw).decode() + #raw_b64 = base64.b64encode(raw).decode() mime = resp.headers['content-type'] @@ -372,15 +386,15 @@ async def _create_avatar(webhook_id: int, avatar_url) -> str: if mime not in STATIC_IMAGE_MIMES: raise BadRequest('invalid mime type for given url') - b64_data = f'data:{mime};base64,{raw_b64}' + #b64_data = f'data:{mime};base64,{raw_b64}' # TODO: replace this by webhook_avatars - icon = await app.icons.put( - 'user', webhook_id, b64_data, - always_icon=True, size=(128, 128) - ) + #icon = await app.icons.put( + # 'user', webhook_id, b64_data, + # always_icon=True, size=(128, 128) + #) - return icon.icon_hash + return await _webhook_avy_redir(webhook_id, avatar_url) @bp.route('/webhooks//', methods=['POST']) diff --git a/litecord/embed/schemas.py b/litecord/embed/schemas.py index ef60702..6f3b306 100644 --- a/litecord/embed/schemas.py +++ b/litecord/embed/schemas.py @@ -35,6 +35,11 @@ class EmbedURL: self.raw_url = url self.parsed = parsed + @classmethod + def from_parsed(cls, parsed): + """Make an EmbedURL instance out of an already parsed 6-tuple.""" + return cls(parsed.geturl()) + @property def url(self) -> str: """Return the unparsed URL.""" From ecfe4fa45766b0ec66c85af73aeec6c91fc81181 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 1 May 2019 19:09:42 -0300 Subject: [PATCH 5/7] webhooks: remove unused import --- litecord/blueprints/webhooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 3669972..308e01b 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -18,7 +18,6 @@ along with this program. If not, see . """ import secrets -import base64 import hashlib from typing import Dict, Any, Optional From feaef1e463d71e3a95ee61648229080174413719 Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 2 May 2019 02:03:37 -0300 Subject: [PATCH 6/7] webhooks: finish impl for webhook_avy_redir - migration.command: fix apply_migration call --- litecord/blueprints/icons.py | 5 ++++- litecord/blueprints/webhooks.py | 20 ++++++++++++-------- manage/cmd/migration/command.py | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/litecord/blueprints/icons.py b/litecord/blueprints/icons.py index e5f65b0..b01bac7 100644 --- a/litecord/blueprints/icons.py +++ b/litecord/blueprints/icons.py @@ -18,8 +18,11 @@ along with this program. If not, see . """ from os.path import splitext + from quart import Blueprint, current_app as app, send_file, redirect + from litecord.embed.sanitizer import make_md_req_url +from litecord.embed.schemas import EmbedURL bp = Blueprint('images', __name__) @@ -63,7 +66,7 @@ async def _get_default_user_avatar(default_id: int): async def _handle_webhook_avatar(md_url_redir: str): - md_url = make_md_req_url(None, 'img', md_url_redir) + md_url = make_md_req_url(app.config, 'img', EmbedURL(md_url_redir)) return redirect(md_url) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index 308e01b..a86bae0 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -21,6 +21,7 @@ import secrets import hashlib from typing import Dict, Any, Optional +import asyncpg from quart import Blueprint, jsonify, current_app as app, request from litecord.auth import token_check @@ -348,19 +349,22 @@ async def create_message_webhook(guild_id, channel_id, webhook_id, data): return message_id -async def _webhook_avy_redir(webhook_id: int, avatar_url): - eu_avatar_url = EmbedURL.from_parsed(avatar_url) - url_hash = hashlib.sha256(eu_avatar_url.to_md_path).hexdigest() +async def _webhook_avy_redir(webhook_id: int, avatar_url: EmbedURL): + """Create a row on webhook_avatars.""" + url_hash = hashlib.sha256(avatar_url.to_md_path.encode()).hexdigest() - await app.db.execute(""" - INSERT INTO webhook_avatars (webhook_id, hash, md_url_redir) - VALUES ($1, $2, $3) - """, webhook_id, url_hash, proxify(eu_avatar_url)) + try: + await app.db.execute(""" + INSERT INTO webhook_avatars (webhook_id, hash, md_url_redir) + VALUES ($1, $2, $3) + """, webhook_id, url_hash, avatar_url.url) + except asyncpg.UniqueViolationError: + pass return url_hash -async def _create_avatar(webhook_id: int, avatar_url) -> str: +async def _create_avatar(webhook_id: int, avatar_url: EmbedURL) -> str: """Create an avatar for a webhook out of an avatar URL, given when executing the webhook. diff --git a/manage/cmd/migration/command.py b/manage/cmd/migration/command.py index e769b6c..1f1f91f 100644 --- a/manage/cmd/migration/command.py +++ b/manage/cmd/migration/command.py @@ -228,7 +228,7 @@ async def migrate_cmd(app, _args): migration = ctx.scripts.get(idx) print('applying', migration.id, migration.name) - # await apply_migration(app, migration) + await apply_migration(app, migration) def setup(subparser): From 4c4a583ccdd07af2abec6df7d55ae1b82cba0599 Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 2 May 2019 02:10:47 -0300 Subject: [PATCH 7/7] webhooks: remove unused import --- litecord/blueprints/webhooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litecord/blueprints/webhooks.py b/litecord/blueprints/webhooks.py index a86bae0..08712d3 100644 --- a/litecord/blueprints/webhooks.py +++ b/litecord/blueprints/webhooks.py @@ -43,7 +43,7 @@ from litecord.blueprints.channel.messages import ( msg_create_request, msg_create_check_content, msg_add_attachment, msg_guild_text_mentions ) -from litecord.embed.sanitizer import fill_embed, fetch_raw_img, proxify +from litecord.embed.sanitizer import fill_embed, fetch_raw_img from litecord.embed.messages import process_url_embed, is_media_url from litecord.embed.schemas import EmbedURL from litecord.utils import pg_set_json