From 060f2251898f610cf35facfe06c05ec142218bb5 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 5 Dec 2018 19:07:43 -0300 Subject: [PATCH] channel.messages: add url autoembeds related to #27 - embed.sanitizer: handle external config and session objects - storage: add Storage.execute_with_json --- litecord/blueprints/channel/messages.py | 73 ++++++++++++++++++++++++- litecord/embed/sanitizer.py | 32 ++++++++--- litecord/storage.py | 6 ++ 3 files changed, 102 insertions(+), 9 deletions(-) diff --git a/litecord/blueprints/channel/messages.py b/litecord/blueprints/channel/messages.py index 9f4d3ea..144c14c 100644 --- a/litecord/blueprints/channel/messages.py +++ b/litecord/blueprints/channel/messages.py @@ -1,5 +1,6 @@ -from quart import Blueprint, request, current_app as app, jsonify +import re +from quart import Blueprint, request, current_app as app, jsonify from logbook import Logger @@ -12,7 +13,7 @@ from litecord.snowflake import get_snowflake from litecord.schemas import validate, MESSAGE_CREATE from litecord.utils import pg_set_json -from litecord.embed.sanitizer import fill_embed +from litecord.embed.sanitizer import fill_embed, proxify, fetch_metadata log = Logger(__name__) @@ -226,6 +227,70 @@ async def _guild_text_mentions(payload: dict, guild_id: int, """, user_id, channel_id) +async def process_url_embed(config, storage, dispatcher, session, payload: dict): + message_id = int(payload['id']) + channel_id = int(payload['channel_id']) + + # if we already have embeds + # we shouldn't add our own. + embeds = payload['embeds'] + + if embeds: + log.debug('url processor: ignoring existing embeds @ mid {}', + message_id) + return + + # use regex to get URLs + urls = re.findall(r'(https?://\S+)', payload['content']) + urls = urls[:5] + + new_embeds = [] + + # fetch metadata for each url + for url in urls: + img_proxy_url = proxify(url, config=config) + meta = await fetch_metadata(url, config=config, session=session) + + if not meta['image']: + continue + + new_embeds.append({ + 'type': 'image', + 'url': url, + 'thumbnail': { + 'width': meta['width'], + 'height': meta['height'], + 'url': url, + 'proxy_url': img_proxy_url + } + }) + + # update if we got embeds + if not new_embeds: + return + + log.debug('made {} thumbnail embeds for mid {}', + len(new_embeds), message_id) + + await storage.execute_with_json(""" + UPDATE messages + SET embeds = $1 + WHERE messages.id = $2 + """, new_embeds, message_id) + + update_payload = { + 'id': str(message_id), + 'channel_id': str(channel_id), + 'embeds': new_embeds, + } + + if 'guild_id' in payload: + update_payload['guild_id'] = payload['guild_id'] + + await dispatcher.dispatch( + 'channel', channel_id, 'MESSAGE_UPDATE', update_payload) + + @bp.route('//messages', methods=['POST']) async def _create_message(channel_id): """Create a message.""" @@ -276,6 +341,10 @@ async def _create_message(channel_id): await app.dispatcher.dispatch('channel', channel_id, 'MESSAGE_CREATE', payload) + app.sched.spawn( + process_url_embed(app.config, app.storage, app.dispatcher, app.session, + payload)) + # update read state for the author await app.db.execute(""" UPDATE user_read_state diff --git a/litecord/embed/sanitizer.py b/litecord/embed/sanitizer.py index 681604f..b3536e5 100644 --- a/litecord/embed/sanitizer.py +++ b/litecord/embed/sanitizer.py @@ -8,6 +8,8 @@ from typing import Dict, Any from logbook import Logger from quart import current_app as app +from litecord.embed.schemas import EmbedURL + log = Logger(__name__) Embed = Dict[str, Any] @@ -55,12 +57,18 @@ def path_exists(embed: Embed, components: str): return False -def proxify(url) -> str: +def proxify(url, *, config=None) -> str: """Return a mediaproxy url for the given EmbedURL.""" - md_base_url = app.config['MEDIA_PROXY'] + if not config: + config = app.config + + if isinstance(url, str): + url = EmbedURL(url) + + md_base_url = config['MEDIA_PROXY'] parsed = url.parsed - proto = 'https' if app.config['IS_SSL'] else 'http' + proto = 'https' if config['IS_SSL'] else 'http' return ( # base mediaproxy url @@ -69,18 +77,28 @@ def proxify(url) -> str: ) -async def fetch_metadata(url) -> dict: +async def fetch_metadata(url, *, config=None, session=None) -> dict: """Fetch metadata for a url.""" + + if session is None: + session = app.session + + if config is None: + config = app.config + + if isinstance(url, str): + url = EmbedURL(url) + parsed = url.parsed md_path = f'{parsed.scheme}/{parsed.netloc}{parsed.path}' - md_base_url = app.config['MEDIA_PROXY'] - proto = 'https' if app.config['IS_SSL'] else 'http' + md_base_url = config['MEDIA_PROXY'] + proto = 'https' if config['IS_SSL'] else 'http' request_url = f'{proto}://{md_base_url}/meta/{md_path}' - async with app.session.get(request_url) as resp: + async with session.get(request_url) as resp: if resp.status != 200: return diff --git a/litecord/storage.py b/litecord/storage.py index c7d76f2..33d4578 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -64,6 +64,12 @@ class Storage: await pg_set_json(con) return await con.fetch(query, *args) + async def execute_with_json(self, query: str, *args): + """Execute a SQL statement with JSON/JSONB support.""" + async with self.db.acquire() as con: + await pg_set_json(con) + return await con.execute(query, *args) + async def get_user(self, user_id, secure=False) -> Dict[str, Any]: """Get a single user payload.""" user_id = int(user_id)