channel.messages: add url autoembeds

related to #27

 - embed.sanitizer: handle external config and session objects
 - storage: add Storage.execute_with_json

litecord/blueprints/channel/messages.py

@@ -1,5 +1,6 @@
-from quart import Blueprint, request, current_app as app, jsonify
+import re
 
+from quart import Blueprint, request, current_app as app, jsonify
 from logbook import Logger
 
 
@@ -12,7 +13,7 @@ from litecord.snowflake import get_snowflake
 from litecord.schemas import validate, MESSAGE_CREATE
 from litecord.utils import pg_set_json
 
-from litecord.embed.sanitizer import fill_embed
+from litecord.embed.sanitizer import fill_embed, proxify, fetch_metadata
 
 
 log = Logger(__name__)
@@ -226,6 +227,70 @@ async def _guild_text_mentions(payload: dict, guild_id: int,
         """, user_id, channel_id)
 
 
+async def process_url_embed(config, storage, dispatcher, session, payload: dict):
+    message_id = int(payload['id'])
+    channel_id = int(payload['channel_id'])
+
+    # if we already have embeds
+    # we shouldn't add our own.
+    embeds = payload['embeds']
+
+    if embeds:
+        log.debug('url processor: ignoring existing embeds @ mid {}',
+                  message_id)
+        return
+
+    # use regex to get URLs
+    urls = re.findall(r'(https?://\S+)', payload['content'])
+    urls = urls[:5]
+
+    new_embeds = []
+
+    # fetch metadata for each url
+    for url in urls:
+        img_proxy_url = proxify(url, config=config)
+        meta = await fetch_metadata(url, config=config, session=session)
+
+        if not meta['image']:
+            continue
+
+        new_embeds.append({
+            'type': 'image',
+            'url': url,
+            'thumbnail': {
+                'width': meta['width'],
+                'height': meta['height'],
+                'url': url,
+                'proxy_url': img_proxy_url
+            }
+        })
+
+    # update if we got embeds
+    if not new_embeds:
+        return
+
+    log.debug('made {} thumbnail embeds for mid {}',
+              len(new_embeds), message_id)
+
+    await storage.execute_with_json("""
+    UPDATE messages
+    SET embeds = $1
+    WHERE messages.id = $2
+    """, new_embeds, message_id)
+
+    update_payload = {
+        'id': str(message_id),
+        'channel_id': str(channel_id),
+        'embeds': new_embeds,
+    }
+
+    if 'guild_id' in payload:
+        update_payload['guild_id'] = payload['guild_id']
+
+    await dispatcher.dispatch(
+        'channel', channel_id, 'MESSAGE_UPDATE', update_payload)
+
+
 @bp.route('/<int:channel_id>/messages', methods=['POST'])
 async def _create_message(channel_id):
     """Create a message."""
@@ -276,6 +341,10 @@ async def _create_message(channel_id):
     await app.dispatcher.dispatch('channel', channel_id,
                                   'MESSAGE_CREATE', payload)
 
+    app.sched.spawn(
+        process_url_embed(app.config, app.storage, app.dispatcher, app.session,
+                          payload))
+
     # update read state for the author
     await app.db.execute("""
     UPDATE user_read_state

litecord/embed/sanitizer.py

@@ -8,6 +8,8 @@ from typing import Dict, Any
 from logbook import Logger
 from quart import current_app as app
 
+from litecord.embed.schemas import EmbedURL
+
 log = Logger(__name__)
 Embed = Dict[str, Any]
 
@@ -55,12 +57,18 @@ def path_exists(embed: Embed, components: str):
     return False
 
 
-def proxify(url) -> str:
+def proxify(url, *, config=None) -> str:
     """Return a mediaproxy url for the given EmbedURL."""
 
-    md_base_url = app.config['MEDIA_PROXY']
+    if not config:
+        config = app.config
+
+    if isinstance(url, str):
+        url = EmbedURL(url)
+
+    md_base_url = config['MEDIA_PROXY']
     parsed = url.parsed
-    proto = 'https' if app.config['IS_SSL'] else 'http'
+    proto = 'https' if config['IS_SSL'] else 'http'
 
     return (
         # base mediaproxy url
@@ -69,18 +77,28 @@ def proxify(url) -> str:
     )
 
 
-async def fetch_metadata(url) -> dict:
+async def fetch_metadata(url, *, config=None, session=None) -> dict:
     """Fetch metadata for a url."""
+
+    if session is None:
+        session = app.session
+
+    if config is None:
+        config = app.config
+
+    if isinstance(url, str):
+        url = EmbedURL(url)
+
     parsed = url.parsed
 
     md_path = f'{parsed.scheme}/{parsed.netloc}{parsed.path}'
 
-    md_base_url = app.config['MEDIA_PROXY']
-    proto = 'https' if app.config['IS_SSL'] else 'http'
+    md_base_url = config['MEDIA_PROXY']
+    proto = 'https' if config['IS_SSL'] else 'http'
 
     request_url = f'{proto}://{md_base_url}/meta/{md_path}'
 
-    async with app.session.get(request_url) as resp:
+    async with session.get(request_url) as resp:
         if resp.status != 200:
             return
 

litecord/storage.py

@@ -64,6 +64,12 @@ class Storage:
             await pg_set_json(con)
             return await con.fetch(query, *args)
 
+    async def execute_with_json(self, query: str, *args):
+        """Execute a SQL statement with JSON/JSONB support."""
+        async with self.db.acquire() as con:
+            await pg_set_json(con)
+            return await con.execute(query, *args)
+
     async def get_user(self, user_id, secure=False) -> Dict[str, Any]:
         """Get a single user payload."""
         user_id = int(user_id)