Merge branch 'webhook-avatars-rewrite' into 'master'

Webhook avatars rewrite

See merge request litecord/litecord!34

litecord/blueprints/icons.py

@@ -18,7 +18,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 from os.path import splitext
-from quart import Blueprint, current_app as app, send_file
+
+from quart import Blueprint, current_app as app, send_file, redirect
+
+from litecord.embed.sanitizer import make_md_req_url
+from litecord.embed.schemas import EmbedURL
 
 bp = Blueprint('images', __name__)
 
@@ -54,14 +58,32 @@ async def _get_guild_icon(guild_id: int, icon_file: str):
     return await send_icon('guild', guild_id, icon_hash, ext=ext)
 
 
-@bp.route('/embed/avatars/<int:discrim>.png')
+@bp.route('/embed/avatars/<int:default_id>.png')
-async def _get_default_user_avatar(discrim: int):
+async def _get_default_user_avatar(default_id: int):
+    # TODO: how do we determine which assets to use for this?
+    # I don't think we can use discord assets.
     pass
 
 
+async def _handle_webhook_avatar(md_url_redir: str):
+    md_url = make_md_req_url(app.config, 'img', EmbedURL(md_url_redir))
+    return redirect(md_url)
+
+
 @bp.route('/avatars/<int:user_id>/<avatar_file>')
 async def _get_user_avatar(user_id, avatar_file):
     avatar_hash, ext = splitext_(avatar_file)
+
+    # first, check if this is a webhook avatar to redir to
+    md_url_redir = await app.db.fetchval("""
+    SELECT md_url_redir
+    FROM webhook_avatars
+    WHERE webhook_id = $1 AND hash = $2
+    """, user_id, avatar_hash)
+
+    if md_url_redir:
+        return await _handle_webhook_avatar(md_url_redir)
+
     return await send_icon('user', user_id, avatar_hash, ext=ext)
 
 

litecord/blueprints/webhooks.py

@@ -18,9 +18,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 import secrets
-import base64
+import hashlib
 from typing import Dict, Any, Optional
 
+import asyncpg
 from quart import Blueprint, jsonify, current_app as app, request
 
 from litecord.auth import token_check
@@ -44,8 +45,10 @@ from litecord.blueprints.channel.messages import (
 )
 from litecord.embed.sanitizer import fill_embed, fetch_raw_img
 from litecord.embed.messages import process_url_embed, is_media_url
+from litecord.embed.schemas import EmbedURL
 from litecord.utils import pg_set_json
 from litecord.enums import MessageType
+from litecord.images import STATIC_IMAGE_MIMES
 
 bp = Blueprint('webhooks', __name__)
 
@@ -346,12 +349,27 @@ async def create_message_webhook(guild_id, channel_id, webhook_id, data):
     return message_id
 
 
-async def _create_avatar(webhook_id: int, avatar_url):
+async def _webhook_avy_redir(webhook_id: int, avatar_url: EmbedURL):
+    """Create a row on webhook_avatars."""
+    url_hash = hashlib.sha256(avatar_url.to_md_path.encode()).hexdigest()
+
+    try:
+        await app.db.execute("""
+        INSERT INTO webhook_avatars (webhook_id, hash, md_url_redir)
+        VALUES ($1, $2, $3)
+        """, webhook_id, url_hash, avatar_url.url)
+    except asyncpg.UniqueViolationError:
+        pass
+
+    return url_hash
+
+
+async def _create_avatar(webhook_id: int, avatar_url: EmbedURL) -> str:
     """Create an avatar for a webhook out of an avatar URL,
     given when executing the webhook.
 
-    Litecord will query that URL via mediaproxy and store the data
+    Litecord will write an URL that redirects to the given avatar_url,
-    via IconManager.
+    using mediaproxy.
     """
     if avatar_url.scheme not in ('http', 'https'):
         raise BadRequest('invalid avatar url scheme')
@@ -359,18 +377,27 @@ async def _create_avatar(webhook_id: int, avatar_url):
     if not is_media_url(avatar_url):
         raise BadRequest('url is not media url')
 
+    # we still fetch the URL to check its validity, mimetypes, etc
+    # but in the end, we will store it under the webhook_avatars table,
+    # not IconManager.
     resp, raw = await fetch_raw_img(avatar_url)
-    raw_b64 = base64.b64encode(raw).decode()
+    #raw_b64 = base64.b64encode(raw).decode()
 
     mime = resp.headers['content-type']
-    b64_data = f'data:{mime};base64,{raw_b64}'
 
-    icon = await app.icons.put(
+    # TODO: apng checks are missing (for this and everywhere else)
-        'user', webhook_id, b64_data,
+    if mime not in STATIC_IMAGE_MIMES:
-        always_icon=True, size=(128, 128)
+        raise BadRequest('invalid mime type for given url')
-    )
 
-    return icon.icon_hash
+    #b64_data = f'data:{mime};base64,{raw_b64}'
+
+    # TODO: replace this by webhook_avatars
+    #icon = await app.icons.put(
+    #    'user', webhook_id, b64_data,
+    #    always_icon=True, size=(128, 128)
+    #)
+
+    return await _webhook_avy_redir(webhook_id, avatar_url)
 
 
 @bp.route('/webhooks/<int:webhook_id>/<webhook_token>', methods=['POST'])
@@ -399,6 +426,10 @@ async def execute_webhook(webhook_id: int, webhook_token):
     given_embeds = j.get('embeds', [])
 
     webhook = await get_webhook(webhook_id)
+
+    # webhooks have TWO avatars. one is from settings, the other is from
+    # the json's icon_url. one can be handled gracefully by IconManager,
+    # but the other can't, at all.
     avatar = webhook['avatar']
 
     if 'avatar_url' in j and j['avatar_url'] is not None:

litecord/embed/sanitizer.py

@@ -97,7 +97,7 @@ def _md_base(config) -> tuple:
     return proto, md_base_url
 
 
-def _make_md_req_url(config, scope: str, url):
+def make_md_req_url(config, scope: str, url):
     """Make a mediaproxy request URL given the config, scope, and the url
     to be proxied."""
     proto, base_url = _md_base(config)
@@ -112,7 +112,7 @@ def proxify(url, *, config=None) -> str:
     if isinstance(url, str):
         url = EmbedURL(url)
 
-    return _make_md_req_url(config, 'img', url)
+    return make_md_req_url(config, 'img', url)
 
 
 async def _md_client_req(config, session, scope: str,
@@ -152,7 +152,7 @@ async def _md_client_req(config, session, scope: str,
     if not isinstance(url, EmbedURL):
         url = EmbedURL(url)
 
-    request_url = _make_md_req_url(config, scope, url)
+    request_url = make_md_req_url(config, scope, url)
 
     async with session.get(request_url) as resp:
         if resp.status == 200:

litecord/embed/schemas.py

@@ -35,6 +35,11 @@ class EmbedURL:
         self.raw_url = url
         self.parsed = parsed
 
+    @classmethod
+    def from_parsed(cls, parsed):
+        """Make an EmbedURL instance out of an already parsed 6-tuple."""
+        return cls(parsed.geturl())
+
     @property
     def url(self) -> str:
         """Return the unparsed URL."""

litecord/images.py

@@ -50,6 +50,11 @@ MIMES = {
     'webp': 'image/webp',
 }
 
+STATIC_IMAGE_MIMES = [
+    'image/png',
+    'image/jpeg',
+    'image/webp'
+]
 
 def get_ext(mime: str) -> str:
     if mime in EXTENSIONS:

manage/cmd/migration/command.py

@@ -228,7 +228,7 @@ async def migrate_cmd(app, _args):
         migration = ctx.scripts.get(idx)
 
         print('applying', migration.id, migration.name)
-        # await apply_migration(app, migration)
+        await apply_migration(app, migration)
 
 
 def setup(subparser):

manage/cmd/migration/scripts/1_webhook_avatars.sql

@@ -0,0 +1,13 @@
+-- webhook_avatars table. check issue 46.
+CREATE TABLE IF NOT EXISTS webhook_avatars (
+    webhook_id bigint,
+
+    -- this is e.g a sha256 hash of EmbedURL.to_md_url
+    hash text,
+
+    -- we don't hardcode the mediaproxy url here for obvious reasons.
+    -- the output of EmbedURL.to_md_url goes here.
+    md_url_redir text,
+
+    PRIMARY KEY (webhook_id, hash)
+);