diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 87277d8..14cc31b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,6 @@ tests:
script:
- ls
- cp config.ci.py config.py
- - psql -h postgres -U postgres -f schema.sql
- pipenv run ./manage.py migrate
- pipenv run ./manage.py setup_tests
- tox
diff --git a/README.md b/README.md
index f95757b..a696c84 100644
--- a/README.md
+++ b/README.md
@@ -92,9 +92,6 @@ It's recommended to create a separate user for the `litecord` database.
```sh
# Create the PostgreSQL database.
$ createdb litecord
-
-# Apply the base schema to the database.
-$ psql -f schema.sql litecord
```
Copy the `config.example.py` file and edit it to configure your instance (
diff --git a/manage/cmd/migration/command.py b/manage/cmd/migration/command.py
index 845bec2..e769b6c 100644
--- a/manage/cmd/migration/command.py
+++ b/manage/cmd/migration/command.py
@@ -19,6 +19,8 @@ along with this program. If not, see .
import inspect
import os
+import datetime
+
from pathlib import Path
from dataclasses import dataclass
from collections import namedtuple
@@ -32,6 +34,12 @@ log = Logger(__name__)
Migration = namedtuple('Migration', 'id name path')
+# line of change, 4 april 2019, at 1am (gmt+0)
+BREAK = datetime.datetime(2019, 4, 4, 1)
+
+# if a database has those tables, it ran 0_base.sql.
+HAS_BASE = ['users', 'guilds', 'e']
+
@dataclass
class MigrationContext:
@@ -42,7 +50,8 @@ class MigrationContext:
@property
def latest(self):
"""Return the latest migration ID."""
- return 0 if len(self.scripts) == 0 else max(self.scripts.keys())
+ return 0 if not self.scripts else max(self.scripts.keys())
+
def make_migration_ctx() -> MigrationContext:
"""Create the MigrationContext instance."""
@@ -73,7 +82,6 @@ def make_migration_ctx() -> MigrationContext:
async def _ensure_changelog(app, ctx):
# make sure we have the migration table up
-
try:
await app.db.execute("""
CREATE TABLE migration_log (
@@ -87,52 +95,123 @@ async def _ensure_changelog(app, ctx):
PRIMARY KEY (change_num)
);
""")
-
- # if we were able to create the
- # migration_log table, insert that we are
- # on the latest version.
- await app.db.execute("""
- INSERT INTO migration_log (change_num, description)
- VALUES ($1, $2)
- """, ctx.latest, 'migration setup')
except asyncpg.DuplicateTableError:
log.debug('existing migration table')
+ # NOTE: this is a migration breakage,
+ # only applying to databases that had their first migration
+ # before 4 april 2019 (more on BREAK)
-async def apply_migration(app, migration: Migration):
- """Apply a single migration."""
- migration_sql = migration.path.read_text(encoding='utf-8')
+ # if migration_log is empty, just assume this is new
+ first = await app.db.fetchval("""
+ SELECT apply_ts FROM migration_log
+ ORDER BY apply_ts ASC
+ LIMIT 1
+ """) or BREAK
+ if first < BREAK:
+ log.info('deleting migration_log due to migration structure change')
+ await app.db.execute("DROP TABLE migration_log")
+ await _ensure_changelog(app, ctx)
+
+async def _insert_log(app, migration_id: int, description) -> bool:
try:
await app.db.execute("""
INSERT INTO migration_log (change_num, description)
VALUES ($1, $2)
- """, migration.id, f'migration: {migration.name}')
- except asyncpg.UniqueViolationError:
- log.warning('already applied {}', migration.id)
- return
+ """, migration_id, description)
- await app.db.execute(migration_sql)
- log.info('applied {}', migration.id)
+ return True
+ except asyncpg.UniqueViolationError:
+ log.warning('already inserted {}', migration_id)
+ return False
+
+
+async def _delete_log(app, migration_id: int):
+ await app.db.execute("""
+ DELETE FROM migration_log WHERE change_num = $1
+ """, migration_id)
+
+
+async def apply_migration(app, migration: Migration) -> bool:
+ """Apply a single migration.
+
+ Tries to insert it to the migration logs first, and if it exists,
+ skips it.
+
+ If any error happens while migrating, this will rollback the log,
+ by removing it from the logs.
+
+ Returns a boolean signaling if this failed or not.
+ """
+ migration_sql = migration.path.read_text(encoding='utf-8')
+
+ res = await _insert_log(
+ app, migration.id, f'migration: {migration.name}')
+
+ if not res:
+ return False
+
+ try:
+ await app.db.execute(migration_sql)
+ log.info('applied {} {}', migration.id, migration.name)
+
+ return True
+ except:
+ log.exception('failed to run migration, rollbacking log')
+ await _delete_log(app, migration.id)
+
+ return False
+
+
+async def _check_base(app) -> bool:
+ """Return if the current database has ran the 0_base.sql
+ file."""
+ try:
+ for table in HAS_BASE:
+ await app.db.execute(f"""
+ SELECT * FROM {table} LIMIT 0
+ """)
+ except asyncpg.UndefinedTableError:
+ return False
+
+ return True
async def migrate_cmd(app, _args):
"""Main migration command.
- This makes sure the database
- is updated.
+ This makes sure the database is updated, here's the steps:
+ - create the migration_log table, or recreate it (due to migration
+ changes in 4 april 2019)
+ - check the latest local point in migration_log
+ - check if the database is on the base schema
"""
-
ctx = make_migration_ctx()
+ # ensure there is a migration_log table
await _ensure_changelog(app, ctx)
- # local point in the changelog
+ # check HAS_BASE tables, and if they exist, implicitly
+ # assume this has the base schema.
+ has_base = await _check_base(app)
+
+ # fetch latest local migration that has been run on this database
local_change = await app.db.fetchval("""
SELECT max(change_num)
FROM migration_log
""")
+ # if base exists, add it to logs, if not, apply (and add to logs)
+ if has_base:
+ await _insert_log(app, 0, 'migration setup (from existing)')
+ else:
+ await apply_migration(app, ctx.scripts[0])
+
+ # after that check the current local_change
+ # and the latest migration to be run
+
+ # if no migrations, then we are on migration 0 (which is base)
local_change = local_change or 0
latest_change = ctx.latest
@@ -149,7 +228,7 @@ async def migrate_cmd(app, _args):
migration = ctx.scripts.get(idx)
print('applying', migration.id, migration.name)
- await apply_migration(app, migration)
+ # await apply_migration(app, migration)
def setup(subparser):
diff --git a/schema.sql b/manage/cmd/migration/scripts/0_base.sql
similarity index 95%
rename from schema.sql
rename to manage/cmd/migration/scripts/0_base.sql
index 4c0941f..cd057bd 100644
--- a/schema.sql
+++ b/manage/cmd/migration/scripts/0_base.sql
@@ -27,18 +27,8 @@ CREATE TABLE IF NOT EXISTS user_conn_apps (
name text NOT NULL
);
-INSERT INTO user_conn_apps (id, name) VALUES (0, 'Twitch');
-INSERT INTO user_conn_apps (id, name) VALUES (1, 'Youtube');
-INSERT INTO user_conn_apps (id, name) VALUES (2, 'Steam');
-INSERT INTO user_conn_apps (id, name) VALUES (3, 'Reddit');
-INSERT INTO user_conn_apps (id, name) VALUES (4, 'Facebook');
-INSERT INTO user_conn_apps (id, name) VALUES (5, 'Twitter');
-INSERT INTO user_conn_apps (id, name) VALUES (6, 'Spotify');
-INSERT INTO user_conn_apps (id, name) VALUES (7, 'XBOX');
-INSERT INTO user_conn_apps (id, name) VALUES (8, 'Battle.net');
-INSERT INTO user_conn_apps (id, name) VALUES (9, 'Skype');
-INSERT INTO user_conn_apps (id, name) VALUES (10, 'League of Legends');
-
+-- there was a chain of INSERTs here with hardcoded names and stuff.
+-- removed it because we aren't in the best business of hardcoding.
CREATE TABLE IF NOT EXISTS instance_invites (
code text PRIMARY KEY,
@@ -52,24 +42,6 @@ CREATE TABLE IF NOT EXISTS instance_invites (
);
--- main attachments table
-CREATE TABLE IF NOT EXISTS attachments (
- id bigint PRIMARY KEY,
-
- -- keeping channel_id and message_id
- -- make a way "better" attachment url.
- channel_id bigint REFERENCES channels (id),
- message_id bigint REFERENCES messages (id),
-
- filename text NOT NULL,
- filesize integer,
-
- image boolean DEFAULT FALSE,
-
- -- only not null if image=true
- height integer DEFAULT NULL,
- width integer DEFAULT NULL
-);
CREATE TABLE IF NOT EXISTS icons (
@@ -607,7 +579,10 @@ CREATE TABLE IF NOT EXISTS channel_overwrites (
-- columns in private keys can't have NULL values,
-- so instead we use a custom constraint with UNIQUE
-ALTER TABLE channel_overwrites ADD CONSTRAINT channel_overwrites_uniq
+ALTER TABLE channel_overwrites
+ DROP CONSTRAINT IF EXISTS channel_overwrites_uniq;
+ALTER TABLE channel_overwrites
+ ADD CONSTRAINT channel_overwrites_uniq
UNIQUE (channel_id, target_role, target_user);
@@ -688,7 +663,11 @@ CREATE TABLE IF NOT EXISTS message_reactions (
emoji_text text
);
-ALTER TABLE message_reactions ADD CONSTRAINT message_reactions_main_uniq
+-- unique constraint over multiple columns instead of a primary key
+ALTER TABLE message_reactions
+ DROP CONSTRAINT IF EXISTS message_reactions_main_uniq;
+ALTER TABLE message_reactions
+ ADD CONSTRAINT message_reactions_main_uniq
UNIQUE (message_id, user_id, emoji_id, emoji_text);
CREATE TABLE IF NOT EXISTS channel_pins (
@@ -696,3 +675,23 @@ CREATE TABLE IF NOT EXISTS channel_pins (
message_id bigint REFERENCES messages (id) ON DELETE CASCADE,
PRIMARY KEY (channel_id, message_id)
);
+
+
+-- main attachments table
+CREATE TABLE IF NOT EXISTS attachments (
+ id bigint PRIMARY KEY,
+
+ -- keeping channel_id and message_id
+ -- make a way "better" attachment url.
+ channel_id bigint REFERENCES channels (id),
+ message_id bigint REFERENCES messages (id),
+
+ filename text NOT NULL,
+ filesize integer,
+
+ image boolean DEFAULT FALSE,
+
+ -- only not null if image=true
+ height integer DEFAULT NULL,
+ width integer DEFAULT NULL
+);
diff --git a/manage/cmd/migration/scripts/10_add_attachments_table.sql b/manage/cmd/migration/scripts/10_add_attachments_table.sql
deleted file mode 100644
index b1f979d..0000000
--- a/manage/cmd/migration/scripts/10_add_attachments_table.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-CREATE TABLE IF NOT EXISTS attachments (
- id bigint PRIMARY KEY,
-
- channel_id bigint REFERENCES channels (id),
- message_id bigint REFERENCES messages (id),
-
- filename text NOT NULL,
- filesize integer,
-
- image boolean DEFAULT FALSE,
-
- -- only not null if image=true
- height integer DEFAULT NULL,
- width integer DEFAULT NULL
-);
diff --git a/manage/cmd/migration/scripts/11_voice_regions_servers.sql b/manage/cmd/migration/scripts/11_voice_regions_servers.sql
deleted file mode 100644
index 398c91a..0000000
--- a/manage/cmd/migration/scripts/11_voice_regions_servers.sql
+++ /dev/null
@@ -1,37 +0,0 @@
--- voice region data
--- NOTE: do NOT remove any rows. use deprectated=true and
--- DELETE FROM voice_servers instead.
-CREATE TABLE IF NOT EXISTS voice_regions (
- -- always lowercase
- id text PRIMARY KEY,
-
- -- "Russia", "Brazil", "Antartica", etc
- name text NOT NULL,
-
- -- we don't have the concept of vip guilds yet, but better
- -- future proof.
- vip boolean DEFAULT FALSE,
-
- deprecated boolean DEFAULT FALSE,
-
- -- we don't have the concept of custom regions too. we don't have the
- -- concept of official guilds either, but i'm keeping this in
- custom boolean DEFAULT FALSE
-);
-
--- voice server pool. when someone wants to connect to voice, we choose
--- a server that is in the same region the guild is too, and choose the one
--- with the best health value
-CREATE TABLE IF NOT EXISTS voice_servers (
- -- hostname is a reachable url, e.g "brazil2.example.com"
- hostname text PRIMARY KEY,
- region_id text REFERENCES voice_regions (id),
-
- -- health values are more thoroughly defined in the LVSP documentation
- last_health float default 0.5
-);
-
-
-ALTER TABLE guilds DROP COLUMN IF EXISTS region;
-ALTER TABLE guilds ADD COLUMN
- region text REFERENCES voice_regions (id);
diff --git a/manage/cmd/migration/scripts/12_remove_features_table.sql b/manage/cmd/migration/scripts/12_remove_features_table.sql
deleted file mode 100644
index 7c8df16..0000000
--- a/manage/cmd/migration/scripts/12_remove_features_table.sql
+++ /dev/null
@@ -1,5 +0,0 @@
-DROP TABLE guild_features;
-DROP TABLE features;
-
--- this should do the trick
-ALTER TABLE guilds ADD COLUMN features text[] NOT NULL DEFAULT '{}';
diff --git a/manage/cmd/migration/scripts/13_add_vanity_invites_table.sql b/manage/cmd/migration/scripts/13_add_vanity_invites_table.sql
deleted file mode 100644
index 73aa914..0000000
--- a/manage/cmd/migration/scripts/13_add_vanity_invites_table.sql
+++ /dev/null
@@ -1,5 +0,0 @@
--- vanity url table, the mapping is 1-1 for guilds and vanity urls
-CREATE TABLE IF NOT EXISTS vanity_invites (
- guild_id bigint REFERENCES guilds (id) PRIMARY KEY,
- code text REFERENCES invites (code) ON DELETE CASCADE
-);
diff --git a/manage/cmd/migration/scripts/14_add_guild_description.sql b/manage/cmd/migration/scripts/14_add_guild_description.sql
deleted file mode 100644
index c9dbecb..0000000
--- a/manage/cmd/migration/scripts/14_add_guild_description.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-ALTER TABLE guilds ADD COLUMN description text DEFAULT NULL;
-ALTER TABLE guilds ADD COLUMN banner text DEFAULT NULL;
diff --git a/manage/cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql b/manage/cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql
deleted file mode 100644
index a6e110f..0000000
--- a/manage/cmd/migration/scripts/15_drop_nullable_webhook_avatar.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-ALTER TABLE webhooks ALTER COLUMN avatar DROP NOT NULL;
-ALTER TABLE webhooks ALTER COLUMN avatar SET DEFAULT NULL;
diff --git a/manage/cmd/migration/scripts/16_messages_webhooks.sql b/manage/cmd/migration/scripts/16_messages_webhooks.sql
deleted file mode 100644
index fc8ba66..0000000
--- a/manage/cmd/migration/scripts/16_messages_webhooks.sql
+++ /dev/null
@@ -1,17 +0,0 @@
--- this is a tricky one. blame discord
-
--- first, remove all messages made by webhooks (safety check)
-DELETE FROM messages WHERE author_id is null;
-
--- delete the column, removing the fkey. no connection anymore.
-ALTER TABLE messages DROP COLUMN webhook_id;
-
--- add a message_webhook_info table. more on that in Storage._inject_author
-CREATE TABLE IF NOT EXISTS message_webhook_info (
- message_id bigint REFERENCES messages (id) PRIMARY KEY,
-
- webhook_id bigint,
- name text DEFAULT '',
- avatar text DEFAULT NULL
-);
-
diff --git a/manage/cmd/migration/scripts/1_message_embed_type.sql b/manage/cmd/migration/scripts/1_message_embed_type.sql
deleted file mode 100644
index 8650558..0000000
--- a/manage/cmd/migration/scripts/1_message_embed_type.sql
+++ /dev/null
@@ -1,6 +0,0 @@
--- unused tables
-DROP TABLE message_embeds;
-DROP TABLE embeds;
-
-ALTER TABLE messages
- ADD COLUMN embeds jsonb DEFAULT '[]'
diff --git a/manage/cmd/migration/scripts/2_icons_table.sql b/manage/cmd/migration/scripts/2_icons_table.sql
deleted file mode 100644
index ffe6fd9..0000000
--- a/manage/cmd/migration/scripts/2_icons_table.sql
+++ /dev/null
@@ -1,35 +0,0 @@
-
--- new icons table
-CREATE TABLE IF NOT EXISTS icons (
- scope text NOT NULL,
- key text,
- hash text UNIQUE NOT NULL,
- mime text NOT NULL,
- PRIMARY KEY (scope, hash, mime)
-);
-
--- dummy attachments table for now.
-CREATE TABLE IF NOT EXISTS attachments (
- id bigint NOT NULL,
- PRIMARY KEY (id)
-);
-
--- remove the old columns referencing the files table
-ALTER TABLE users DROP COLUMN avatar;
-ALTER TABLE users ADD COLUMN avatar text REFERENCES icons (hash) DEFAULT NULL;
-
-ALTER TABLE group_dm_channels DROP COLUMN icon;
-ALTER TABLE group_dm_channels ADD COLUMN icon text REFERENCES icons (hash);
-
-ALTER TABLE guild_emoji DROP COLUMN image;
-ALTER TABLE guild_emoji ADD COLUMN image text REFERENCES icons (hash);
-
-ALTER TABLE guilds DROP COLUMN icon;
-ALTER TABLE guilds ADD COLUMN icon text REFERENCES icons (hash) DEFAULT NULL;
-
--- this one is a change from files to the attachments table
-ALTER TABLE message_attachments DROP COLUMN attachment;
-ALTER TABLE guild_emoji ADD COLUMN attachment bigint REFERENCES attachments (id);
-
--- remove files table
-DROP TABLE files;
diff --git a/manage/cmd/migration/scripts/3_drop_constraints_icons_hash.sql b/manage/cmd/migration/scripts/3_drop_constraints_icons_hash.sql
deleted file mode 100644
index 59ef69e..0000000
--- a/manage/cmd/migration/scripts/3_drop_constraints_icons_hash.sql
+++ /dev/null
@@ -1,9 +0,0 @@
--- drop main primary key
--- since hash can now be nullable
-ALTER TABLE icons DROP CONSTRAINT "icons_pkey";
-
--- remove not null from hash column
-ALTER TABLE icons ALTER COLUMN hash DROP NOT NULL;
-
--- add new primary key, without hash
-ALTER TABLE icons ADD CONSTRAINT icons_pkey PRIMARY KEY (scope, key);
diff --git a/manage/cmd/migration/scripts/4_add_instance_invites.sql b/manage/cmd/migration/scripts/4_add_instance_invites.sql
deleted file mode 100644
index ed215f1..0000000
--- a/manage/cmd/migration/scripts/4_add_instance_invites.sql
+++ /dev/null
@@ -1,8 +0,0 @@
-CREATE TABLE IF NOT EXISTS instance_invites (
- code text PRIMARY KEY,
-
- created_at timestamp without time zone default (now() at time zone 'utc'),
-
- uses bigint DEFAULT 0,
- max_uses bigint DEFAULT -1
-);
diff --git a/manage/cmd/migration/scripts/5_add_messages_guild_id.sql b/manage/cmd/migration/scripts/5_add_messages_guild_id.sql
deleted file mode 100644
index 07b5c5e..0000000
--- a/manage/cmd/migration/scripts/5_add_messages_guild_id.sql
+++ /dev/null
@@ -1 +0,0 @@
-ALTER TABLE messages ADD COLUMN guild_id bigint REFERENCES guilds (id) ON DELETE CASCADE;
diff --git a/manage/cmd/migration/scripts/6_emoji_require_colon_true.sql b/manage/cmd/migration/scripts/6_emoji_require_colon_true.sql
deleted file mode 100644
index 3eb7401..0000000
--- a/manage/cmd/migration/scripts/6_emoji_require_colon_true.sql
+++ /dev/null
@@ -1,5 +0,0 @@
--- require_colons seems to be true for all custom emoji.
-ALTER TABLE guild_emoji ALTER COLUMN require_colons SET DEFAULT true;
-
--- retroactively update all other emojis
-UPDATE guild_emoji SET require_colons=true;
diff --git a/manage/cmd/migration/scripts/7_text_channels_rate_limit_per_user.sql b/manage/cmd/migration/scripts/7_text_channels_rate_limit_per_user.sql
deleted file mode 100644
index 4976086..0000000
--- a/manage/cmd/migration/scripts/7_text_channels_rate_limit_per_user.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-ALTER TABLE guild_text_channels
-ADD COLUMN rate_limit_per_user bigint DEFAULT 0;
diff --git a/manage/cmd/migration/scripts/8_roles_default_color.sql b/manage/cmd/migration/scripts/8_roles_default_color.sql
deleted file mode 100644
index b15fb18..0000000
--- a/manage/cmd/migration/scripts/8_roles_default_color.sql
+++ /dev/null
@@ -1,9 +0,0 @@
--- update roles.color default to 0
-ALTER TABLE roles
- ALTER COLUMN color SET DEFAULT 0;
-
--- update all existing guild default roles to
--- color=0
-UPDATE roles
- SET color = 0
-WHERE roles.id = roles.guild_id;
diff --git a/manage/cmd/migration/scripts/9_nullable_emails.sql b/manage/cmd/migration/scripts/9_nullable_emails.sql
deleted file mode 100644
index 85eb7c2..0000000
--- a/manage/cmd/migration/scripts/9_nullable_emails.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-ALTER TABLE users ALTER COLUMN email DROP NOT NULL;
-ALTER TABLE users ALTER COLUMN email SET DEFAULT NULL;
\ No newline at end of file
diff --git a/manage/main.py b/manage/main.py
index 5402c67..805b28c 100644
--- a/manage/main.py
+++ b/manage/main.py
@@ -62,17 +62,22 @@ def main(config):
cfg = getattr(config, config.MODE)
app = FakeApp(cfg.__dict__)
- loop.run_until_complete(init_app_db(app))
- init_app_managers(app)
-
# initialize argparser
parser = init_parser()
+ loop.run_until_complete(init_app_db(app))
+
try:
if len(argv) < 2:
parser.print_help()
return
+ # only init app managers when we aren't migrating
+ # as the managers require it
+ # and the migrate command also sets the db up
+ if argv[1] != 'migrate':
+ init_app_managers(app)
+
args = parser.parse_args()
loop.run_until_complete(args.func(app, args))
except Exception: