diff --git a/Pipfile b/Pipfile index 9c1a0f5..53bf257 100644 --- a/Pipfile +++ b/Pipfile @@ -19,6 +19,7 @@ aiohttp = "==3.4.4" pytest = "==3.10.1" pytest-asyncio = "==0.9.0" pyflakes = "*" +mypy = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 29c671a..8697992 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b546ad1edfe79457cb4da95e19fd17506b7adabe6a43acbc0906fb12cfda68b2" + "sha256": "4332c948a4bf656d0b95f4753eed6a3793dcb5ee9cd3538a443d7094a584e6cd" }, "pipfile-spec": 6, "requires": { @@ -82,10 +82,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "bcrypt": { "hashes": [ @@ -142,40 +142,36 @@ }, "cffi": { "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + "sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f", + "sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11", + "sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d", + "sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891", + "sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf", + "sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c", + "sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed", + "sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b", + "sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a", + "sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585", + "sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea", + "sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f", + "sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33", + "sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145", + "sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a", + "sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3", + "sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f", + "sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd", + "sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804", + "sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d", + "sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92", + "sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f", + "sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84", + "sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb", + "sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7", + "sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7", + "sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35", + "sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889" ], - "version": "==1.11.5" + "version": "==1.12.2" }, "chardet": { "hashes": [ @@ -213,10 +209,10 @@ }, "h2": { "hashes": [ - "sha256:4be613e35caad5680dc48f98f3bf4e7338c7c429e6375a5137be7fbe45219981", - "sha256:b2962f883fa392a23cbfcc4ad03c335bcc661be0cf9627657b589f0df2206e64" + "sha256:c8f387e0e4878904d4978cd688a3195f6b169d49b1ffa572a3d347d7adc5e09f", + "sha256:fd07e865a3272ac6ef195d8904de92dc7b38dc28297ec39cfa22716b6d62e6eb" ], - "version": "==3.0.1" + "version": "==3.1.0" }, "hpack": { "hashes": [ @@ -227,17 +223,17 @@ }, "hypercorn": { "hashes": [ - "sha256:3931144309c40341a46a2d054ac550bbd012a1f1a803774b5d6a3add90f52259", - "sha256:4df03fbc101efb4faf0b0883863ff7e620f94310e309311ceafaadb38ee1fa36" + "sha256:97bad5887ff543e2dff0a584d1a084e702789a26df5c8fb027ac1efab32274c5", + "sha256:b90799a1bc84f00ee999071e259f194087881b85c7240994ba9d86c4ceff3305" ], - "version": "==0.4.2" + "version": "==0.5.3" }, "hyperframe": { "hashes": [ - "sha256:87567c9eb1540de1e7f48805adf00e87856409342fdebd0cd20cf5d381c38b69", - "sha256:a25944539db36d6a2e47689e7915dcee562b3f8d10c6cdfa0d53c91ed692fb04" + "sha256:5187962cb16dcc078f23cb5a4b110098d546c3f41ff2d4038a9896893bbd0b40", + "sha256:a9f5c17f2cc3c719b917c4f33ed1c61bd1f8dfac4b1bd23b7c80b3400971b41f" ], - "version": "==5.1.0" + "version": "==5.2.0" }, "idna": { "hashes": [ @@ -273,36 +269,36 @@ }, "markupsafe": { "hashes": [ - "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", - "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", - "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", - "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", - "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", - "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", - "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", - "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", - "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", - "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", - "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", - "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", - "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", - "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", - "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", - "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", - "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", - "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", - "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", - "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", - "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", - "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", - "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", - "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", - "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", - "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", - "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", - "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" ], - "version": "==1.1.0" + "version": "==1.1.1" }, "multidict": { "hashes": [ @@ -340,39 +336,39 @@ }, "pillow": { "hashes": [ - "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360", - "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7", - "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e", - "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93", - "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c", - "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8", - "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b", - "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0", - "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd", - "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb", - "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581", - "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64", - "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace", - "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956", - "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60", - "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16", - "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37", - "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae", - "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a", - "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073", - "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8", - "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb", - "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409", - "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f", - "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2", - "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4", - "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74", - "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e", - "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1", - "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888" + "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e", + "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7", + "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a", + "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3", + "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1", + "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1", + "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7", + "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1", + "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3", + "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055", + "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf", + "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f", + "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f", + "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239", + "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe", + "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c", + "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697", + "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494", + "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356", + "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6", + "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000", + "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f", + "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c", + "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca", + "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8", + "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3", + "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad", + "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9", + "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc", + "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e" ], "index": "pypi", - "version": "==5.3.0" + "version": "==5.4.1" }, "pycparser": { "hashes": [ @@ -393,10 +389,10 @@ }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" }, "sortedcontainers": { "hashes": [ @@ -407,11 +403,11 @@ }, "typing-extensions": { "hashes": [ - "sha256:2a6c6e78e291a4b6cbd0bbfd30edc0baaa366de962129506ec8fe06bdec66457", - "sha256:51e7b7f3dcabf9ad22eed61490f3b8d23d9922af400fe6656cb08e66656b701f", - "sha256:55401f6ed58ade5638eb566615c150ba13624e2f0c1eedd080fc3c1b6cb76f1d" + "sha256:07b2c978670896022a43c4b915df8958bec4a6b84add7f2c87b2b728bda3ba64", + "sha256:f3f0e67e1d42de47b5c67c32c9b26641642e9170fe7e292991793705cd5fef7c", + "sha256:fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71" ], - "version": "==3.6.6" + "version": "==3.7.2" }, "websockets": { "hashes": [ @@ -442,70 +438,86 @@ }, "wsproto": { "hashes": [ - "sha256:1fcb726d448f1b9bcbea884e26621af5ddd01d2d502941a024f4c727828b6009", - "sha256:6a51cf18d9de612892b9c1d38a8c1bdadec0cfe15de61cd5c0f09174bf0c7e82" + "sha256:c013342d7a9180486713c6c986872e4fe24e18a21ccbece314939d8b58312e0e", + "sha256:fd6020d825022247053400306448e161d8740bdd52e328e5553cd9eee089f705" ], - "version": "==0.12.0" + "version": "==0.13.0" }, "yarl": { "hashes": [ - "sha256:2556b779125621b311844a072e0ed367e8409a18fa12cbd68eb1258d187820f9", - "sha256:4aec0769f1799a9d4496827292c02a7b1f75c0bab56ab2b60dd94ebb57cbd5ee", - "sha256:55369d95afaacf2fa6b49c84d18b51f1704a6560c432a0f9a1aeb23f7b971308", - "sha256:6c098b85442c8fe3303e708bbb775afd0f6b29f77612e8892627bcab4b939357", - "sha256:9182cd6f93412d32e009020a44d6d170d2093646464a88aeec2aef50592f8c78", - "sha256:c8cbc21bbfa1dd7d5386d48cc814fe3d35b80f60299cdde9279046f399c3b0d8", - "sha256:db6f70a4b09cde813a4807843abaaa60f3b15fb4a2a06f9ae9c311472662daa1", - "sha256:f17495e6fe3d377e3faac68121caef6f974fcb9e046bc075bcff40d8e5cc69a4", - "sha256:f85900b9cca0c67767bb61b2b9bd53208aaa7373dae633dbe25d179b4bf38aa7" + "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9", + "sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f", + "sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb", + "sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320", + "sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842", + "sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0", + "sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829", + "sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310", + "sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4", + "sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8", + "sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1" ], - "version": "==1.2.6" + "version": "==1.3.0" } }, "develop": { "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.1" + "version": "==1.3.0" }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", + "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" ], - "version": "==4.3.0" + "version": "==6.0.0" + }, + "mypy": { + "hashes": [ + "sha256:308c274eb8482fbf16006f549137ddc0d69e5a589465e37b99c4564414363ca7", + "sha256:e80fd6af34614a0e898a57f14296d0dacb584648f0339c2e000ddbf0f4cc2f8d" + ], + "index": "pypi", + "version": "==0.670" + }, + "mypy-extensions": { + "hashes": [ + "sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812", + "sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e" + ], + "version": "==0.4.1" }, "pluggy": { "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", + "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" ], - "version": "==0.8.0" + "version": "==0.9.0" }, "py": { "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" ], - "version": "==1.7.0" + "version": "==1.8.0" }, "pyflakes": { "hashes": [ - "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", - "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], "index": "pypi", - "version": "==2.0.0" + "version": "==2.1.1" }, "pytest": { "hashes": [ @@ -525,10 +537,34 @@ }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" + }, + "typed-ast": { + "hashes": [ + "sha256:035a54ede6ce1380599b2ce57844c6554666522e376bd111eb940fbc7c3dad23", + "sha256:037c35f2741ce3a9ac0d55abfcd119133cbd821fffa4461397718287092d9d15", + "sha256:049feae7e9f180b64efacbdc36b3af64a00393a47be22fa9cb6794e68d4e73d3", + "sha256:19228f7940beafc1ba21a6e8e070e0b0bfd1457902a3a81709762b8b9039b88d", + "sha256:2ea681e91e3550a30c2265d2916f40a5f5d89b59469a20f3bad7d07adee0f7a6", + "sha256:3a6b0a78af298d82323660df5497bcea0f0a4a25a0b003afd0ce5af049bd1f60", + "sha256:5385da8f3b801014504df0852bf83524599df890387a3c2b17b7caa3d78b1773", + "sha256:606d8afa07eef77280c2bf84335e24390055b478392e1975f96286d99d0cb424", + "sha256:69245b5b23bbf7fb242c9f8f08493e9ecd7711f063259aefffaeb90595d62287", + "sha256:6f6d839ab09830d59b7fa8fb6917023d8cb5498ee1f1dbd82d37db78eb76bc99", + "sha256:730888475f5ac0e37c1de4bd05eeb799fdb742697867f524dc8a4cd74bcecc23", + "sha256:9819b5162ffc121b9e334923c685b0d0826154e41dfe70b2ede2ce29034c71d8", + "sha256:9e60ef9426efab601dd9aa120e4ff560f4461cf8442e9c0a2b92548d52800699", + "sha256:af5fbdde0690c7da68e841d7fc2632345d570768ea7406a9434446d7b33b0ee1", + "sha256:b64efdbdf3bbb1377562c179f167f3bf301251411eb5ac77dec6b7d32bcda463", + "sha256:bac5f444c118aeb456fac1b0b5d14c6a71ea2a42069b09c176f75e9bd4c186f6", + "sha256:bda9068aafb73859491e13b99b682bd299c1b5fd50644d697533775828a28ee0", + "sha256:d659517ca116e6750101a1326107d3479028c5191f0ecee3c7203c50f5b915b0", + "sha256:eddd3fb1f3e0f82e5915a899285a39ee34ce18fd25d89582bc89fc9fb16cd2c6" + ], + "version": "==1.3.1" } } } diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..7476b25 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +# Internal documentation + +The Litecord Voice Server Protocol (LVSP) is documented here. diff --git a/docs/admin_api.md b/docs/admin_api.md new file mode 100644 index 0000000..3c4649c --- /dev/null +++ b/docs/admin_api.md @@ -0,0 +1,45 @@ +# Litecord Admin API + +the base path is `/api/v6/admin`. + +## GET `/voice/regions/` + +Return a list of voice server objects for the region. + +Returns empty list if the region does not exist. + +| field | type | description | +| --: | :-- | :-- | +| hostname | string | the hostname of the voice server | +| last\_health | float | the health of the voice server | + +## PUT `/voice/regions` + +Create a voice region. + +Receives JSON body as input, returns a list of voice region objects as output. + +| field | type | description | +| --: | :-- | :-- | +| id | string | id of the voice region, "brazil", "us-east", "eu-west", etc | +| name | string | name of the voice region | +| vip | Optional[bool] | if voice region is vip-only, default false | +| deprecated | Optional[bool] | if voice region is deprecated, default false | +| custom | Optional[bool] | if voice region is custom-only, default false | + +## PUT `/voice/regions//server` + +Create a voice server for a region. + +Returns empty body with 204 status code on success. + +| field | type | description | +| --: | :-- | :-- | +| hostname | string | the hostname of the voice server | + +## PUT `/voice/regions//deprecate` + +Mark a voice region as deprecated. Disables any voice actions on guilds that are +using the voice region. + +Returns empty body with 204 status code on success. diff --git a/docs/lvsp.md b/docs/lvsp.md new file mode 100644 index 0000000..bbd31ba --- /dev/null +++ b/docs/lvsp.md @@ -0,0 +1,228 @@ +# Litecord Voice Server Protocol (LVSP) + +LVSP is a protocol for Litecord to communicate with an external component +dedicated for voice data. The voice server is responsible for the +Voice Websocket Discord and Voice UDP connections. + +LVSP runs over a *long-lived* websocket with TLS. The encoding is JSON. + +## OP code table + +"client" is litecord. "server" is the voice server. + +| opcode | name | sent by | +| --: | :-- | :-- | +| 0 | HELLO | server | +| 1 | IDENTIFY | client | +| 2 | RESUME | client | +| 3 | READY | server | +| 4 | HEARTBEAT | client | +| 5 | HEARTBEAT\_ACK | server | +| 6 | INFO | client / server | + +## Message structure + +Message data is defined by each opcode. + +**Note:** the `snowflake` type follows the same rules as the Discord Gateway's +snowflake type: A string encoding a Discord Snowflake. + +| field | type | description | +| --: | :-- | :-- | +| op | integer, opcode | operator code | +| d | map[string, any] | message data | + +## High level overview + + - connect, receive HELLO + - send IDENTIFY or RESUME + - if RESUME, process incoming messages as they were post-ready + - receive READY + - start HEARTBEAT'ing + - send INFO messages as needed + +## Error codes + +| code | meaning | +| --: | :-- | +| 4000 | general error. reconnect | +| 4001 | authentication failure | +| 4002 | decode error, given message failed to decode as json | + +## HELLO message + +Sent by the server when a connection is established. + +| field | type | description | +| --: | :-- | :-- | +| heartbeat\_interval | integer | amount of milliseconds to heartbeat with | +| nonce | string | random 10-character string used in authentication | + +## IDENTIFY message + +Sent by the client to identify itself. + +| field | type | description | +| --: | :-- | :-- | +| token | string | `HMAC(SHA256, key=[secret shared between server and client]), message=[nonce from HELLO]` | + +## READY message + + - The `health` field is described with more detail in the `HEARTBEAT_ACK` + message. + +| field | type | description | +| --: | :-- | :-- | +| `health` | Health | server health | + +## HEARTBEAT message + +Sent by the client as a keepalive / health monitoring method. + +The server MUST reply with a HEARTBEAT\_ACK message back in a reasonable +time period. + +There are no other fields in this message. + +## HEARTBEAT\_ACK message + +Sent by the server in reply to a HEARTBEAT message coming from the client. + +The `health` field is a measure of the servers's overall health. It is a +float going from 0 to 1, where 0 is the worst health possible, and 1 is the +best health possible. + +Servers SHOULD use the same algorithm to determine health, it CAN be based off: + - Machine resource usage (RAM, CPU, etc), however they're too general and can + be unreliable. + - Total users connected. + - Total bandwidth used in some X amount of time. + +Among others. + +| field | type | description | +| --: | :-- | :-- | +| health | float | server health | + +## INFO message + +Sent by either client or a server to send information between eachother. +The INFO message is extensible in which many request / response scenarios +are laid on. + +*This message type MUST be replayable.* + +| field | type | description | +| --: | :-- | :-- | +| type | InfoType | info type | +| data | Any | info data, varies depending on InfoType | + +### InfoType Enum + +| value | name | description | +| --: | :-- | :-- | +| 0 | CHANNEL\_REQ | channel assignment request | +| 1 | CHANNEL\_ASSIGN | channel assignment reply | +| 2 | CHANNEL\_DESTROY | channel destroy | +| 3 | VST\_CREATE | voice state create request | +| 4 | VST\_DONE | voice state created | +| 5 | VST\_UPDATE | voice state update | +| 6 | VST\_LEAVE | voice state leave | + +### CHANNEL\_REQ + +Request a channel to be created inside the voice server. + +The Server MUST reply back with a CHANNEL\_ASSIGN when resources are +allocated for the channel. + +| field | type | description | +| --: | :-- | :-- | +| channel\_id | snowflake | channel id | +| guild\_id | Optional[snowflake] | guild id, not provided if dm / group dm | + +### CHANNEL\_ASSIGN + +Sent by the Server to signal the successful creation of a voice channel. + +| field | type | description | +| --: | :-- | :-- | +| channel\_id | snowflake | channel id | +| guild\_id | Optional[snowflake] | guild id, not provided if dm / group dm | +| token | string | authentication token | + +### CHANNEL\_DESTROY + +Sent by the client to signal the destruction of a voice channel. Be it +a channel being deleted, or all members in it leaving. + +Same data as CHANNEL\_ASSIGN, but without `token`. + +### VST\_CREATE + +Sent by the client to create a voice state. + +| field | type | description | +| --: | :-- | :-- | +| user\_id | snowflake | user id | +| channel\_id | snowflake | channel id | +| guild\_id | Optional[snowflake] | guild id. not provided if dm / group dm | + +### VST\_DONE + +Sent by the server to indicate the success of a VST\_CREATE. + +Has the same fields as VST\_CREATE, but with extras: + +| field | type | description | +| --: | :-- | :-- | +| session\_id | string | session id for the voice state | + +### VST\_DESTROY + +Sent by the client when a user is leaving a channel OR moving between channels +in a guild. More on state transitions later on. + +| field | type | description | +| --: | :-- | :-- | +| session\_id | string | session id for the voice state | + +## Common logic scenarios + +### User joins an unitialized voice channel + +Since the channel is unitialized, both logic on initialization AND +user join is here. + + - Client will send a CHANNEL\_REQ. + - Client MAY send a VST\_CREATE right after as well. + - The Server MUST process CHANNEL\_REQ first, so the Server can keep + a lock on channel operations while it is initialized. + - Reply with CHANNEL\_ASSIGN once initialization is done. + - Process VST\_CREATE + +### Updating a voice channel + + - Client sends CHANNEL\_UPDATE. + - Server DOES NOT reply. + +### Destroying a voice channel + + - Client sends CHANNEL\_DESTROY. + - Server MUST disconnect any users currently connected with its + voice websocket. + +### User joining an (initialized) voice channel + + - Client sends VST\_CREATE + - Server sends VST\_DONE + +### User leaves a channel + + - Client sends VST\_DESTROY with the old fields + +### User moves a channel + + - Client sends VST\_DESTROY with the old fields + - Client sends VST\_CREATE with the new fields + - Server sends VST\_DONE diff --git a/litecord/admin_schemas.py b/litecord/admin_schemas.py new file mode 100644 index 0000000..3896dc9 --- /dev/null +++ b/litecord/admin_schemas.py @@ -0,0 +1,31 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +VOICE_SERVER = { + 'hostname': {'type': 'string', 'maxlength': 255, 'required': True} +} + +VOICE_REGION = { + 'id': {'type': 'string', 'maxlength': 255, 'required': True}, + 'name': {'type': 'string', 'maxlength': 255, 'required': True}, + + 'vip': {'type': 'boolean', 'default': False}, + 'deprecated': {'type': 'boolean', 'default': False}, + 'custom': {'type': 'boolean', 'default': False}, +} diff --git a/litecord/auth.py b/litecord/auth.py index 8294d7e..715865f 100644 --- a/litecord/auth.py +++ b/litecord/auth.py @@ -29,6 +29,7 @@ from quart import request, current_app as app from litecord.errors import Forbidden, Unauthorized, BadRequest from litecord.snowflake import get_snowflake +from litecord.enums import UserFlags log = Logger(__name__) @@ -100,6 +101,23 @@ async def token_check(): return user_id +async def admin_check(): + """Check if the user is an admin.""" + user_id = await token_check() + + flags = await app.db.fetchval(""" + SELECT flags + FROM users + WHERE id = $1 + """, user_id) + + flags = UserFlags.from_int(flags) + if not flags.is_staff: + raise Unauthorized('you are not staff') + + return user_id + + async def hash_data(data: str, loop=None) -> str: """Hash information with bcrypt.""" loop = loop or app.loop diff --git a/litecord/gateway/__init__.py b/litecord/blueprints/admin_api/__init__.py similarity index 89% rename from litecord/gateway/__init__.py rename to litecord/blueprints/admin_api/__init__.py index 3b62d06..d209cf9 100644 --- a/litecord/gateway/__init__.py +++ b/litecord/blueprints/admin_api/__init__.py @@ -17,6 +17,6 @@ along with this program. If not, see . """ -__all__ = ['websocket_handler'] +from .voice import bp as voice -from .gateway import websocket_handler +__all__ = ['voice'] diff --git a/litecord/blueprints/admin_api/voice.py b/litecord/blueprints/admin_api/voice.py new file mode 100644 index 0000000..e700b27 --- /dev/null +++ b/litecord/blueprints/admin_api/voice.py @@ -0,0 +1,129 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +import asyncpg +from quart import Blueprint, jsonify, current_app as app, request +from logbook import Logger + +from litecord.auth import admin_check +from litecord.schemas import validate +from litecord.admin_schemas import VOICE_SERVER, VOICE_REGION +from litecord.errors import BadRequest + +log = Logger(__name__) +bp = Blueprint('voice_admin', __name__) + + +@bp.route('/regions/', methods=['GET']) +async def get_region_servers(region): + """Return a list of all servers for a region.""" + await admin_check() + servers = await app.voice.voice_server_list(region) + return jsonify(servers) + + +@bp.route('/regions', methods=['PUT']) +async def insert_new_region(): + """Create a voice region.""" + await admin_check() + j = validate(await request.get_json(), VOICE_REGION) + + j['id'] = j['id'].lower() + + await app.db.execute(""" + INSERT INTO voice_regions (id, name, vip, deprecated, custom) + VALUES ($1, $2, $3, $4, $5) + """, j['id'], j['name'], j['vip'], j['deprecated'], j['custom']) + + regions = await app.storage.all_voice_regions() + region_count = len(regions) + + # if region count is 1, this is the first region to be created, + # so we should update all guilds to that region + if region_count == 1: + res = await app.db.execute(""" + UPDATE guilds + SET region = $1 + """, j['id']) + + log.info('updating guilds to first voice region: {}', res) + + return jsonify(regions) + + +@bp.route('/regions//servers', methods=['PUT']) +async def put_region_server(region): + """Insert a voice server to a region""" + await admin_check() + j = validate(await request.get_json(), VOICE_SERVER) + + try: + await app.db.execute(""" + INSERT INTO voice_servers (hostname, region_id) + VALUES ($1, $2) + """, j['hostname'], region) + except asyncpg.UniqueViolationError: + raise BadRequest('voice server already exists with given hostname') + + return '', 204 + + +@bp.route('/regions//deprecate', methods=['PUT']) +async def deprecate_region(region): + """Deprecate a voice region.""" + await admin_check() + + # TODO: write this + await app.voice.disable_region(region) + + await app.db.execute(""" + UPDATE voice_regions + SET deprecated = true + WHERE id = $1 + """, region) + + return '', 204 + + +async def guild_region_check(app_): + """Check all guilds for voice region inconsistencies. + + Since the voice migration caused all guilds.region columns + to become NULL, we need to remove such NULLs if we have more + than one region setup. + """ + + regions = await app_.storage.all_voice_regions() + + if not regions: + log.info('region check: no regions to move guilds to') + return + + res = await app_.db.execute(""" + UPDATE guilds + SET region = ( + SELECT id + FROM voice_regions + OFFSET floor(random()*$1) + LIMIT 1 + ) + WHERE region = NULL + """, len(regions)) + + log.info('region check: updating guild.region=null: {!r}', res) diff --git a/litecord/blueprints/channel/dm_checks.py b/litecord/blueprints/channel/dm_checks.py index 743815d..1d0851a 100644 --- a/litecord/blueprints/channel/dm_checks.py +++ b/litecord/blueprints/channel/dm_checks.py @@ -58,11 +58,11 @@ async def dm_pre_check(user_id: int, channel_id: int, peer_id: int): user_settings = await app.user_storage.get_user_settings(user_id) peer_settings = await app.user_storage.get_user_settings(peer_id) - restricted_user = [int(v) for v in user_settings['restricted_guilds']] - restricted_peer = [int(v) for v in peer_settings['restricted_guilds']] + restricted_user_ = [int(v) for v in user_settings['restricted_guilds']] + restricted_peer_ = [int(v) for v in peer_settings['restricted_guilds']] - restricted_user = set(restricted_user) - restricted_peer = set(restricted_peer) + restricted_user = set(restricted_user_) + restricted_peer = set(restricted_peer_) mutual_guilds -= restricted_user mutual_guilds -= restricted_peer diff --git a/litecord/blueprints/channel/messages.py b/litecord/blueprints/channel/messages.py index cd978eb..41850e2 100644 --- a/litecord/blueprints/channel/messages.py +++ b/litecord/blueprints/channel/messages.py @@ -217,7 +217,8 @@ async def _guild_text_mentions(payload: dict, guild_id: int, # for the users that have a state # in the channel. if mentions_here: - uids = [] + uids = set() + await app.db.execute(""" UPDATE user_read_state SET mention_count = mention_count + 1 @@ -229,7 +230,7 @@ async def _guild_text_mentions(payload: dict, guild_id: int, # that might not have read permissions # to the channel. if mentions_everyone: - uids = [] + uids = set() member_ids = await app.storage.get_member_ids(guild_id) @@ -407,9 +408,6 @@ async def _create_message(channel_id): await _dm_pre_dispatch(channel_id, user_id) await _dm_pre_dispatch(channel_id, guild_id) - if payload['webhook_id'] == None: - payload.pop('webhook_id', None) - await app.dispatcher.dispatch('channel', channel_id, 'MESSAGE_CREATE', payload) diff --git a/litecord/blueprints/channels.py b/litecord/blueprints/channels.py index 2922bfb..2690440 100644 --- a/litecord/blueprints/channels.py +++ b/litecord/blueprints/channels.py @@ -18,6 +18,7 @@ along with this program. If not, see . """ import time +from typing import List, Optional from quart import Blueprint, request, current_app as app, jsonify from logbook import Logger @@ -262,8 +263,11 @@ async def _update_pos(channel_id, pos: int): """, pos, channel_id) -async def _mass_chan_update(guild_id, channel_ids: int): +async def _mass_chan_update(guild_id, channel_ids: List[Optional[int]]): for channel_id in channel_ids: + if channel_id is None: + continue + chan = await app.storage.get_channel(channel_id) await app.dispatcher.dispatch( 'guild', guild_id, 'CHANNEL_UPDATE', chan) @@ -337,7 +341,7 @@ async def _update_channel_common(channel_id, guild_id: int, j: dict): if 'position' in j: channel_data = await app.storage.get_channel_data(guild_id) - chans = [None * len(channel_data)] + chans = [None] * len(channel_data) for chandata in channel_data: chans.insert(chandata['position'], int(chandata['id'])) @@ -393,7 +397,7 @@ async def _common_guild_chan(channel_id, j: dict): """, j[field], channel_id) -async def _update_text_channel(channel_id: int, j: dict): +async def _update_text_channel(channel_id: int, j: dict, _user_id: int): # first do the specific ones related to guild_text_channels for field in [field for field in j.keys() if field in ('topic', 'rate_limit_per_user')]: @@ -406,7 +410,7 @@ async def _update_text_channel(channel_id: int, j: dict): await _common_guild_chan(channel_id, j) -async def _update_voice_channel(channel_id: int, j: dict): +async def _update_voice_channel(channel_id: int, j: dict, _user_id: int): # first do the specific ones in guild_voice_channels for field in [field for field in j.keys() if field in ('bitrate', 'user_limit')]: diff --git a/litecord/blueprints/gateway.py b/litecord/blueprints/gateway.py index 562040b..05303f0 100644 --- a/litecord/blueprints/gateway.py +++ b/litecord/blueprints/gateway.py @@ -29,7 +29,7 @@ bp = Blueprint('gateway', __name__) def get_gw(): """Get the gateway's web""" proto = 'wss://' if app.config['IS_SSL'] else 'ws://' - return f'{proto}{app.config["WEBSOCKET_URL"]}/ws' + return f'{proto}{app.config["WEBSOCKET_URL"]}' @bp.route('/gateway') diff --git a/litecord/blueprints/guild/members.py b/litecord/blueprints/guild/members.py index 8dd9204..3558168 100644 --- a/litecord/blueprints/guild/members.py +++ b/litecord/blueprints/guild/members.py @@ -68,7 +68,7 @@ async def get_members(guild_id): async def _update_member_roles(guild_id: int, member_id: int, - wanted_roles: list): + wanted_roles: set): """Update the roles a member has.""" # first, fetch all current roles diff --git a/litecord/blueprints/guild/roles.py b/litecord/blueprints/guild/roles.py index bd419cb..5dfcbe7 100644 --- a/litecord/blueprints/guild/roles.py +++ b/litecord/blueprints/guild/roles.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import List, Dict +from typing import List, Dict, Tuple from quart import Blueprint, request, current_app as app, jsonify from logbook import Logger @@ -184,10 +184,11 @@ async def _role_pairs_update(guild_id: int, pairs: list): await _role_update_dispatch(role_1, guild_id) await _role_update_dispatch(role_2, guild_id) +PairList = List[Tuple[Tuple[int, int], Tuple[int, int]]] def gen_pairs(list_of_changes: List[Dict[str, int]], current_state: Dict[int, int], - blacklist: List[int] = None) -> List[tuple]: + blacklist: List[int] = None) -> PairList: """Generate a list of pairs that, when applied to the database, will generate the desired state given in list_of_changes. @@ -256,13 +257,16 @@ def gen_pairs(list_of_changes: List[Dict[str, int]], # position we want to change to element_2 = current_state.get(new_pos_1) + if element_2 is None: + continue + # if there is, is that existing channel being # swapped to another position? new_pos_2 = preferred_state.get(element_2) # if its being swapped to leave space, add it # to the pairs list - if new_pos_2: + if new_pos_2 is not None: pairs.append( ((element_1, new_pos_1), (element_2, new_pos_2)) ) diff --git a/litecord/blueprints/invites.py b/litecord/blueprints/invites.py index f97bd65..f6fb585 100644 --- a/litecord/blueprints/invites.py +++ b/litecord/blueprints/invites.py @@ -91,7 +91,7 @@ async def invite_precheck_gdm(user_id: int, channel_id: int): async def _inv_check_age(inv: dict): - if inv['max_age'] is 0: + if inv['max_age'] != 0: return now = datetime.datetime.utcnow() diff --git a/litecord/blueprints/user/billing.py b/litecord/blueprints/user/billing.py index eb4de46..f37744e 100644 --- a/litecord/blueprints/user/billing.py +++ b/litecord/blueprints/user/billing.py @@ -148,8 +148,6 @@ async def get_payment_source(user_id: int, source_id: int, db=None) -> dict: if not db: db = app.db - source = {} - source_type = await db.fetchval(""" SELECT source_type FROM user_payment_sources diff --git a/litecord/blueprints/voice.py b/litecord/blueprints/voice.py index 85e4121..a06eec1 100644 --- a/litecord/blueprints/voice.py +++ b/litecord/blueprints/voice.py @@ -17,13 +17,94 @@ along with this program. If not, see . """ -from quart import Blueprint, jsonify +from typing import Optional +from collections import Counter +from random import choice + +from quart import Blueprint, jsonify, current_app as app + +from litecord.blueprints.auth import token_check bp = Blueprint('voice', __name__) +def _majority_region_count(regions: list) -> str: + """Return the first most common element in a given list.""" + counter = Counter(regions) + common = counter.most_common(1) + region, _count = common[0] + + return region + + +async def _choose_random_region() -> Optional[str]: + """Give a random voice region.""" + regions = await app.db.fetch(""" + SELECT id + FROM voice_regions + """) + + regions = [r['id'] for r in regions] + + if not regions: + return None + + return choice(regions) + + +async def _majority_region_any(user_id) -> Optional[str]: + """Calculate the most likely region to make the user happy, but + this is based on the guilds the user is IN, instead of the guilds + the user owns.""" + guilds = await app.user_storage.get_user_guilds(user_id) + + if not guilds: + return await _choose_random_region() + + res = [] + + for guild_id in guilds: + region = await app.db.fetchval(""" + SELECT region + FROM guilds + WHERE id = $1 + """, guild_id) + + res.append(region) + + most_common = _majority_region_count(res) + + if most_common is None: + return await _choose_random_region() + + return most_common + + +async def majority_region(user_id: int) -> Optional[str]: + """Given a user ID, give the most likely region for the user to be + happy with.""" + regions = await app.db.fetch(""" + SELECT region + FROM guilds + WHERE owner_id = $1 + """, user_id) + + if not regions: + return await _majority_region_any(user_id) + + regions = [r['region'] for r in regions] + return _majority_region_count(regions) + + @bp.route('/regions', methods=['GET']) async def voice_regions(): - return jsonify([ - {'name': 'Brazil', 'deprecated': False, 'id': 'Brazil', 'optimal': True, 'vip': True} - ]) + """Return voice regions.""" + user_id = await token_check() + + best_region = await majority_region(user_id) + regions = await app.storage.all_voice_regions() + + for region in regions: + region['optimal'] = region['id'] == best_region + + return jsonify(regions) diff --git a/litecord/embed/sanitizer.py b/litecord/embed/sanitizer.py index 4b0b0b9..ef912c4 100644 --- a/litecord/embed/sanitizer.py +++ b/litecord/embed/sanitizer.py @@ -22,7 +22,7 @@ litecord.embed.sanitizer sanitize embeds by giving common values such as type: rich """ -from typing import Dict, Any +from typing import Dict, Any, Optional, Union, List from logbook import Logger from quart import current_app as app @@ -44,7 +44,7 @@ def sanitize_embed(embed: Embed) -> Embed: }} -def path_exists(embed: Embed, components: str): +def path_exists(embed: Embed, components_in: Union[List[str], str]): """Tell if a given path exists in an embed (or any dictionary). The components string is formatted like this: @@ -54,10 +54,10 @@ def path_exists(embed: Embed, components: str): """ # get the list of components given - if isinstance(components, str): - components = components.split('.') + if isinstance(components_in, str): + components = components_in.split('.') else: - components = list(components) + components = list(components_in) # if there are no components, we reached the end of recursion # and can return true @@ -96,7 +96,7 @@ def proxify(url, *, config=None) -> str: ) -async def fetch_metadata(url, *, config=None, session=None) -> dict: +async def fetch_metadata(url, *, config=None, session=None) -> Optional[Dict]: """Fetch metadata for a url.""" if session is None: @@ -123,7 +123,7 @@ async def fetch_metadata(url, *, config=None, session=None) -> dict: log.warning('failed to generate meta for {!r}: {} {!r}', url, resp.status, body) - return + return None return await resp.json() diff --git a/litecord/enums.py b/litecord/enums.py index 60037d6..4107fd5 100644 --- a/litecord/enums.py +++ b/litecord/enums.py @@ -83,6 +83,12 @@ GUILD_CHANS = (ChannelType.GUILD_TEXT, ChannelType.GUILD_CATEGORY) +VOICE_CHANNELS = ( + ChannelType.DM, ChannelType.GUILD_VOICE, + ChannelType.GUILD_CATEGORY +) + + class ActivityType(EasyEnum): PLAYING = 0 STREAMING = 1 diff --git a/litecord/errors.py b/litecord/errors.py index 1e9942a..64c0ebf 100644 --- a/litecord/errors.py +++ b/litecord/errors.py @@ -133,8 +133,18 @@ class MissingPermissions(Forbidden): class WebsocketClose(Exception): @property def code(self): + from_class = getattr(self, 'close_code', None) + + if from_class: + return from_class + return self.args[0] @property def reason(self): + from_class = getattr(self, 'close_code', None) + + if from_class: + return self.args[0] + return self.args[1] diff --git a/litecord/gateway/websocket.py b/litecord/gateway/websocket.py index ca2d444..c2977b7 100644 --- a/litecord/gateway/websocket.py +++ b/litecord/gateway/websocket.py @@ -30,7 +30,7 @@ from logbook import Logger import earl from litecord.auth import raw_token_check -from litecord.enums import RelationshipType +from litecord.enums import RelationshipType, ChannelType from litecord.schemas import validate, GW_STATUS_UPDATE from litecord.utils import ( task_wrapper, LitecordJSONEncoder, yield_chunks @@ -47,15 +47,20 @@ from litecord.gateway.errors import ( DecodeError, UnknownOPCode, InvalidShard, ShardingRequired ) +from litecord.storage import int_ + log = Logger(__name__) + WebsocketProperties = collections.namedtuple( 'WebsocketProperties', 'v encoding compress zctx tasks' ) WebsocketObjects = collections.namedtuple( - 'WebsocketObjects', ('db', 'state_manager', 'storage', - 'loop', 'dispatcher', 'presence', 'ratelimiter', - 'user_storage') + 'WebsocketObjects', ( + 'db', 'state_manager', 'storage', + 'loop', 'dispatcher', 'presence', 'ratelimiter', + 'user_storage', 'voice' + ) ) @@ -113,7 +118,7 @@ class GatewayWebsocket: self.ext = WebsocketObjects( app.db, app.state_manager, app.storage, app.loop, app.dispatcher, app.presence, app.ratelimiter, - app.user_storage + app.user_storage, app.voice ) self.storage = self.ext.storage @@ -230,7 +235,7 @@ class GatewayWebsocket: 's': None }) - def _check_ratelimit(self, key: str, ratelimit_key: str): + def _check_ratelimit(self, key: str, ratelimit_key): ratelimit = self.ext.ratelimiter.get_ratelimit(f'_ws.{key}') bucket = ratelimit.get_bucket(ratelimit_key) return bucket.update_rate_limit() @@ -287,7 +292,7 @@ class GatewayWebsocket: await self.send(payload) - async def _make_guild_list(self) -> List[int]: + async def _make_guild_list(self) -> List[Dict[str, Any]]: user_id = self.state.user_id guild_ids = await self._guild_ids() @@ -598,16 +603,85 @@ class GatewayWebsocket: # setting new presence to state await self.update_status(presence) + def voice_key(self, channel_id: int, guild_id: int): + """Voice state key.""" + return (self.state.user_id, self.state.session_id) + + async def _vsu_get_prop(self, state, data): + """Get voice state properties from data, fallbacking to + user settings.""" + try: + # TODO: fetch from settings if not provided + self_deaf = bool(data['self_deaf']) + self_mute = bool(data['self_mute']) + except (KeyError, ValueError): + pass + + return { + 'deaf': state.deaf, + 'mute': state.mute, + 'self_deaf': self_deaf, + 'self_mute': self_mute, + } + async def handle_4(self, payload: Dict[str, Any]): """Handle OP 4 Voice Status Update.""" data = payload['d'] - # for now, ignore - log.debug('got VSU cid={} gid={} deaf={} mute={} video={}', - data.get('channel_id'), - data.get('guild_id'), - data.get('self_deaf'), - data.get('self_mute'), - data.get('self_video')) + + if not self.state: + return + + channel_id = int_(data.get('channel_id')) + guild_id = int_(data.get('guild_id')) + + # if its null and null, disconnect the user from any voice + # TODO: maybe just leave from DMs? idk... + if channel_id is None and guild_id is None: + return await self.ext.voice.leave_all(self.state.user_id) + + # if guild is not none but channel is, we are leaving + # a guild's channel + if channel_id is None: + return await self.ext.voice.leave(guild_id, self.state.user_id) + + # fetch an existing state given user and guild OR user and channel + chan_type = ChannelType( + await self.storage.get_chan_type(channel_id) + ) + + state_id2 = channel_id + + if chan_type == ChannelType.GUILD_VOICE: + state_id2 = guild_id + + # a voice state key is a Tuple[int, int] + # - [0] is the user id + # - [1] is the channel id or guild id + + # the old approach was a (user_id, session_id), but + # that does not work. + + # this works since users can be connected to many channels + # using a single gateway websocket connection. HOWEVER, + # they CAN NOT enter two channels in a single guild. + + # this state id format takes care of that. + voice_key = (self.state.user_id, state_id2) + voice_state = await self.ext.voice.get_state(voice_key) + + if voice_state is None: + return await self.ext.voice.create_state(voice_key, data) + + same_guild = guild_id == voice_state.guild_id + same_channel = channel_id == voice_state.channel_id + + prop = await self._vsu_get_prop(voice_state, data) + + if same_guild and same_channel: + return await self.ext.voice.update_state(voice_state, prop) + + if same_guild and not same_channel: + return await self.ext.voice.move_state(voice_state, channel_id) async def _handle_5(self, payload: Dict[str, Any]): """Handle OP 5 Voice Server Ping. @@ -698,7 +772,7 @@ class GatewayWebsocket: await self._resume(range(seq, state.seq)) - async def _req_guild_members(self, guild_id: str, user_ids: List[int], + async def _req_guild_members(self, guild_id, user_ids: List[int], query: str, limit: int): try: guild_id = int(guild_id) diff --git a/litecord/images.py b/litecord/images.py index d4c1ea9..1db1ece 100644 --- a/litecord/images.py +++ b/litecord/images.py @@ -22,6 +22,7 @@ import mimetypes import asyncio import base64 import tempfile +from typing import Optional from dataclasses import dataclass from hashlib import sha256 @@ -67,22 +68,33 @@ def get_mime(ext: str): @dataclass class Icon: """Main icon class""" - key: str - icon_hash: str - mime: str + key: Optional[str] + icon_hash: Optional[str] + mime: Optional[str] @property - def as_path(self) -> str: + def as_path(self) -> Optional[str]: """Return a filesystem path for the given icon.""" + if self.mime is None: + return None + ext = get_ext(self.mime) return str(IMAGE_FOLDER / f'{self.key}_{self.icon_hash}.{ext}') @property - def as_pathlib(self) -> str: + def as_pathlib(self) -> Optional[Path]: + """Get a Path instance of this icon.""" + if self.as_path is None: + return None + return Path(self.as_path) @property - def extension(self) -> str: + def extension(self) -> Optional[str]: + """Get the extension of this icon.""" + if self.mime is None: + return None + return get_ext(self.mime) @@ -91,7 +103,7 @@ class ImageError(Exception): pass -def to_raw(data_type: str, data: str) -> bytes: +def to_raw(data_type: str, data: str) -> Optional[bytes]: """Given a data type in the data URI and data, give the raw bytes being encoded.""" if data_type == 'base64': @@ -176,7 +188,7 @@ def _gen_update_sql(scope: str) -> str: """ -def _invalid(kwargs: dict): +def _invalid(kwargs: dict) -> Optional[Icon]: """Send an invalid value.""" if not kwargs.get('always_icon', False): return None @@ -272,7 +284,8 @@ class IconManager: return Icon(icon.key, icon.icon_hash, target_mime) - async def generic_get(self, scope, key, icon_hash, **kwargs) -> Icon: + async def generic_get(self, scope, key, icon_hash, + **kwargs) -> Optional[Icon]: """Get any icon.""" log.debug('GET {} {} {}', scope, key, icon_hash) @@ -300,10 +313,17 @@ class IconManager: icon = Icon(icon_row['key'], icon_row['hash'], icon_row['mime']) + # ensure we aren't messing with NULLs everywhere. + if icon.as_pathlib is None: + return None + if not icon.as_pathlib.exists(): await self.delete(icon) return None + if icon.extension is None: + return None + if 'ext' in kwargs and kwargs['ext'] != icon.extension: return await self._convert_ext(icon, kwargs['ext']) diff --git a/litecord/permissions.py b/litecord/permissions.py index 589b976..4b062e8 100644 --- a/litecord/permissions.py +++ b/litecord/permissions.py @@ -18,6 +18,7 @@ along with this program. If not, see . """ import ctypes +from typing import Optional from quart import current_app as app @@ -198,7 +199,8 @@ async def role_permissions(guild_id: int, role_id: int, async def compute_overwrites(base_perms: Permissions, user_id, channel_id: int, - guild_id: int = None, storage=None): + guild_id: Optional[int] = None, + storage=None): """Compute the permissions in the context of a channel.""" if not storage: storage = app.storage @@ -211,8 +213,12 @@ async def compute_overwrites(base_perms: Permissions, # list of overwrites overwrites = await storage.chan_overwrites(channel_id) + # if the channel isn't a guild, we should just return + # ALL_PERMISSIONS. the old approach was calling guild_from_channel + # again, but it is already passed by get_permissions(), so its + # redundant. if not guild_id: - guild_id = await storage.guild_from_channel(channel_id) + return ALL_PERMISSIONS # make it a map for better usage overwrites = {int(o['id']): o for o in overwrites} diff --git a/litecord/presence.py b/litecord/presence.py index a8c267e..bf7bad9 100644 --- a/litecord/presence.py +++ b/litecord/presence.py @@ -127,7 +127,7 @@ class PresenceManager: # shards that are in lazy guilds with 'everyone' # enabled - in_lazy = [] + in_lazy: List[str] = [] for member_list in lists: session_ids = await member_list.pres_update( diff --git a/litecord/pubsub/channel.py b/litecord/pubsub/channel.py index 2b32a1d..15e4010 100644 --- a/litecord/pubsub/channel.py +++ b/litecord/pubsub/channel.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import Any +from typing import Any, List from logbook import Logger @@ -54,13 +54,13 @@ class ChannelDispatcher(DispatcherWithState): VAL_TYPE = int async def dispatch(self, channel_id, - event: str, data: Any): + event: str, data: Any) -> List[str]: """Dispatch an event to a channel.""" # get everyone who is subscribed # and store the number of states we dispatched the event to user_ids = self.state[channel_id] dispatched = 0 - sessions = [] + sessions: List[str] = [] # making a copy of user_ids since # we'll modify it later on. @@ -84,7 +84,7 @@ class ChannelDispatcher(DispatcherWithState): await self.unsub(channel_id, user_id) continue - cur_sess = 0 + cur_sess = [] if event in ('CHANNEL_CREATE', 'CHANNEL_UPDATE') \ and data.get('type') == ChannelType.GROUP_DM.value: diff --git a/litecord/pubsub/dispatcher.py b/litecord/pubsub/dispatcher.py index 4b60019..dd03ef2 100644 --- a/litecord/pubsub/dispatcher.py +++ b/litecord/pubsub/dispatcher.py @@ -17,9 +17,7 @@ along with this program. If not, see . """ -""" -litecord.pubsub.dispatcher: main dispatcher class -""" +from typing import List from collections import defaultdict from logbook import Logger @@ -82,7 +80,8 @@ class Dispatcher: """ raise NotImplementedError - async def _dispatch_states(self, states: list, event: str, data) -> int: + async def _dispatch_states(self, states: list, event: str, + data) -> List[str]: """Dispatch an event to a list of states.""" res = [] diff --git a/litecord/pubsub/lazy_guild.py b/litecord/pubsub/lazy_guild.py index 3e9b7b3..2cc9527 100644 --- a/litecord/pubsub/lazy_guild.py +++ b/litecord/pubsub/lazy_guild.py @@ -28,7 +28,9 @@ lazy guilds: import asyncio from collections import defaultdict -from typing import Any, List, Dict, Union +from typing import ( + Any, List, Dict, Union, Optional, Iterable, Iterator, Tuple, Set +) from dataclasses import dataclass, asdict, field from logbook import Logger @@ -39,7 +41,7 @@ from litecord.permissions import ( ) from litecord.utils import index_by_func from litecord.utils import mmh3 - +from litecord.gateway.state import GatewayState log = Logger(__name__) @@ -113,7 +115,7 @@ class MemberList: yield group, self.data[group.gid] @property - def iter_non_empty(self) -> tuple: + def iter_non_empty(self) -> Iterator[Tuple[GroupInfo, List[int]]]: """Only iterate through non-empty groups. Note that while the offline group can be empty, it is always @@ -265,7 +267,7 @@ class GuildMemberList: #: store the states that are subscribed to the list. # type is {session_id: set[list]} - self.state = defaultdict(set) + self.state: Dict[str, Set[List[int, int]]] = defaultdict(set) self._list_lock = asyncio.Lock() @@ -359,7 +361,7 @@ class GuildMemberList: # then the final perms for that role if # any overwrite exists in the channel final_perms = overwrite_find_mix( - role_perms, self.list.overwrites, group.gid) + role_perms, self.list.overwrites, int(group.gid)) # update the group's permissions # with the mixed ones @@ -423,7 +425,7 @@ class GuildMemberList: async def _get_group_for_member(self, member_id: int, roles: List[Union[str, int]], - status: str) -> GroupID: + status: str) -> Optional[GroupID]: """Return a fitting group ID for the member.""" member_roles = list(map(int, roles)) @@ -463,15 +465,15 @@ class GuildMemberList: self.list.members[member_id] = member self.list.data[group_id].append(member_id) - def _display_name(self, member_id: int) -> str: + def _display_name(self, member_id: int) -> Optional[str]: """Get the display name for a given member. This is more efficient than the old function (not method) of same name, as we dont need to pass nickname information to it. """ - member = self.list.members.get(member_id) - - if not member_id: + try: + member = self.list.members[member_id] + except KeyError: return None username = member['user']['username'] @@ -578,7 +580,7 @@ class GuildMemberList: if not self.state: self._set_empty_list() - def _get_state(self, session_id: str): + def _get_state(self, session_id: str) -> Optional[GatewayState]: """Get the state for a session id. Wrapper for :meth:`StateManager.fetch_raw` @@ -589,7 +591,7 @@ class GuildMemberList: except KeyError: return None - async def _dispatch_sess(self, session_ids: List[str], + async def _dispatch_sess(self, session_ids: Iterable[str], operations: List[Operation]): """Dispatch a GUILD_MEMBER_LIST_UPDATE to the given session ids.""" @@ -613,11 +615,12 @@ class GuildMemberList: } states = map(self._get_state, session_ids) - states = filter(lambda state: state is not None, states) - dispatched = [] for state in states: + if state is None: + continue + await state.ws.dispatch( 'GUILD_MEMBER_LIST_UPDATE', payload) @@ -625,7 +628,8 @@ class GuildMemberList: return dispatched - async def _resync(self, session_ids: int, item_index: int) -> List[str]: + async def _resync(self, session_ids: List[str], + item_index: int) -> List[str]: """Send a SYNC event to all states that are subscribed to an item. Returns @@ -660,7 +664,7 @@ class GuildMemberList: return result - async def _resync_by_item(self, item_index: int): + async def _resync_by_item(self, item_index: Optional[int]): """Resync but only giving the item index.""" if item_index is None: return [] @@ -729,7 +733,7 @@ class GuildMemberList: # send SYNCs to the state that requested await self._dispatch_sess([session_id], ops) - def _get_item_index(self, user_id: Union[str, int]) -> int: + def _get_item_index(self, user_id: Union[str, int]) -> Optional[int]: """Get the item index a user is on.""" # NOTE: this is inefficient user_id = int(user_id) @@ -749,7 +753,7 @@ class GuildMemberList: return None - def _get_group_item_index(self, group_id: GroupID) -> int: + def _get_group_item_index(self, group_id: GroupID) -> Optional[int]: """Get the item index a group is on.""" index = 0 @@ -773,7 +777,7 @@ class GuildMemberList: return False - def _get_subs(self, item_index: int) -> filter: + def _get_subs(self, item_index: int) -> Iterable[str]: """Get the list of subscribed states to a given item.""" return filter( lambda sess_id: self._is_subbed(item_index, sess_id), @@ -1141,7 +1145,7 @@ class GuildMemberList: # when bots come along. self.list.data[new_group.gid] = [] - def _get_role_as_group_idx(self, role_id: int) -> int: + def _get_role_as_group_idx(self, role_id: int) -> Optional[int]: """Get a group index representing the given role id. Returns @@ -1338,7 +1342,10 @@ class GuildMemberList: log.debug('there are {} session ids to resync (for item {})', len(sess_ids_resync), role_item_index) - return await self._resync(sess_ids_resync, role_item_index) + if role_item_index is not None: + return await self._resync(sess_ids_resync, role_item_index) + + return [] async def chan_update(self): """Called then a channel's data has been updated.""" @@ -1443,10 +1450,10 @@ class LazyGuildDispatcher(Dispatcher): # remove it from guild map as well guild_id = gml.guild_id - self.guild_map[guild_id].pop(channel_id) + self.guild_map[guild_id].remove(channel_id) gml.close() - except KeyError: + except (KeyError, ValueError): pass async def chan_update(self, channel_id: int): diff --git a/litecord/schemas.py b/litecord/schemas.py index 8b4fd28..3683334 100644 --- a/litecord/schemas.py +++ b/litecord/schemas.py @@ -147,7 +147,7 @@ class LitecordValidator(Validator): def validate(reqjson: Union[Dict, List], schema: Dict, - raise_err: bool = True) -> Union[Dict, List]: + raise_err: bool = True) -> Dict: """Validate a given document (user-input) and give the correct document as a result. """ diff --git a/litecord/storage.py b/litecord/storage.py index 15b1871..1530a65 100644 --- a/litecord/storage.py +++ b/litecord/storage.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional from logbook import Logger @@ -52,11 +52,17 @@ def str_(val): return maybe(str, val) -def _filter_recipients(recipients: List[Dict[str, Any]], user_id: int): +def int_(val): + return maybe(int, val) + + +def bool_(val): + return maybe(int, val) + + +def _filter_recipients(recipients: List[Dict[str, Any]], user_id: str): """Filter recipients in a list of recipients, removing the one that is reundant (ourselves).""" - user_id = str(user_id) - return list(filter( lambda recipient: recipient['id'] != user_id, recipients)) @@ -69,7 +75,7 @@ class Storage: self.db = app.db self.presence = None - async def fetchrow_with_json(self, query: str, *args): + async def fetchrow_with_json(self, query: str, *args) -> Any: """Fetch a single row with JSON/JSONB support.""" # the pool by itself doesn't have # set_type_codec, so we must set it manually @@ -78,19 +84,19 @@ class Storage: await pg_set_json(con) return await con.fetchrow(query, *args) - async def fetch_with_json(self, query: str, *args): + async def fetch_with_json(self, query: str, *args) -> List[Any]: """Fetch many rows with JSON/JSONB support.""" async with self.db.acquire() as con: await pg_set_json(con) return await con.fetch(query, *args) - async def execute_with_json(self, query: str, *args): + async def execute_with_json(self, query: str, *args) -> str: """Execute a SQL statement with JSON/JSONB support.""" async with self.db.acquire() as con: await pg_set_json(con) return await con.execute(query, *args) - async def get_user(self, user_id, secure=False) -> Dict[str, Any]: + async def get_user(self, user_id, secure=False) -> Optional[Dict[str, Any]]: """Get a single user payload.""" user_id = int(user_id) @@ -107,7 +113,7 @@ class Storage: """, user_id) if not user_row: - return + return None duser = dict(user_row) @@ -133,14 +139,14 @@ class Storage: """Search a user""" if len(discriminator) < 4: # how do we do this in f-strings again..? - discriminator = '%04d' % discriminator + discriminator = '%04d' % int(discriminator) return await self.db.fetchval(""" SELECT id FROM users WHERE username = $1 AND discriminator = $2 """, username, discriminator) - async def get_guild(self, guild_id: int, user_id=None) -> Dict: + async def get_guild(self, guild_id: int, user_id=None) -> Optional[Dict]: """Get gulid payload.""" row = await self.db.fetchrow(""" SELECT id::text, owner_id::text, name, icon, splash, @@ -155,7 +161,7 @@ class Storage: """, guild_id) if not row: - return + return None drow = dict(row) @@ -165,14 +171,36 @@ class Storage: return drow async def _member_basic(self, guild_id: int, member_id: int): - return await self.db.fetchrow(""" - SELECT user_id, nickname, joined_at, deafened, muted + row = await self.db.fetchrow(""" + SELECT user_id, nickname, joined_at, + deafened AS deaf, muted AS mute FROM members WHERE guild_id = $1 and user_id = $2 """, guild_id, member_id) + if row is None: + return None + + row = dict(row) + row['joined_at'] = timestamp_(row['joined_at']) + return row + + async def _member_basic_with_roles(self, guild_id: int, + member_id: int): + basic = await self._member_basic(guild_id, member_id) + + if basic is None: + return None + + basic = dict(basic) + roles = await self.get_member_role_ids(guild_id, member_id) + + return {**basic, **{ + 'roles': roles + }} + async def get_member_role_ids(self, guild_id: int, - member_id: int) -> List[int]: + member_id: int) -> List[str]: """Get a list of role IDs that are on a member.""" roles = await self.db.fetch(""" SELECT role_id::text @@ -197,6 +225,7 @@ class Storage: async def _member_dict(self, row, guild_id, member_id) -> Dict[str, Any]: roles = await self.get_member_role_ids(guild_id, member_id) + return { 'user': await self.get_user(member_id), 'nick': row['nickname'], @@ -205,18 +234,18 @@ class Storage: # the user since it is known that everyone has # that role. 'roles': roles, - 'joined_at': timestamp_(row['joined_at']), - 'deaf': row['deafened'], - 'mute': row['muted'], + 'joined_at': row['joined_at'], + 'deaf': row['deaf'], + 'mute': row['mute'], } async def get_member_data_one(self, guild_id: int, - member_id: int) -> Dict[str, Any]: + member_id: int) -> Optional[Dict[str, Any]]: """Get data about one member in a guild.""" basic = await self._member_basic(guild_id, member_id) if not basic: - return + return None return await self._member_dict(basic, guild_id, member_id) @@ -238,7 +267,8 @@ class Storage: async def get_member_data(self, guild_id: int) -> List[Dict[str, Any]]: """Get member information on a guild.""" members_basic = await self.db.fetch(""" - SELECT user_id, nickname, joined_at, deafened, muted + SELECT user_id, nickname, joined_at, + deafened AS deaf, muted AS mute FROM members WHERE guild_id = $1 """, guild_id) @@ -314,6 +344,7 @@ class Storage: return {**row, **dict(vrow)} log.warning('unknown channel type: {}', chan_type) + return row async def get_chan_type(self, channel_id: int) -> int: """Get the channel type integer, given channel ID.""" @@ -368,7 +399,7 @@ class Storage: return [r['member_id'] for r in user_ids] async def _gdm_recipients(self, channel_id: int, - reference_id: int = None) -> List[int]: + reference_id: int = None) -> List[Dict]: """Get the list of users that are recipients of the given Group DM.""" recipients = await self.gdm_recipient_ids(channel_id) @@ -378,13 +409,17 @@ class Storage: if user_id == reference_id: continue - res.append( - await self.get_user(user_id) - ) + user = await self.get_user(user_id) + + if user is None: + continue + + res.append(user) return res - async def get_channel(self, channel_id: int, **kwargs) -> Dict[str, Any]: + async def get_channel(self, channel_id: int, + **kwargs) -> Optional[Dict[str, Any]]: """Fetch a single channel's information.""" chan_type = await self.get_chan_type(channel_id) ctype = ChannelType(chan_type) @@ -493,7 +528,7 @@ class Storage: return channels async def get_role(self, role_id: int, - guild_id: int = None) -> Dict[str, Any]: + guild_id: int = None) -> Optional[Dict[str, Any]]: """get a single role's information.""" guild_field = 'AND guild_id = $2' if guild_id else '' @@ -511,7 +546,7 @@ class Storage: """, *args) if not row: - return + return None return dict(row) @@ -527,6 +562,27 @@ class Storage: return list(map(dict, roledata)) + async def guild_voice_states(self, guild_id: int, + user_id=None) -> List[Dict[str, Any]]: + """Get a list of voice states for the given guild.""" + channel_ids = await self.get_channel_ids(guild_id) + + res = [] + + for channel_id in channel_ids: + states = await self.app.voice.fetch_states(channel_id) + + jsonified = [s.as_json_for(user_id) for s in states.values()] + + # discord does NOT insert guild_id to voice states on the + # guild voice state list. + for state in jsonified: + state.pop('guild_id') + + res.extend(jsonified) + + return res + async def get_guild_extra(self, guild_id: int, user_id=None, large=None) -> Dict: """Get extra information about a guild.""" @@ -567,18 +623,20 @@ class Storage: ), 'emojis': await self.get_guild_emojis(guild_id), - - # TODO: voice state management - 'voice_states': [], + 'voice_states': await self.guild_voice_states(guild_id), }} - async def get_guild_full(self, guild_id: int, - user_id: int, large_count: int = 250) -> Dict: + async def get_guild_full(self, guild_id: int, user_id: int, + large_count: int = 250) -> Optional[Dict]: """Get full information on a guild. This is a very expensive operation. """ guild = await self.get_guild(guild_id, user_id) + + if guild is None: + return None + extra = await self.get_guild_extra(guild_id, user_id, large_count) return {**guild, **extra} @@ -729,12 +787,13 @@ class Storage: return res - async def _inject_author(self, res): + async def _inject_author(self, res: dict): """Inject a pseudo-user object when the message is made by a webhook.""" author_id, webhook_id = res['author_id'], res['webhook_id'] if author_id is not None: res['author'] = await self.get_user(res['author_id']) + res.pop('webhook_id') elif webhook_id is not None: res['author'] = { 'id': webhook_id, @@ -744,7 +803,8 @@ class Storage: res.pop('author_id') - async def get_message(self, message_id: int, user_id=None) -> Dict: + async def get_message(self, message_id: int, + user_id: Optional[int] = None) -> Optional[Dict]: """Get a single message's payload.""" row = await self.fetchrow_with_json(""" SELECT id::text, channel_id::text, author_id, webhook_id, content, @@ -755,7 +815,7 @@ class Storage: """, message_id) if not row: - return + return None res = dict(row) res['nonce'] = str(res['nonce']) @@ -813,8 +873,12 @@ class Storage: res['attachments'] = await self.get_attachments(message_id) - # TODO: res['member'] for partial member data - # of the author + # if message is not from a dm, guild_id is None and so, _member_basic + # will just return None + res['member'] = await self._member_basic_with_roles(guild_id, user_id) + + if res['member'] is None: + res.pop('member') pin_id = await self.db.fetchval(""" SELECT message_id @@ -832,7 +896,7 @@ class Storage: return res - async def get_invite(self, invite_code: str) -> dict: + async def get_invite(self, invite_code: str) -> Optional[Dict]: """Fetch invite information given its code.""" invite = await self.db.fetchrow(""" SELECT code, guild_id, channel_id @@ -861,6 +925,10 @@ class Storage: dinv['guild'] = {} chan = await self.get_channel(invite['channel_id']) + + if chan is None: + return None + dinv['channel'] = { 'id': chan['id'], 'name': chan['name'], @@ -893,7 +961,8 @@ class Storage: 'approximate_member_count': len(mids), } - async def get_invite_metadata(self, invite_code: str) -> Dict[str, Any]: + async def get_invite_metadata(self, + invite_code: str) -> Optional[Dict[str, Any]]: """Fetch invite metadata (max_age and friends).""" invite = await self.db.fetchrow(""" SELECT code, inviter, created_at, uses, @@ -903,7 +972,7 @@ class Storage: """, invite_code) if invite is None: - return + return None dinv = dict_(invite) inviter = await self.get_user(invite['inviter']) @@ -911,17 +980,18 @@ class Storage: return dinv - async def get_dm(self, dm_id: int, user_id: int = None): + async def get_dm(self, dm_id: int, user_id: int = None) -> Optional[Dict]: + """Get a DM channel.""" dm_chan = await self.get_channel(dm_id) - if user_id: + if user_id and dm_chan: dm_chan['recipients'] = _filter_recipients( - dm_chan['recipients'], user_id + dm_chan['recipients'], str(user_id) ) return dm_chan - async def guild_from_channel(self, channel_id: int): + async def guild_from_channel(self, channel_id: int) -> int: """Get the guild id coming from a channel id.""" return await self.db.fetchval(""" SELECT guild_id @@ -944,7 +1014,7 @@ class Storage: return parties[0] - async def get_emoji(self, emoji_id: int) -> Dict: + async def get_emoji(self, emoji_id: int) -> Optional[Dict[str, Any]]: """Get a single emoji.""" row = await self.db.fetchrow(""" SELECT id::text, name, animated, managed, @@ -954,7 +1024,7 @@ class Storage: """, emoji_id) if not row: - return + return None drow = dict(row) @@ -993,3 +1063,12 @@ class Storage: """, role_id) return [r['id'] for r in rows] + + async def all_voice_regions(self) -> List[Dict[str, Any]]: + """Return a list of all voice regions.""" + rows = await self.db.fetch(""" + SELECT id, name, vip, deprecated, custom + FROM voice_regions + """) + + return list(map(dict, rows)) diff --git a/litecord/utils.py b/litecord/utils.py index 5647a37..513fafa 100644 --- a/litecord/utils.py +++ b/litecord/utils.py @@ -19,14 +19,15 @@ along with this program. If not, see . import asyncio import json +from typing import Any, Iterable, Optional, Sequence + from logbook import Logger -from typing import Any from quart.json import JSONEncoder log = Logger(__name__) -async def async_map(function, iterable) -> list: +async def async_map(function, iterable: Iterable) -> list: """Map a coroutine to an iterable.""" res = [] @@ -51,7 +52,7 @@ def dict_get(mapping, key, default): return mapping.get(key) or default -def index_by_func(function, indexable: iter) -> int: +def index_by_func(function, indexable: Sequence[Any]) -> Optional[int]: """Search in an idexable and return the index number for an iterm that has func(item) = True.""" for index, item in enumerate(indexable): @@ -66,7 +67,7 @@ def _u(val): return val % 0x100000000 -def mmh3(key: str, seed: int = 0): +def mmh3(inp_str: str, seed: int = 0): """MurMurHash3 implementation. This seems to match Discord's JavaScript implementaiton. @@ -74,7 +75,7 @@ def mmh3(key: str, seed: int = 0): Based off https://github.com/garycourt/murmurhash-js/blob/master/murmurhash3_gc.js """ - key = [ord(c) for c in key] + key = [ord(c) for c in inp_str] remainder = len(key) & 3 bytecount = len(key) - remainder @@ -160,7 +161,7 @@ async def pg_set_json(con): ) -def yield_chunks(input_list: list, chunk_size: int): +def yield_chunks(input_list: Sequence[Any], chunk_size: int): """Yield successive n-sized chunks from l. Taken from https://stackoverflow.com/a/312464. diff --git a/litecord/voice/lvsp_conn.py b/litecord/voice/lvsp_conn.py new file mode 100644 index 0000000..3902674 --- /dev/null +++ b/litecord/voice/lvsp_conn.py @@ -0,0 +1,200 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +import json +import asyncio +from typing import Dict + +import websockets +from logbook import Logger + +from litecord.voice.lvsp_opcodes import OPCodes as OP, InfoTable, InfoReverse + +log = Logger(__name__) + + +class LVSPConnection: + """Represents a single LVSP connection.""" + def __init__(self, lvsp, region: str, hostname: str): + self.lvsp = lvsp + self.app = lvsp.app + + self.region = region + self.hostname = hostname + + self.conn = None + self.health = 0.5 + + self._hb_task = None + self._hb_interval = None + + @property + def _log_id(self): + return f'region={self.region} hostname={self.hostname}' + + async def send(self, payload): + """Send a payload down the websocket.""" + msg = json.dumps(payload) + await self.conn.send(msg) + + async def recv(self): + """Receive a payload.""" + msg = await self.conn.recv() + msg = json.dumps(msg) + return msg + + async def send_op(self, opcode: int, data: dict): + """Send a message with an OP code included""" + await self.send({ + 'op': opcode, + 'd': data + }) + + async def send_info(self, info_type: str, info_data: Dict): + """Send an INFO message down the websocket.""" + await self.send({ + 'op': OP.info, + 'd': { + 'type': InfoTable[info_type.upper()], + 'data': info_data + } + }) + + async def _heartbeater(self, hb_interval: int): + try: + await asyncio.sleep(hb_interval) + + # TODO: add self._seq + await self.send_op(OP.heartbeat, { + 's': 0 + }) + + # give the server 300 milliseconds to reply. + await asyncio.sleep(300) + await self.conn.close(4000, 'heartbeat timeout') + except asyncio.CancelledError: + pass + + def _start_hb(self): + self._hb_task = self.app.loop.create_task( + self._heartbeater(self._hb_interval) + ) + + def _stop_hb(self): + self._hb_task.cancel() + + async def _handle_0(self, msg): + """Handle HELLO message.""" + data = msg['d'] + + # nonce = data['nonce'] + self._hb_interval = data['heartbeat_interval'] + + # TODO: send identify + + async def _update_health(self, new_health: float): + """Update the health value of a given voice server.""" + self.health = new_health + + await self.app.db.execute(""" + UPDATE voice_servers + SET health = $1 + WHERE hostname = $2 + """, new_health, self.hostname) + + async def _handle_3(self, msg): + """Handle READY message. + + We only start heartbeating after READY. + """ + await self._update_health(msg['health']) + self._start_hb() + + async def _handle_5(self, msg): + """Handle HEARTBEAT_ACK.""" + self._stop_hb() + await self._update_health(msg['health']) + self._start_hb() + + async def _handle_6(self, msg): + """Handle INFO messages.""" + info = msg['d'] + info_type_str = InfoReverse[info['type']].lower() + + try: + info_handler = getattr(self, f'_handle_info_{info_type_str}') + except AttributeError: + return + + await info_handler(info['data']) + + async def _handle_info_channel_assign(self, data: dict): + """called by the server once we got a channel assign.""" + try: + channel_id = data['channel_id'] + channel_id = int(channel_id) + except (TypeError, ValueError): + return + + try: + guild_id = data['guild_id'] + guild_id = int(guild_id) + except (TypeError, ValueError): + guild_id = None + + main_key = guild_id if guild_id is not None else channel_id + await self.lvsp.assign(main_key, self.hostname) + + async def _loop(self): + while True: + msg = await self.recv() + + try: + opcode = msg['op'] + handler = getattr(self, f'_handle_{opcode}') + await handler(msg) + except (KeyError, AttributeError): + # TODO: error codes in LVSP + raise Exception('invalid op code') + + async def start(self): + """Try to start a websocket connection.""" + try: + self.conn = await websockets.connect(f'wss://{self.hostname}') + except Exception: + log.exception('failed to start lvsp conn to {}', self.hostname) + + async def run(self): + """Start the websocket.""" + await self.start() + + try: + if not self.conn: + log.error('failed to start lvsp connection, stopping') + return + + await self._loop() + except websockets.exceptions.ConnectionClosed as err: + log.warning('conn close, {}, err={}', self._log_id, err) + # except WebsocketClose as err: + # log.warning('ws close, state={} err={}', self.state, err) + # await self.conn.close(code=err.code, reason=err.reason) + except Exception as err: + log.exception('An exception has occoured. {}', self._log_id) + await self.conn.close(code=4000, reason=repr(err)) diff --git a/litecord/voice/lvsp_manager.py b/litecord/voice/lvsp_manager.py new file mode 100644 index 0000000..78484e0 --- /dev/null +++ b/litecord/voice/lvsp_manager.py @@ -0,0 +1,146 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +from typing import Optional +from collections import defaultdict + +from logbook import Logger + +from litecord.voice.lvsp_conn import LVSPConnection + +log = Logger(__name__) + +class LVSPManager: + """Manager class for Litecord Voice Server Protocol (LVSP) connections. + + Spawns :class:`LVSPConnection` as needed, etc. + """ + def __init__(self, app, voice): + self.app = app + self.voice = voice + + # map servers to LVSPConnection + self.conns = {} + + # maps regions to server hostnames + self.servers = defaultdict(list) + + # maps Union[GuildID, DMId, GroupDMId] to server hostnames + self.assign = {} + + self.app.loop.create_task(self._spawn()) + + async def _spawn(self): + """Spawn LVSPConnection for each region.""" + + regions = await self.app.db.fetch(""" + SELECT id + FROM voice_regions + WHERE deprecated = false + """) + + regions = [r['id'] for r in regions] + + if not regions: + log.warning('no regions are setup') + return + + for region in regions: + self.app.loop.create_task( + self._spawn_region(region) + ) + + async def _spawn_region(self, region: str): + """Spawn a region. Involves fetching all the hostnames + for the regions and spawning a LVSPConnection for each.""" + servers = await self.app.db.fetch(""" + SELECT hostname + FROM voice_servers + WHERE region_id = $1 + """, region) + + if not servers: + log.warning('region {} does not have servers', region) + return + + servers = [r['hostname'] for r in servers] + self.servers[region] = servers + + for hostname in servers: + conn = LVSPConnection(self, region, hostname) + self.conns[hostname] = conn + + self.app.loop.create_task( + conn.run() + ) + + async def del_conn(self, conn): + """Delete a connection from the connection pool.""" + try: + self.servers[conn.region].remove(conn.hostname) + except KeyError: + pass + + try: + self.conns.pop(conn.hostname) + except KeyError: + pass + + async def guild_region(self, guild_id: int) -> Optional[str]: + """Return the voice region of a guild.""" + return await self.app.db.fetchval(""" + SELECT region + FROM guilds + WHERE id = $1 + """, guild_id) + + def get_health(self, hostname: str) -> float: + """Get voice server health, given hostname.""" + try: + conn = self.conns[hostname] + except KeyError: + return -1 + + return conn.health + + async def get_guild_server(self, guild_id: int) -> Optional[str]: + """Get a voice server for the given guild, assigns + one if there isn't any""" + + try: + hostname = self.assign[guild_id] + except KeyError: + region = await self.guild_region(guild_id) + + # sort connected servers by health + sorted_servers = sorted( + self.servers[region], + key=self.get_health + ) + + try: + hostname = sorted_servers[0] + except IndexError: + return None + + return hostname + + async def assign_conn(self, key: int, hostname: str): + """Assign a connection to a given key in the assign map""" + self.assign[key] = hostname diff --git a/litecord/voice/lvsp_opcodes.py b/litecord/voice/lvsp_opcodes.py new file mode 100644 index 0000000..4cd0b3c --- /dev/null +++ b/litecord/voice/lvsp_opcodes.py @@ -0,0 +1,41 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +class OPCodes: + """LVSP OP codes.""" + hello = 0 + identify = 1 + resume = 2 + ready = 3 + heartbeat = 4 + heartbeat_ack = 5 + info = 6 + + +InfoTable = { + 'CHANNEL_REQ': 0, + 'CHANNEL_ASSIGN': 1, + 'CHANNEL_UPDATE': 2, + 'CHANNEL_DESTROY': 3, + 'VST_CREATE': 4, + 'VST_UPDATE': 5, + 'VST_LEAVE': 6, +} + +InfoReverse = {v: k for k, v in InfoTable.items()} diff --git a/litecord/voice/manager.py b/litecord/voice/manager.py new file mode 100644 index 0000000..b927cd3 --- /dev/null +++ b/litecord/voice/manager.py @@ -0,0 +1,258 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +from typing import Tuple, Dict, List +from collections import defaultdict +from dataclasses import fields + +from logbook import Logger + +from litecord.permissions import get_permissions +from litecord.enums import ChannelType, VOICE_CHANNELS +from litecord.voice.state import VoiceState +from litecord.voice.lvsp_manager import LVSPManager + + +VoiceKey = Tuple[int, int] +log = Logger(__name__) + + +def _construct_state(state_dict: dict) -> VoiceState: + """Create a VoiceState instance out of a dictionary with the + VoiceState fields as keys.""" + state_fields = fields(VoiceState) + args = [state_dict[field.name] for field in state_fields] + return VoiceState(*args) + + +class VoiceManager: + """Main voice manager class.""" + def __init__(self, app): + self.app = app + + # double dict, first key is guild/channel id, second key is user id + self.states = defaultdict(dict) + self.lvsp = LVSPManager(app, self) + + # TODO: map channel ids to voice servers + + async def can_join(self, user_id: int, channel_id: int) -> int: + """Return if a user can join a channel.""" + + channel = await self.app.storage.get_channel(channel_id) + ctype = ChannelType(channel['type']) + + if ctype not in VOICE_CHANNELS: + return + + states = await self.app.voice.state_count(channel_id) + + # get_permissions returns ALL_PERMISSIONS when + # the channel isn't from a guild + perms = await get_permissions( + user_id, channel_id, storage=self.app.storage + ) + + # hacky user_limit but should work, as channels not + # in guilds won't have that field. + is_full = states >= channel.get('user_limit', 100) + is_bot = (await self.app.storage.get_user(user_id))['bot'] + is_manager = perms.bits.manage_channels + + # if the channel is full AND: + # - user is not a bot + # - user is not manage channels + # then it fails + if not is_bot and not is_manager and is_full: + return + + # all good + return True + + async def state_count(self, channel_id: int) -> int: + """Get the current amount of voice states in a channel.""" + return len(self.states[channel_id]) + + async def fetch_states(self, channel_id: int) -> Dict[int, VoiceState]: + """Fetch the states of the given channel.""" + # since the state key is (user_id, guild_id | channel_id), we need + # to determine which kind of search we want to do. + guild_id = await self.app.storage.guild_from_channel(channel_id) + + # if there isn't a guild for the channel, it is a dm or group dm. + # those are simple to handle. + if not guild_id: + return dict(self.states[channel_id]) + + # guild states hold a dict mapping user ids to guild states, + # same as channels, thats the structure. + guild_states = self.states[guild_id] + res = {} + + # iterate over all users with states and add the channel matches + # into res + for user_id, state in guild_states.items(): + if state.channel_id == channel_id: + res[user_id] = state + + return res + + async def get_state(self, voice_key: VoiceKey) -> VoiceState: + """Get a single VoiceState for a user in a channel. Returns None + if no VoiceState is found.""" + user_id, sec_key_id = voice_key + + try: + return self.states[sec_key_id][user_id] + except KeyError: + return None + + async def del_state(self, voice_key: VoiceKey): + """Delete a given voice state.""" + user_id, sec_key_id = voice_key + + try: + # TODO: tell that to the voice server of the channel. + self.states[sec_key_id].pop(user_id) + except KeyError: + pass + + async def update_state(self, state: VoiceState, prop: dict): + """Update a state in a channel""" + # construct a new state based on the old one + properties + new_state_dict = dict(state.as_json) + + for field in prop: + # NOTE: this should not happen, ever. + if field in ('channel_id', 'user_id'): + raise ValueError('properties are updating channel or user') + + new_state_dict[field] = prop[field] + + new_state = _construct_state(new_state_dict) + + # TODO: dispatch to voice server + self.states[state.key][state.user_id] = new_state + + async def move_channels(self, old_voice_key: VoiceKey, channel_id: int): + """Move a user between channels.""" + await self.del_state(old_voice_key) + await self.create_state(old_voice_key, {'channel_id': channel_id}) + + async def _lvsp_info_guild(self, guild_id, info_type, info_data): + hostname = await self.lvsp.get_guild_server(guild_id) + if hostname is None: + log.error('no voice server for guild id {}', guild_id) + return + + conn = self.lvsp.get_conn(hostname) + await conn.send_info(info_type, info_data) + + async def _create_ctx_guild(self, guild_id, channel_id): + await self._lvsp_info_guild(guild_id, 'CHANNEL_REQ', { + 'guild_id': str(guild_id), + 'channel_id': str(channel_id), + }) + + async def _start_voice_guild(self, voice_key: VoiceKey, data: dict): + """Start a voice context in a guild.""" + user_id, guild_id = voice_key + channel_id = int(data['channel_id']) + + existing_states = self.states[voice_key] + channel_exists = any( + state.channel_id == channel_id for state in existing_states + ) + + if not channel_exists: + await self._create_ctx_guild(guild_id, channel_id) + + await self._lvsp_info_guild(guild_id, 'VST_CREATE', { + 'user_id': str(user_id), + 'guild_id': str(guild_id), + 'channel_id': str(channel_id), + }) + + async def create_state(self, voice_key: VoiceKey, data: dict): + """Creates (or tries to create) a voice state. + + Depending on the VoiceKey given, it will use the guild's voice + region or assign one based on the starter of a call, or the owner of + a Group DM. + + Once a region is assigned, it'll choose the best voice server + and send a request to it. + """ + + # TODO: handle CALL events. + + # compare if this voice key is for a guild or a channel + _uid, id2 = voice_key + guild = await self.app.storage.get_guild(id2) + + # if guild not found, then we are dealing with a dm or group dm + if not guild: + ctype = await self.app.storage.get_chan_type(id2) + ctype = ChannelType(ctype) + + if ctype == ChannelType.GROUP_DM: + # await self._start_voice_dm(voice_key) + pass + elif ctype == ChannelType.DM: + # await self._start_voice_gdm(voice_key) + pass + + return + + # if guild found, then data.channel_id exists, and we treat it + # as a guild + await self._start_voice_guild(voice_key, data) + + async def leave_all(self, user_id: int) -> int: + """Leave all voice channels.""" + + # iterate over every state finding matches + + # NOTE: we copy the current states dict since we're modifying + # on iteration. this is SLOW. + + # TODO: better solution instead of copying, maybe we can generate + # a list of tasks to run that actually do the deletion by themselves + # instead of us generating a delete. then only start running them later + # on. + for sec_key_id, states in dict(self.states).items(): + for state in states: + if state.user_id != user_id: + continue + + await self.del_state((user_id, sec_key_id)) + + async def leave(self, guild_id: int, user_id: int): + """Make a user leave a channel IN A GUILD.""" + await self.del_state((guild_id, user_id)) + + async def voice_server_list(self, region: str) -> List[dict]: + """Get a list of voice server objects""" + rows = await self.app.db.fetch(""" + SELECT hostname, last_health + FROM voice_servers + WHERE region_id = $1 + """, region) + + return list(map(dict, rows)) diff --git a/litecord/voice/state.py b/litecord/voice/state.py new file mode 100644 index 0000000..d5e8732 --- /dev/null +++ b/litecord/voice/state.py @@ -0,0 +1,61 @@ +""" + +Litecord +Copyright (C) 2018-2019 Luna Mendes + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +""" + +from dataclasses import dataclass, asdict + + +@dataclass +class VoiceState: + """Represents a voice state.""" + guild_id: int + channel_id: int + user_id: int + session_id: str + deaf: bool + mute: bool + self_deaf: bool + self_mute: bool + suppressed_by: int + + @property + def key(self): + """Get the second part of a key identifying a state.""" + return self.channel_id if self.guild_id is None else self.guild_id + + @property + def as_json(self): + """Return JSON-serializable dict.""" + return asdict(self) + + def as_json_for(self, user_id: int): + """Generate JSON-serializable version, given a user ID.""" + self_dict = asdict(self) + + if user_id is None: + return self_dict + + # state.suppress is defined by the user + # that is currently viewing the state. + + # a better approach would be actually using + # the suppressed_by field for backend efficiency. + self_dict['suppress'] = user_id == self.suppressed_by + self_dict.pop('suppressed_by') + + return self_dict diff --git a/manage/cmd/migration/scripts/11_voice_regions_servers.sql b/manage/cmd/migration/scripts/11_voice_regions_servers.sql new file mode 100644 index 0000000..398c91a --- /dev/null +++ b/manage/cmd/migration/scripts/11_voice_regions_servers.sql @@ -0,0 +1,37 @@ +-- voice region data +-- NOTE: do NOT remove any rows. use deprectated=true and +-- DELETE FROM voice_servers instead. +CREATE TABLE IF NOT EXISTS voice_regions ( + -- always lowercase + id text PRIMARY KEY, + + -- "Russia", "Brazil", "Antartica", etc + name text NOT NULL, + + -- we don't have the concept of vip guilds yet, but better + -- future proof. + vip boolean DEFAULT FALSE, + + deprecated boolean DEFAULT FALSE, + + -- we don't have the concept of custom regions too. we don't have the + -- concept of official guilds either, but i'm keeping this in + custom boolean DEFAULT FALSE +); + +-- voice server pool. when someone wants to connect to voice, we choose +-- a server that is in the same region the guild is too, and choose the one +-- with the best health value +CREATE TABLE IF NOT EXISTS voice_servers ( + -- hostname is a reachable url, e.g "brazil2.example.com" + hostname text PRIMARY KEY, + region_id text REFERENCES voice_regions (id), + + -- health values are more thoroughly defined in the LVSP documentation + last_health float default 0.5 +); + + +ALTER TABLE guilds DROP COLUMN IF EXISTS region; +ALTER TABLE guilds ADD COLUMN + region text REFERENCES voice_regions (id); diff --git a/run.py b/run.py index ddebbe8..398896d 100644 --- a/run.py +++ b/run.py @@ -24,7 +24,7 @@ import asyncpg import logbook import logging import websockets -from quart import Quart, g, jsonify, request +from quart import Quart, jsonify, request from logbook import StreamHandler, Logger from logbook.compat import redirect_logging from aiohttp import ClientSession @@ -54,14 +54,17 @@ from litecord.blueprints.user import ( user_settings, user_billing, fake_store ) -from litecord.blueprints.user.billing_job import ( - payment_job +from litecord.blueprints.user.billing_job import payment_job + +from litecord.blueprints.admin_api import ( + voice as voice_admin ) +from litecord.blueprints.admin_api.voice import guild_region_check + from litecord.ratelimits.handler import ratelimit_handler from litecord.ratelimits.main import RatelimitManager -from litecord.gateway import websocket_handler from litecord.errors import LitecordError from litecord.gateway.state_manager import StateManager from litecord.storage import Storage @@ -70,6 +73,9 @@ from litecord.dispatcher import EventDispatcher from litecord.presence import PresenceManager from litecord.images import IconManager from litecord.jobs import JobManager +from litecord.voice.manager import VoiceManager + +from litecord.gateway.gateway import websocket_handler from litecord.utils import LitecordJSONEncoder @@ -135,7 +141,9 @@ def set_blueprints(app_): icons: -1, attachments: -1, nodeinfo: -1, - static: -1 + static: -1, + + voice_admin: '/admin/voice' } for bp, suffix in bps.items(): @@ -232,6 +240,8 @@ def init_app_managers(app_): app_.storage.presence = app_.presence + app_.voice = VoiceManager(app_) + async def api_index(app_): to_find = {} @@ -290,33 +300,43 @@ async def post_app_start(app_): # we'll need to start a billing job app_.sched.spawn(payment_job(app_)) app_.sched.spawn(api_index(app_)) + app_.sched.spawn(guild_region_check(app_)) -@app.before_serving -async def app_before_serving(): - log.info('opening db') - await init_app_db(app) - - g.app = app - g.loop = asyncio.get_event_loop() - - app.session = ClientSession() - - init_app_managers(app) - - # start the websocket, etc - host, port = app.config['WS_HOST'], app.config['WS_PORT'] +def start_websocket(host, port, ws_handler) -> asyncio.Future: + """Start a websocket. Returns the websocket future""" log.info(f'starting websocket at {host} {port}') async def _wrapper(ws, url): # We wrap the main websocket_handler # so we can pass quart's app object. - await websocket_handler(app, ws, url) + await ws_handler(app, ws, url) - ws_future = websockets.serve(_wrapper, host, port) + return websockets.serve(_wrapper, host, port) + +@app.before_serving +async def app_before_serving(): + """Callback for variable setup. + + Also sets up the websocket handlers. + """ + log.info('opening db') + await init_app_db(app) + + app.session = ClientSession() + + init_app_managers(app) await post_app_start(app) - await ws_future + + # start gateway websocket + # voice websocket is handled by the voice server + ws_fut = start_websocket( + app.config['WS_HOST'], app.config['WS_PORT'], + websocket_handler + ) + + await ws_fut @app.after_serving diff --git a/schema.sql b/schema.sql index 5eab831..269ff91 100644 --- a/schema.sql +++ b/schema.sql @@ -318,6 +318,40 @@ CREATE TABLE IF NOT EXISTS user_read_state ( PRIMARY KEY (user_id, channel_id) ); + +-- voice region data +-- NOTE: do NOT remove any rows. use deprectated=true and +-- DELETE FROM voice_servers instead. +CREATE TABLE IF NOT EXISTS voice_regions ( + -- always lowercase + id text PRIMARY KEY, + + -- "Russia", "Brazil", "Antartica", etc + name text NOT NULL, + + -- we don't have the concept of vip guilds yet, but better + -- future proof. + vip boolean DEFAULT FALSE, + + deprecated boolean DEFAULT FALSE, + + -- we don't have the concept of custom regions too. we don't have the + -- concept of official guilds either, but i'm keeping this in + custom boolean DEFAULT FALSE +); + +-- voice server pool. when someone wants to connect to voice, we choose +-- a server that is in the same region the guild is too, and choose the one +-- with the best health value +CREATE TABLE IF NOT EXISTS voice_servers ( + -- hostname is a reachable url, e.g "brazil2.example.com" + hostname text PRIMARY KEY, + region_id text REFERENCES voice_regions (id), + + -- health values are more thoroughly defined in the LVSP documentation + last_health float default 0.5 +); + CREATE TABLE IF NOT EXISTS guilds ( id bigint PRIMARY KEY NOT NULL, @@ -326,7 +360,7 @@ CREATE TABLE IF NOT EXISTS guilds ( splash text DEFAULT NULL, owner_id bigint NOT NULL REFERENCES users (id), - region text NOT NULL, + region text NOT NULL REFERENCES voice_regions (id), /* default no afk channel afk channel is voice-only. diff --git a/tests/conftest.py b/tests/conftest.py index d77c012..66c5e6f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,7 @@ import asyncio import sys import os +import socket import pytest # this is very hacky. @@ -36,8 +37,10 @@ def _test_app(unused_tcp_port, event_loop): # reassign an unused tcp port for websockets # since the config might give a used one. - main_app.config['WS_PORT'] = unused_tcp_port - main_app.config['WEBSOCKET_URL'] = f'localhost:{unused_tcp_port}' + ws_port = unused_tcp_port + + main_app.config['WS_PORT'] = ws_port + main_app.config['WEBSOCKET_URL'] = f'localhost:{ws_port}' # make sure we're calling the before_serving hooks event_loop.run_until_complete(main_app.startup())