diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..7ccd52e --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +omit = + */app.py + */wsgi.py + */config.py + docs/* + */tests* diff --git a/Pipfile b/Pipfile index cbd6ffe..363c1f7 100644 --- a/Pipfile +++ b/Pipfile @@ -8,10 +8,15 @@ arxiv-base = "*" flask = "*" requests = "*" arxiv-auth = "==0.1.0rc6" +"boto3" = "*" +moto = "==1.3.4" [dev-packages] jsonschema = "*" openapi-spec-validator = "*" +"nose2" = "*" +coveralls = "*" +coverage = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index e898086..6806499 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f39eedc08e1a9d7bdff19435886555898d232ccc48b1766ab9847e8e243bb8af" + "sha256": "3cb557e6d2b5a9d9567880e8f98fd1b4fc765a6a3d54004e165d13592cd9ac33" }, "pipfile-spec": 6, "requires": { @@ -30,19 +30,41 @@ "index": "pypi", "version": "==0.10.1" }, + "asn1crypto": { + "hashes": [ + "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", + "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" + ], + "version": "==0.24.0" + }, + "aws-xray-sdk": { + "hashes": [ + "sha256:72791618feb22eaff2e628462b0d58f398ce8c1bacfa989b7679817ab1fad60c", + "sha256:9e7ba8dd08fd2939376c21423376206bff01d0deaea7d7721c6b35921fed1943" + ], + "version": "==0.95" + }, + "boto": { + "hashes": [ + "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8", + "sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a" + ], + "version": "==2.49.0" + }, "boto3": { "hashes": [ - "sha256:2f3592f19847c6da02662e7a688a0d26731a36b06357c87da96257712a533dcf", - "sha256:f2be145131cdc0837fb759f04156895f86eb21e26318158c611fcb8e9b6734c5" + "sha256:3937eaba7cc429d1593824397f1d7dda4c9565c1269e8d3f665cee3c1b1b955c", + "sha256:ae1bc72a3921807ef8c0daa6c4bd147e6caa630df263e5ea5e8c36535e17ee73" ], - "version": "==1.8.8" + "index": "pypi", + "version": "==1.8.9" }, "botocore": { "hashes": [ - "sha256:c05733dd4fed3c79283fb77f4e9c8d5a65571c45e7eec7a6158bfed54446c3ac", - "sha256:e3c9ba538e3e60ac414a84e0bcc7ec3fb64716b6004cbf99a68aad39b7d67e08" + "sha256:1294d1422b278dac28869d5b96156171dd8d5b609da28fec006add8a81a31dfa", + "sha256:91c32b97dd2f1046b1de3ce5c8bc19cf4993bc631864c9fcbf51f261bc3a7aab" ], - "version": "==1.11.8" + "version": "==1.11.9" }, "certifi": { "hashes": [ @@ -51,6 +73,43 @@ ], "version": "==2018.8.24" }, + "cffi": { + "hashes": [ + "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", + "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", + "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", + "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", + "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", + "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", + "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", + "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", + "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", + "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", + "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", + "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", + "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", + "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", + "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", + "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", + "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", + "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", + "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", + "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", + "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", + "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", + "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", + "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", + "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", + "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", + "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", + "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", + "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", + "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", + "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", + "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + ], + "version": "==1.11.5" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -65,6 +124,51 @@ ], "version": "==6.7" }, + "cookies": { + "hashes": [ + "sha256:15bee753002dff684987b8df8c235288eb8d45f8191ae056254812dfd42c81d3", + "sha256:d6b698788cae4cfa4e62ef8643a9ca332b79bd96cb314294b864ae8d7eb3ee8e" + ], + "version": "==2.2.1" + }, + "cryptography": { + "hashes": [ + "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", + "sha256:10b48e848e1edb93c1d3b797c83c72b4c387ab0eb4330aaa26da8049a6cbede0", + "sha256:17db09db9d7c5de130023657be42689d1a5f60502a14f6f745f6f65a6b8195c0", + "sha256:227da3a896df1106b1a69b1e319dce218fa04395e8cc78be7e31ca94c21254bc", + "sha256:2cbaa03ac677db6c821dac3f4cdfd1461a32d0615847eedbb0df54bb7802e1f7", + "sha256:31db8febfc768e4b4bd826750a70c79c99ea423f4697d1dab764eb9f9f849519", + "sha256:4a510d268e55e2e067715d728e4ca6cd26a8e9f1f3d174faf88e6f2cb6b6c395", + "sha256:6a88d9004310a198c474d8a822ee96a6dd6c01efe66facdf17cb692512ae5bc0", + "sha256:76936ec70a9b72eb8c58314c38c55a0336a2b36de0c7ee8fb874a4547cadbd39", + "sha256:7e3b4aecc4040928efa8a7cdaf074e868af32c58ffc9bb77e7bf2c1a16783286", + "sha256:8168bcb08403ef144ff1fb880d416f49e2728101d02aaadfe9645883222c0aa5", + "sha256:8229ceb79a1792823d87779959184a1bf95768e9248c93ae9f97c7a2f60376a1", + "sha256:8a19e9f2fe69f6a44a5c156968d9fc8df56d09798d0c6a34ccc373bb186cee86", + "sha256:8d10113ca826a4c29d5b85b2c4e045ffa8bad74fb525ee0eceb1d38d4c70dfd6", + "sha256:be495b8ec5a939a7605274b6e59fbc35e76f5ad814ae010eb679529671c9e119", + "sha256:dc2d3f3b1548f4d11786616cf0f4415e25b0fbecb8a1d2cd8c07568f13fdde38", + "sha256:e4aecdd9d5a3d06c337894c9a6e2961898d3f64fe54ca920a72234a3de0f9cb3", + "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9", + "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f" + ], + "version": "==2.3.1" + }, + "docker": { + "hashes": [ + "sha256:6c4da20ef40e8d3eaf650f1488d91452b9a1128045481d7169fd34665ffa90ee", + "sha256:bc693be5a84b3b9e5aaf156068c5c0a445ee5138c638c3fbc857133bf67ebe07" + ], + "version": "==3.5.0" + }, + "docker-pycreds": { + "hashes": [ + "sha256:0a941b290764ea7286bd77f54c0ace43b86a8acd6eb9ead3de9840af52384079", + "sha256:8b0e956c8d206f832b06aa93a710ba2c3bcbacb5a314449c040b0b814355bbff" + ], + "version": "==0.3.0" + }, "docutils": { "hashes": [ "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", @@ -73,6 +177,13 @@ ], "version": "==0.14" }, + "ecdsa": { + "hashes": [ + "sha256:40d002cf360d0e035cf2cb985e1308d41aaa087cbfc135b2dc2d844296ea546c", + "sha256:64cf1ee26d1cde3c73c6d7d107f835fed7c6a2904aef9eac223d57ad800c43fa" + ], + "version": "==0.13" + }, "flask": { "hashes": [ "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", @@ -81,6 +192,12 @@ "index": "pypi", "version": "==1.0.2" }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", @@ -108,6 +225,18 @@ ], "version": "==0.9.3" }, + "jsondiff": { + "hashes": [ + "sha256:2d0437782de9418efa34e694aa59f43d7adb1899bd9a793f063867ddba8f7893" + ], + "version": "==1.1.1" + }, + "jsonpickle": { + "hashes": [ + "sha256:545b3bee0d65e1abb4baa1818edcc9ec239aa9f2ffbfde8084d71c056180054f" + ], + "version": "==0.9.6" + }, "jsonschema": { "hashes": [ "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", @@ -121,12 +250,41 @@ ], "version": "==1.0" }, + "mock": { + "hashes": [ + "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", + "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" + ], + "version": "==2.0.0" + }, + "moto": { + "hashes": [ + "sha256:7c86d1c3bd6362954afaded735354c11afd22037eb6736152f057a1bff0c8868", + "sha256:b8556c1e0cebf931a698bf7198bb3eaf2287c8c9bb4f3455ea5d2015ad8f1708" + ], + "index": "pypi", + "version": "==1.3.4" + }, "mysqlclient": { "hashes": [ "sha256:ff8ee1be84215e6c30a746b728c41eb0701a46ca76e343af445b35ce6250644f" ], "version": "==1.3.13" }, + "pbr": { + "hashes": [ + "sha256:1b8be50d938c9bb75d0eaf7eda111eec1bf6dc88a62a6412e33bf077457e0f45", + "sha256:b486975c0cafb6beeb50ca0e17ba047647f229087bd74e37f4a7e2cac17d2caa" + ], + "version": "==4.2.0" + }, + "pyaml": { + "hashes": [ + "sha256:66623c52f34d83a2c0fc963e08e8b9d0c13d88404e3b43b1852ef71eda19afa3", + "sha256:f83fc302c52c6b83a15345792693ae0b5bc07ad19f59e318b7617d7123d62990" + ], + "version": "==17.12.1" + }, "pycountry": { "hashes": [ "sha256:195d8174fd6f98f45622d2885826091fc8b2168184422dac574311073ab8386c", @@ -135,6 +293,47 @@ ], "version": "==18.5.26" }, + "pycparser": { + "hashes": [ + "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" + ], + "version": "==2.18" + }, + "pycryptodome": { + "hashes": [ + "sha256:0f027d5da3f3c4c0167f3ccf4a1f56674248120656099df35098dfaf3edff0fb", + "sha256:1e970715407a862b6b4c61f1a8c60734c0fa39f45d36dda46dbd0baf2d8caec2", + "sha256:2652a86850d7873249c64365a61e1052934f1504f11b57dbe76c1a4dc9b5d593", + "sha256:26953969934e09d49b2e370229ef262dd480b46130660086b22fea29df335dea", + "sha256:31d2f9fa32fd651694dbae298682c1afa2337fa6454b32b241164c2ffe96e1c1", + "sha256:53cd63d379224ea52d8ba2012fe8acf9eb682aa819c3a9a02397fe3e6b4315a4", + "sha256:637bfc8bfb68d477619c54b56d912117abca05306a222ccf03dfc09ab6b4e5a4", + "sha256:6b8a3753e31b058d48bdd26c50c049a04f35f0f05c0d866c63fc90fc9b8dc5d7", + "sha256:7bd1c4671b3a2c8d647731e9c34115efa928ff12d0ef1bef68f0f7af984bc239", + "sha256:7cb057b700d688bb37082b0086d061462dde18c1fbbe355615db87f3bf97ffa4", + "sha256:7d7e07e885cee42b222ab190ea292f144aaf6e915ea3d1bf9e2f812fc2ad9f18", + "sha256:80c55dd2246a17b4af18bd615711b90c8df4b780451692f627a38a636d0792ae", + "sha256:864249afa1d801c7a2abb3fbed0e9e8ce4844a8f68daff8028a40634f69f0135", + "sha256:927ce443c5183ee7738ce113ecf656842fafbad1d6f4ab726dc12abc8adabfad", + "sha256:9dd8fb9d76fde52c01dcc6d24dc384ddb60ff6fb96216a58017dacc5580600e3", + "sha256:abd859f70a9cad653644b0415adfe6223f708093296747970ec56a8f5537bfb3", + "sha256:b3cb4af317d9b84f6df50f0cfa6840ba69556af637a83fd971537823e13d601a", + "sha256:b4c5d98eb9608bf29b66504dba96494a9fac75b3c0c57dfb557f6e812989a3fe", + "sha256:bc130342d9b6267efaa97ea305b9a46f59c097e95263a6d65abc7890331535e7", + "sha256:c26f706a8b8e1e44076126bfe0319b7eb9038350d5b6ff55c86b2edb434b3e52", + "sha256:c58539996e2acdb6c5554851cd1b333af889a6aadeb9865127e4bbc17d01ff53", + "sha256:c899042914a780abfc01250d22f5674f60195b8149f161a6481b6f6b7aa81dee", + "sha256:d0468c5c9944859c862d81621985d407097c08c0e18bb537883b9268c6e34bd1", + "sha256:dfa339c6ef6a1f36642db0dd0d442207aa2a071caa122d744222f2a2832c530f", + "sha256:e23b2e13580a4e2d35f7acba674b2b1d1fca9b20a5c0d8ffb98b8fe58c2e7107", + "sha256:e4406a5141d6d5d19ee515ff6c69baf9a7a10006a8490e7447cbc8dcf61f9903", + "sha256:ef50df5404b50109a13e46f4421ffe64a650104e2216e282a49662712b024dae", + "sha256:f459395378709b7aa32bb6e59d55d72d48631840ed6c1c919f63036bd548f375", + "sha256:fa99a0bcadef482300f5308bb9c0041d4f084b085dff0c908350de382df9f87d", + "sha256:fbd0def77da8edd5293e5e3b534763861e1c3f4f645ef3602f718fd709536a77" + ], + "version": "==3.6.6" + }, "pyjwt": { "hashes": [ "sha256:30b1380ff43b55441283cc2b2676b755cca45693ae3097325dea01f3d110628c", @@ -147,9 +346,15 @@ "sha256:1adb80e7a782c12e52ef9a8182bebeb73f1d7e24e374397af06fb4956c8dc5c0", "sha256:e27001de32f627c22380a688bcc43ce83504a7bc5da472209b4c70f02829f0b8" ], - "markers": "python_version >= '2.7'", "version": "==2.7.3" }, + "python-jose": { + "hashes": [ + "sha256:391f860dbe274223d73dd87de25e4117bf09e8fe5f93a417663b1f2d7b591165", + "sha256:3b35cdb0e55a88581ff6d3f12de753aa459e940b50fe7ca5aa25149bc94cb37b" + ], + "version": "==2.0.2" + }, "pytz": { "hashes": [ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", @@ -157,6 +362,22 @@ ], "version": "==2018.5" }, + "pyyaml": { + "hashes": [ + "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", + "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", + "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", + "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", + "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", + "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", + "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", + "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", + "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", + "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", + "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + ], + "version": "==3.13" + }, "redis": { "hashes": [ "sha256:8a1900a9f2a0a44ecf6e8b5eb3e967a9909dfed219ad66df094f27f7d6f330fb", @@ -172,6 +393,13 @@ "index": "pypi", "version": "==2.19.1" }, + "responses": { + "hashes": [ + "sha256:c6082710f4abfb60793899ca5f21e7ceb25aabf321560cc0726f8b59006811c9", + "sha256:f23a29dca18b815d9d64a516b4a0abb1fbdccff6141d988ad8100facb81cf7b3" + ], + "version": "==0.9.0" + }, "s3transfer": { "hashes": [ "sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1", @@ -205,15 +433,115 @@ ], "version": "==2.0.17.1" }, + "websocket-client": { + "hashes": [ + "sha256:03763384c530b331ec3822d0b52ffdc28c3aeb8a900ac8c98b2ceea3128a7b4e", + "sha256:3c9924675eaf0b27ae22feeeab4741bb4149b94820bd3a143eeaf8b62f64d821" + ], + "version": "==0.52.0" + }, "werkzeug": { "hashes": [ "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" ], "version": "==0.14.1" + }, + "wrapt": { + "hashes": [ + "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" + ], + "version": "==1.10.11" + }, + "xmltodict": { + "hashes": [ + "sha256:8f8d7d40aa28d83f4109a7e8aa86e67a4df202d9538be40c0cb1d70da527b0df", + "sha256:add07d92089ff611badec526912747cf87afd4f9447af6661aca074eeaf32615" + ], + "version": "==0.11.0" } }, "develop": { + "certifi": { + "hashes": [ + "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", + "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" + ], + "version": "==2018.8.24" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "coverage": { + "hashes": [ + "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba", + "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed", + "sha256:104ab3934abaf5be871a583541e8829d6c19ce7bde2923b2751e0d3ca44db60a", + "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95", + "sha256:15b111b6a0f46ee1a485414a52a7ad1d703bdf984e9ed3c288a4414d3871dcbd", + "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640", + "sha256:1c383d2ef13ade2acc636556fd544dba6e14fa30755f26812f54300e401f98f2", + "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd", + "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162", + "sha256:2a5b73210bad5279ddb558d9a2bfedc7f4bf6ad7f3c988641d83c40293deaec1", + "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508", + "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249", + "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694", + "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a", + "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287", + "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1", + "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000", + "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1", + "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e", + "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5", + "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062", + "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba", + "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc", + "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc", + "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99", + "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653", + "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c", + "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558", + "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f", + "sha256:9e112fcbe0148a6fa4f0a02e8d58e94470fc6cb82a5481618fea901699bf34c4", + "sha256:ac4fef68da01116a5c117eba4dd46f2e06847a497de5ed1d64bb99a5fda1ef91", + "sha256:b8815995e050764c8610dbc82641807d196927c3dbed207f0a079833ffcf588d", + "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9", + "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd", + "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d", + "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", + "sha256:e4d96c07229f58cb686120f168276e434660e4358cc9cf3b0464210b04913e77", + "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80", + "sha256:f8a923a85cb099422ad5a2e345fe877bbc89a8a8b23235824a93488150e45f6e" + ], + "index": "pypi", + "version": "==4.5.1" + }, + "coveralls": { + "hashes": [ + "sha256:9dee67e78ec17b36c52b778247762851c8e19a893c9a14e921a2fc37f05fac22", + "sha256:aec5a1f5e34224b9089664a1b62217732381c7de361b6ed1b3c394d7187b352a" + ], + "index": "pypi", + "version": "==1.5.0" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "idna": { + "hashes": [ + "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", + "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + ], + "version": "==2.7" + }, "jsonschema": { "hashes": [ "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", @@ -221,6 +549,13 @@ ], "version": "==2.6.0" }, + "nose2": { + "hashes": [ + "sha256:9052f2b46807b63d9bdf68e0768da1f8386368889b50043fd5d0889c470258f3" + ], + "index": "pypi", + "version": "==0.8.0" + }, "openapi-spec-validator": { "hashes": [ "sha256:14684aaec4c4f30e911132ffad3c5863047908251647f49114d249dcc2d41f4e", @@ -246,12 +581,27 @@ ], "version": "==3.13" }, + "requests": { + "hashes": [ + "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", + "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + ], + "index": "pypi", + "version": "==2.19.1" + }, "six": { "hashes": [ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], "version": "==1.11.0" + }, + "urllib3": { + "hashes": [ + "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", + "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + ], + "version": "==1.23" } } } diff --git a/compiler/config.py b/compiler/config.py index 6e50346..4943497 100644 --- a/compiler/config.py +++ b/compiler/config.py @@ -34,3 +34,14 @@ f'/{FILE_MANAGER_PATH}' ) FILE_MANAGER_VERIFY = bool(int(os.environ.get('FILE_MANAGER_VERIFY', '1'))) + +# Configuration for object store. +S3_ENDPOINT = os.environ.get('S3_ENDPOINT', None) +S3_VERIFY = bool(int(os.environ.get('S3_VERIFY', 1))) +S3_BUCKETS = [ + ('arxiv', 'arxiv-compiler'), + ('submission', 'arxiv-compiler-submission') +] +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', None) +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', None) +AWS_REGION = os.environ.get('AWS_REGION', 'us-east-1') diff --git a/compiler/domain.py b/compiler/domain.py index b4e453f..549f4e9 100644 --- a/compiler/domain.py +++ b/compiler/domain.py @@ -1,14 +1,82 @@ """Domain class for the compiler service.""" -from typing import NamedTuple - +from typing import NamedTuple, Optional +import io +from datetime import datetime from .util import ResponseStream +class CompilationStatus(NamedTuple): + """Represents the state of a compilation product in the store.""" + + # These are intended as fixed class attributes, not slots. + PDF = "pdf" # type: ignore + DVI = "dvi" # type: ignore + PS = "ps" # type: ignore + + CURRENT = "current" # type: ignore + IN_PROGRESS = "in_progress" # type: ignore + FAILED = "failed" # type: ignore + + # Here are the actual slots/fields. + source_id: str + + format: str + """ + The target format of the compilation. + + One of :attr:`PDF`, :attr:`DVI`, or :attr:`PS`. + """ + + source_checksum: str + """Checksum of the source tarball from the file management service.""" + + task_id: str + """If a task exists for this compilation, the unique task ID.""" + + status: str + """ + The status of the compilation. + + One of :attr:`CURRENT`, :attr:`IN_PROGRESS`, or :attr:`FAILED`. + + If :attr:`CURRENT`, the current file corresponding to the format of this + compilation status is the product of this compilation. + """ + + @property + def ext(self) -> str: + """Filename extension for the compilation product.""" + return self.format + + def to_dict(self) -> dict: + """Generate a dict representation of this object.""" + return { + 'source_id': self.source_id, + 'format': self.format, + 'source_checksum': self.source_checksum, + 'task_id': self.task_id, + 'status': self.status + } + + +class CompilationProduct(NamedTuple): + """Content of a compilation product itself.""" + + stream: io.BytesIO + """Readable buffer with the product content.""" + + checksum: Optional[str] = None + """The B64-encoded MD5 hash of the compilation product.""" + + status: Optional[CompilationStatus] = None + """Status information about the product.""" + + class SourcePackage(NamedTuple): """Source package content, retrieved from file management service.""" - upload_id: str + source_id: str stream: ResponseStream etag: str @@ -16,5 +84,5 @@ class SourcePackage(NamedTuple): class SourcePackageInfo(NamedTuple): """Current state of the source package in the file managment service.""" - upload_id: str + source_id: str etag: str diff --git a/compiler/services/filemanager.py b/compiler/services/filemanager.py index 3002b00..5ea7a0d 100644 --- a/compiler/services/filemanager.py +++ b/compiler/services/filemanager.py @@ -194,7 +194,7 @@ def get_upload_content(self, upload_id: str) -> SourcePackage: status.HTTP_200_OK) logger.debug('Got response with status %s', response.status_code) return SourcePackage( - upload_id=upload_id, + source_id=upload_id, stream=ResponseStream(response.iter_content(chunk_size=None)), etag=response.headers['ETag'] ) @@ -215,7 +215,7 @@ def get_upload_info(self, upload_id: str) -> SourcePackageInfo: logger.debug('Get upload info for: %s', upload_id) response, headers = self.request('head', f'/{upload_id}/content') logger.debug('Got response with etag %s', headers['ETag']) - return SourcePackageInfo(upload_id=upload_id, etag=headers['ETag']) + return SourcePackageInfo(source_id=upload_id, etag=headers['ETag']) def init_app(app: object = None) -> None: diff --git a/compiler/services/store/__init__.py b/compiler/services/store/__init__.py new file mode 100644 index 0000000..8443289 --- /dev/null +++ b/compiler/services/store/__init__.py @@ -0,0 +1,318 @@ +""" +Content store for compiled representation of paper. + +Uses S3 as the underlying storage facility. + +The intended use pattern is that a client (e.g. API controller) can check for +a compilation using the source ID (e.g. file manager upload_id), the format, +and the checksum of the source package (as reported by the FM service) before +taking any IO-intensive actions. See :meth:`StoreSession.get_status`. + +Similarly, if a client needs to verify that a compilation product is available +for a specific source checksum, they would use :meth:`StoreSession.get_status` +before calling :meth:`StoreSession.retrieve`. For that reason, +:meth:`StoreSession.retrieve` is agnostic about checksums. This cuts down on +an extra GET request to S3 every time we want to get a compiled resource. +""" +import json +from typing import Tuple, Optional, Dict, Union, List, Any, Mapping +from functools import wraps +from hashlib import md5 +from base64 import b64encode +from collections import defaultdict +import boto3 +import botocore +from flask import Flask + +from arxiv.base import logging +from arxiv.base.globals import get_application_global, get_application_config + +from ...domain import CompilationStatus, CompilationProduct + + +logger = logging.getLogger(__name__) + + +class DoesNotExist(RuntimeError): + """The requested content does not exist.""" + + +class StoreSession(object): + """Represents an object store session.""" + + def __init__(self, buckets: List[Tuple[str, str]], version: str, + verify: bool = False, + region_name: Optional[str] = None, + endpoint_url: Optional[str] = None, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None) -> None: + """Initialize with connection config parameters.""" + self.buckets = buckets + self.version = version + self.region_name = region_name + self.endpoint_url = endpoint_url + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + + # Compilation status is cached here, in case we're taking several + # status-related actions in one request/execution context. Saves us + # GET requests, which saves $$$. + self._status: Dict[str, Dict[str, Dict[str, Dict[str, str]]]] = {} + + # Only add credentials to the client if they are explicitly set. + # If they are not set, boto3 falls back to environment variables and + # credentials files. + params: Dict[str, Any] = dict(region_name=region_name) + if aws_access_key_id and aws_secret_access_key: + params.update(dict( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key + )) + if endpoint_url: + params.update(dict( + endpoint_url=endpoint_url, + verify=verify + )) + self.client = boto3.client('s3', **params) + + def _hash(self, body: bytes) -> str: + """Generate an encoded MD5 hash of a bytes.""" + return b64encode(md5(body).digest()).decode('utf-8') + + def _get_status_data(self, source_id: str, bucket: str = 'arxiv') \ + -> Dict[str, Dict[str, Dict[str, str]]]: + if source_id in self._status: + return self._status[source_id] + key = f'{source_id}/status.json' + logger.debug('Get status for upload %s with key %s', source_id, key) + try: + response = self.client.get_object( + Bucket=self._get_bucket(bucket), + Key=key + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == "NoSuchKey": + raise DoesNotExist(f'No status data for {source_id} in bucket' + f' {bucket}.') from e + raise RuntimeError(f'Unhandled exception: {e}') from e + comp_status = json.loads(response['Body'].read().decode('utf-8')) + data: Dict[str, Dict[str, Dict[str, str]]] = defaultdict(dict) + for stat in comp_status: + data[stat['format']][stat['source_checksum']] = stat + self._status[source_id] = data + return dict(data) + + def _set_status_data(self, source_id: str, + status_data: Dict[str, Dict[str, Dict[str, str]]], + bucket: str = 'arxiv') -> None: + key = f'{source_id}/status.json' + logger.debug('Set status for upload %s with key %s', source_id, key) + self._status[source_id] = status_data + comp_status = [ + stat for fmt, dat in status_data.items() for stat in dat.values() + ] + body = json.dumps(comp_status).encode('utf-8') + try: + self.client.put_object( + Body=body, + Bucket=self._get_bucket(bucket), + ContentMD5=self._hash(body), + ContentType='application/json', + Key=key + ) + except botocore.exceptions.ClientError as e: + raise RuntimeError(f'Unhandled exception: {e}') from e + + def get_status(self, source_id: str, format: str, source_checksum: str, + bucket: str = 'arxiv') -> CompilationStatus: + """ + Get the status of a compilation. + + Parameters + ---------- + source_id : str + The unique identifier of the source package. + format: str + Compilation format. See :attr:`CompilationStatus.format`. + source_checksum : str + Base64-encoded MD5 hash of the source package. + bucket : str + + Returns + ------- + :class:`CompilationStatus` + + Raises + ------ + :class:`DoesNotExist` + Raised if no status exists for the provided parameters. + + """ + status_data = self._get_status_data(source_id, bucket=bucket) + + try: + return CompilationStatus(**status_data[format][source_checksum]) + except KeyError as e: + raise DoesNotExist(f'No status data for {source_id} in format ' + f'{format} with source checksum ' + f'{source_checksum} in bucket {bucket}.') from e + + def set_status(self, status: CompilationStatus, bucket: str = 'arxiv') \ + -> None: + """ + Update the status of a compilation. + + Parameters + ---------- + status : :class:`CompilationStatus` + bucket : str + + """ + try: + status_data = self._get_status_data(status.source_id, + bucket=bucket) + except DoesNotExist: + status_data = {} + if status.format not in status_data: + status_data[status.format] = {} + status_data[status.format][status.source_checksum] = status.to_dict() + self._set_status_data(status.source_id, status_data, bucket=bucket) + + def store(self, product: CompilationProduct, bucket: str = 'arxiv') \ + -> None: + """ + Store a compilation product. + + Parameters + ---------- + product : :class:`CompilationProduct` + bucket : str + Default is ``'arxiv'``. Used in conjunction with :attr:`.buckets` + to determine the S3 bucket where this content should be stored. + + """ + if product.status is None or product.status.source_id is None: + raise ValueError('source_id must be set') + + body = product.stream.read() + status = product.status + key = f'{status.source_id}/{status.source_id}.{status.ext}' + try: + self.client.put_object( + Body=body, + Bucket=self._get_bucket(bucket), + ContentMD5=self._hash(body), + ContentType='text/plain', + Key=key, + ) + except botocore.exceptions.ClientError as e: + raise RuntimeError(f'Unhandled exception: {e}') from e + self.set_status(status, bucket=bucket) + + def retrieve(self, source_id: str, format: str, bucket: str = 'arxiv') \ + -> CompilationProduct: + """ + Retrieve a compilation product. + + Parameters + ---------- + source_id : str + format: str + bucket : str + Default is ``'arxiv'``. Used in conjunction with :attr:`.buckets` + to determine the S3 bucket from which the content should be + retrieved + + Returns + ------- + :class:`CompilationProduct` + + """ + key = f'{source_id}/{source_id}.{format}' + try: + response = self.client.get_object( + Bucket=self._get_bucket(bucket), + Key=key + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == "NoSuchKey": + raise DoesNotExist(f'No {format} product for {source_id} in' + f' bucket {bucket}') from e + raise RuntimeError(f'Unhandled exception: {e}') from e + return CompilationProduct(stream=response['Body'], + checksum=response['ETag'][1:-1]) + + def create_bucket(self) -> None: + """Create S3 buckets. This is just for testing.""" + for key, bucket in self.buckets: + self.client.create_bucket(Bucket=bucket) + + def _get_bucket(self, bucket: str) -> str: + try: + name: str = dict(self.buckets)[bucket] + except KeyError as e: + raise RuntimeError(f'No such bucket: {bucket}') from e + return name + + +@wraps(StoreSession.get_status) +def get_status(source_id: str, format: str, source_checksum: str, + bucket: str = 'arxiv') -> CompilationStatus: + """See :func:`StoreSession.get_status`.""" + s = current_session() + return s.get_status(source_id, format, source_checksum, bucket=bucket) + + +@wraps(StoreSession.set_status) +def set_status(status: CompilationStatus, bucket: str = 'arxiv') -> None: + """See :func:`StoreSession.get_status`.""" + return current_session().set_status(status, bucket=bucket) + + +@wraps(StoreSession.store) +def store(product: CompilationProduct, bucket: str = 'arxiv') -> None: + """See :func:`StoreSession.store`.""" + return current_session().store(product, bucket=bucket) + + +@wraps(StoreSession.retrieve) +def retrieve(source_id: str, format: str, bucket: str = 'arxiv') \ + -> CompilationProduct: + """See :func:`StoreSession.retrieve`.""" + return current_session().retrieve(source_id, format, bucket=bucket) + + +def init_app(app: Flask) -> None: + """Set defaults for required configuration parameters.""" + app.config.setdefault('AWS_REGION', 'us-east-1') + app.config.setdefault('AWS_ACCESS_KEY_ID', None) + app.config.setdefault('AWS_SECRET_ACCESS_KEY', None) + app.config.setdefault('S3_ENDPOINT', None) + app.config.setdefault('S3_VERIFY', True) + app.config.setdefault('S3_BUCKET', []) + app.config.setdefault('VERSION', "0.0") + + +def get_session() -> StoreSession: + """Create a new :class:`botocore.client.S3` session.""" + config = get_application_config() + access_key = config.get('AWS_ACCESS_KEY_ID') + secret_key = config.get('AWS_SECRET_ACCESS_KEY') + endpoint = config.get('S3_ENDPOINT') + verify = config.get('S3_VERIFY') + region = config.get('AWS_REGION') + buckets = config.get('S3_BUCKETS') + version = config.get('VERSION') + return StoreSession(buckets, version, verify, region, + endpoint, access_key, secret_key) + + +def current_session() -> StoreSession: + """Get the current store session for this application.""" + g = get_application_global() + if g is None: + return get_session() + if 'store' not in g: + g.store = get_session() + store: StoreSession = g.store + return store diff --git a/compiler/services/store/schema/status.json b/compiler/services/store/schema/status.json new file mode 100644 index 0000000..e903ebe --- /dev/null +++ b/compiler/services/store/schema/status.json @@ -0,0 +1,37 @@ +{ + "title": "Compilation status", + "description": "Describes the state of compilation products in the store", + "additionalProperties": false, + "required": ["compilations"], + "type": "object", + "properties": { + "compilations": { + "type": "array", + "items": { + "properties": { + "source_checksum": { + "description": "Checksum of the source tarball from the file management service.", + "type": "string" + }, + "task_id": { + "description": "If a task exists for this compilation, the unique task ID.", + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ] + }, + "status": { + "description": "The status of the compilation. If `current`, the current file corresponding to the format of this compilation status is the product of this compilation.", + "type": "string", + "enum": ["current", "in_progress", "failed"] + }, + "format": { + "description": "The target format of the compilation.", + "type": "string", + "enum": ["pdf", "dvi", "ps"] + } + } + } + } + } +} diff --git a/compiler/services/store/tests.py b/compiler/services/store/tests.py new file mode 100644 index 0000000..d9daaab --- /dev/null +++ b/compiler/services/store/tests.py @@ -0,0 +1,118 @@ +"""Tests for :mod:`compiler.services.store`.""" + +from unittest import TestCase, mock +from moto import mock_s3 +import io +from datetime import datetime + +from .. import store +from ... import domain + +mock_app_config = mock.MagicMock(return_value={ + 'S3_ENDPOINT': None, + 'S3_VERIFY': True, + 'S3_BUCKETS': [ + ('arxiv', 'arxiv-compiler'), + ('submission', 'arxiv-compiler-submission') + ] +}) + + +class TestStore(TestCase): + """Test methods on :mod:`compiler.services.store`.""" + + @mock_s3 + @mock.patch(f'{store.__name__}.get_application_config', mock_app_config) + def test_set_get_compilation_status(self): + """Test setting and getting compilation status.""" + store.current_session().create_bucket() + status_pdf = domain.CompilationStatus( + source_id='12345', + format=domain.CompilationStatus.PDF, + source_checksum='abc123checksum', + task_id='foo-task-1234-6789', + status=domain.CompilationStatus.IN_PROGRESS + ) + store.set_status(status_pdf) + + retrieved = store.get_status('12345', domain.CompilationStatus.PDF, + 'abc123checksum') + self.assertEqual(status_pdf, retrieved) + + # No compilation product for that checksum. + with self.assertRaises(store.DoesNotExist): + store.get_status('12345', domain.CompilationStatus.PDF, 'foocheck') + # No compilation product for that format. + with self.assertRaises(store.DoesNotExist): + store.get_status('12345', domain.CompilationStatus.PS, + 'abc123checksum') + + # New format for same upload ID/checksum. + status_ps = domain.CompilationStatus( + source_id='12345', + format=domain.CompilationStatus.PS, + source_checksum='abc123checksum', + task_id='foo-task-1234-6789', + status=domain.CompilationStatus.IN_PROGRESS + ) + store.set_status(status_ps) + + retrieved_pdf = store.get_status('12345', domain.CompilationStatus.PDF, + 'abc123checksum') + self.assertEqual(status_pdf, retrieved_pdf) + retrieved_ps = store.get_status('12345', domain.CompilationStatus.PS, + 'abc123checksum') + self.assertEqual(status_ps, retrieved_ps) + + # Change the status of the existing format/checksum. + status_ps_failed = domain.CompilationStatus( + source_id='12345', + format=domain.CompilationStatus.PS, + source_checksum='abc123checksum', + task_id='foo-task-1234-6789', + status=domain.CompilationStatus.FAILED + ) + store.set_status(status_ps_failed) + retrieved_ps = store.get_status('12345', domain.CompilationStatus.PS, + 'abc123checksum') + self.assertEqual(status_ps_failed, retrieved_ps) + + # Same format, new checksum. + status_ps_alt = domain.CompilationStatus( + source_id='12345', + format=domain.CompilationStatus.PS, + source_checksum='someotherchecksum1234', + task_id='foo-task-1234-6710', + status=domain.CompilationStatus.CURRENT + ) + store.set_status(status_ps_alt) + + retrieved_ps = store.get_status('12345', domain.CompilationStatus.PS, + 'someotherchecksum1234') + self.assertEqual(status_ps_alt, retrieved_ps) + + @mock_s3 + @mock.patch(f'{store.__name__}.get_application_config', mock_app_config) + def test_store_retrieve(self): + """Test storing and retrieving compilation products.""" + content = io.BytesIO(b'somepdfcontent') + store.current_session().create_bucket() + status_pdf = domain.CompilationStatus( + source_id='12345', + format=domain.CompilationStatus.PDF, + source_checksum='abc123checksum', + task_id='foo-task-1234-6789', + status=domain.CompilationStatus.CURRENT + ) + product = domain.CompilationProduct(stream=content, status=status_pdf) + store.store(product) + + rstatus_pdf = store.get_status('12345', domain.CompilationStatus.PDF, + 'abc123checksum') + self.assertEqual(rstatus_pdf, status_pdf) + + returned = store.retrieve('12345', domain.CompilationStatus.PDF) + self.assertEqual(returned.stream.read(), b'somepdfcontent') + + with self.assertRaises(store.DoesNotExist): + store.retrieve('12345', domain.CompilationStatus.PS) diff --git a/compiler/services/tests/test_filemanager.py b/compiler/services/tests/test_filemanager.py index fff7862..f5b00dd 100644 --- a/compiler/services/tests/test_filemanager.py +++ b/compiler/services/tests/test_filemanager.py @@ -26,7 +26,7 @@ def test_get_upload_info(self, mock_Session): info = filemanager.get_upload_info(upload_id) self.assertIsInstance(info, domain.SourcePackageInfo) self.assertEqual(info.etag, etag) - self.assertEqual(info.upload_id, upload_id) + self.assertEqual(info.source_id, upload_id) @mock.patch(f'{filemanager.__name__}.requests.Session') def test_get_upload_info_nonexistant(self, mock_Session): @@ -66,7 +66,7 @@ def test_get_upload(self, mock_Session): info = filemanager.get_upload_content(upload_id) self.assertIsInstance(info, domain.SourcePackage) self.assertEqual(info.etag, etag) - self.assertEqual(info.upload_id, upload_id) + self.assertEqual(info.source_id, upload_id) self.assertIsInstance(info.stream, util.ResponseStream) self.assertEqual(info.stream.read(), content) diff --git a/compiler/util.py b/compiler/util.py index 639d97c..7a2fd9c 100644 --- a/compiler/util.py +++ b/compiler/util.py @@ -11,6 +11,6 @@ def __init__(self, iterator: Iterator) -> None: """Set the bytes-producing iterator.""" self._iterator = iterator - def read(self) -> bytes: + def read(self) -> Iterator: """Get bytes from the stream.""" return self._iterator diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..5d04942 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,31 @@ +[mypy] +mypy_path = $MYPYPATH:./sqlalchemy-stubs + +# +# Covered by --strict, with some turned off: +# +disallow_untyped_calls=True +disallow_untyped_defs=True +check_untyped_defs=True +# currently an issue with sql alchemy +disallow_subclassing_any=false +# Need to experiment/think about this one: +disallow_any_decorated=false +warn_redundant_casts=True +warn_return_any=True +warn_unused_ignores=True +# this seems to be at least somewhat non-functioning: +#warn_unused_configs=True +#may be worth reconsidering this one: +no_implicit_optional=True +strict_optional=True + +# +# Other: +# +ignore_missing_imports=True + + +[mypy-sqlalchemy.*] +disallow_untyped_calls=False +disallow_untyped_defs=False