From 970c5978f7e8461e23d1d8fd039b120fc69eae2c Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 22:09:50 +0200 Subject: [PATCH 01/75] Add new api_info index --- deploy/lib/watch_history.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index fc738474..f8b0d123 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -82,6 +82,11 @@ def _create_tables(self): sort_key=Attribute(name="special_progress", type=AttributeType.NUMBER), index_name="special_progress" ) + self.watch_history_table.add_global_secondary_index( + partition_key=Attribute(name="username", type=AttributeType.STRING), + sort_key=Attribute(name="api_info", type=AttributeType.STRING), + index_name="api_info" + ) self.episodes_table = Table( self, From 7bb73532cc27b6b7b5eef02f3b74b4b1990db706 Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 22:48:10 +0200 Subject: [PATCH 02/75] Move cron and tvmaze api from show service --- src/lambdas/cron/update_eps/__init__.py | 19 +++++++ src/layers/api/python/tvmaze.py | 69 +++++++++++++++++++++++++ src/layers/api/python/updates.py | 27 ++++++++++ 3 files changed, 115 insertions(+) create mode 100644 src/lambdas/cron/update_eps/__init__.py create mode 100644 src/layers/api/python/tvmaze.py create mode 100644 src/layers/api/python/updates.py diff --git a/src/lambdas/cron/update_eps/__init__.py b/src/lambdas/cron/update_eps/__init__.py new file mode 100644 index 00000000..174d9a17 --- /dev/null +++ b/src/lambdas/cron/update_eps/__init__.py @@ -0,0 +1,19 @@ +import updates +import watch_history_db + +from tvmaze import TvMazeApi, HTTPError + + +def handle(event, context): + tvmaze_api = TvMazeApi() + tvmaze_updates = tvmaze_api.get_day_updates() + + for tvmaze_id in tvmaze_updates: + try: + watch_history_db.get_items_by_id(tvmaze_id) + except HTTPError: + # Show not present in db, exclude it from updates + continue + + # Post to SNS topic + updates.publish_show_update("tvmaze", tvmaze_id) diff --git a/src/layers/api/python/tvmaze.py b/src/layers/api/python/tvmaze.py new file mode 100644 index 00000000..0888da39 --- /dev/null +++ b/src/layers/api/python/tvmaze.py @@ -0,0 +1,69 @@ +import requests + +import logger + +log = logger.get_logger(__name__) + + +class Error(Exception): + pass + + +class HTTPError(Error): + + def __init__(self, code): + Error.__init__(self, f"Unexpected status code: {code}") + self.code = code + + +class TvMazeApi: + def __init__(self): + self.base_url = "https://api.tvmaze.com" + + log.debug("TvMazeApi base_url: {}".format(self.base_url)) + + def get_show(self, show_id): + res = requests.get(f"{self.base_url}/shows/{show_id}") + + if res.status_code != 200: + raise HTTPError(res.status_code) + return res.json() + + def get_episode(self, episode_id): + res = requests.get(f"{self.base_url}/episodes/{episode_id}") + + if res.status_code != 200: + raise HTTPError(res.status_code) + return res.json() + + def get_day_updates(self): + res = requests.get(f"{self.base_url}/updates/shows?since=day") + if res.status_code != 200: + raise HTTPError(res.status_code) + return res.json() + + def get_show_episodes(self, show_id): + res = requests.get( + f"{self.base_url}/shows/{show_id}/episodes?specials=1" + ) + + if res.status_code != 200: + raise HTTPError(res.status_code) + return res.json() + + def get_show_episodes_count(self, show_id): + episodes = self.get_show_episodes(show_id) + + ep_count = 0 + special_count = 0 + + for e in episodes: + if e["type"] == "regular": + ep_count += 1 + else: + special_count += 1 + + return { + "ep_count": ep_count, + "special_count": special_count, + } diff --git a/src/layers/api/python/updates.py b/src/layers/api/python/updates.py new file mode 100644 index 00000000..9bbd698f --- /dev/null +++ b/src/layers/api/python/updates.py @@ -0,0 +1,27 @@ +import json +import os + +import boto3 + +TOPIC_ARN = os.getenv("UPDATES_TOPIC_ARN") + +topic = None + + +def _get_topic(): + global topic + + if topic is None: + sns = boto3.resource("sns") + topic = sns.Topic(TOPIC_ARN) + + return topic + + +def publish_show_update(api_name, api_id): + _get_topic().publish( + Message=json.dumps({ + "api_name": api_name, + "api_id": api_id, + }) + ) From 7a1c98e4f733003f3c9c578b2ef84c990ee72394 Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 22:48:43 +0200 Subject: [PATCH 03/75] Add fastapi requirements to generate hashes file --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8e88304a..697852ce 100644 --- a/Makefile +++ b/Makefile @@ -14,4 +14,5 @@ generate-hashes: pip-compile --generate-hashes src/layers/api/requirements.in --output-file src/layers/api/requirements.txt --allow-unsafe pip-compile --generate-hashes src/layers/databases/requirements.in --output-file src/layers/databases/requirements.txt --allow-unsafe pip-compile --generate-hashes src/layers/utils/requirements.in --output-file src/layers/utils/requirements.txt --allow-unsafe - pip-compile --generate-hashes deploy/requirements.in --output-file deploy/requirements.txt --allow-unsafe \ No newline at end of file + pip-compile --generate-hashes deploy/requirements.in --output-file deploy/requirements.txt --allow-unsafe + pip-compile --generate-hashes src/lambdas/watch_histories/requirements.in --output-file src/lambdas/watch_histories/requirements.txt \ No newline at end of file From d3630b99d34f883974765cac2f8b03b4169be874 Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 22:48:53 +0200 Subject: [PATCH 04/75] Create new show updates topic --- deploy/lib/watch_history.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index f8b0d123..83181d71 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -36,12 +36,19 @@ def __init__(self, app: core.App, id: str, anime_api_url: str, self.domain_name = domain_name self.layers = {} self.lambdas = {} + self._create_topics() self._create_tables() self._create_lambdas_config() self._create_layers() self._create_lambdas() self._create_gateway() + def _create_topics(self): + self.show_updates_topic = Topic( + self, + "shows_updates", + ) + def _create_tables(self): self.watch_history_table = Table( self, From ad36e2a29fb4190731bc60ad189e2abb4007892a Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 22:49:13 +0200 Subject: [PATCH 05/75] Add config for new lambdas --- deploy/lib/watch_history.py | 48 +++++++++++++++++++++++++++++++++++++ deploy/requirements.txt | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 83181d71..e3ca08d6 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -111,6 +111,27 @@ def _create_tables(self): def _create_lambdas_config(self): self.lambdas_config = { + "api-watch_histories": { + "layers": ["utils", "databases", "api"], + "variables": { + "DATABASE_NAME": self.watch_history_table.table_name, + "EPISODES_DATABASE_NAME": self.episodes_table.table_name, + "LOG_LEVEL": "INFO", + "ANIME_API_URL": self.anime_api_url, + "MOVIE_API_URL": self.movie_api_url, + }, + "concurrent_executions": 100, + "policies": [ + PolicyStatement( + actions=["dynamodb:Query"], + resources=[ + self.watch_history_table.table_arn, + f"{self.watch_history_table.table_arn}/index/*", + ] + ), + ], + "timeout": 60 + }, "api-watch_history": { "layers": ["utils", "databases", "api"], "variables": { @@ -254,6 +275,28 @@ def _create_lambdas_config(self): ], "timeout": 10 }, + "cron-update_eps": { + "layers": ["utils", "databases", "api"], + "variables": { + "DATABASE_NAME": self.watch_history_table.table_name, + "LOG_LEVEL": "INFO", + "UPDATES_TOPIC_ARN": self.show_updates_topic.topic_arn, + }, + "concurrent_executions": 1, + "policies": [ + PolicyStatement( + actions=["dynamodb:Query"], + resources=[ + f"{self.watch_history_table.table_arn}/index/item_id"], + ), + PolicyStatement( + actions=["sns:Publish"], + resources=[self.show_updates_topic.topic_arn], + ) + ], + "timeout": 60, + "memory": 1024 + }, "subscribers-show_updates": { "layers": ["utils", "databases", "api"], "variables": { @@ -407,6 +450,11 @@ def _create_gateway(self): ) routes = { + "watch_histories": { + "method": ["GET"], + "route": "/{proxy+}", + "target_lambda": self.lambdas["api-watch_histories"] + }, "watch_history": { "method": ["GET"], "route": "/watch-history", diff --git a/deploy/requirements.txt b/deploy/requirements.txt index 5e65a30f..d68e8fb9 100644 --- a/deploy/requirements.txt +++ b/deploy/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --generate-hashes --output-file=deploy/requirements.txt deploy/requirements.in +# pip-compile --allow-unsafe --generate-hashes --output-file=deploy/requirements.txt deploy/requirements.in # attrs==21.2.0 \ --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ From b5cb6345522f99c404434199451011aaaa32ccb5 Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 22:49:32 +0200 Subject: [PATCH 06/75] Add new func for getting item by id --- src/layers/databases/python/watch_history_db.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 835d93a6..4276f7e0 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -89,6 +89,19 @@ def get_item(username, collection_name, item_id, include_deleted=False): return res["Items"][0] +def get_item_by_api_id(username, api_info): + res = _get_table().query( + IndexName="api_info", + KeyConditionExpression=Key("username").eq(username) & + Key("api_info").eq(api_info), + ) + + if not res["Items"]: + raise NotFoundError(f"Item with api_info: {api_info} not found") + + return res["Items"][0] + + def update_item(username, collection_name, item_id, data, clean_whitelist=OPTIONAL_FIELDS): data["collection_name"] = collection_name @@ -129,7 +142,8 @@ def update_item(username, collection_name, item_id, data, ) -def change_watched_eps(username, collection_name, item_id, change, special=False): +def change_watched_eps(username, collection_name, item_id, change, + special=False): field_name = "ep" if special: field_name = "special" From 6f5327f661995143ff0a47a96d8a00fd725db031 Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 22:50:00 +0200 Subject: [PATCH 07/75] Start adding fastapi lambda --- src/lambdas/api/watch_histories/__init__.py | 8 +++ .../api/watch_histories/requirements.in | 1 + .../api/watch_histories/requirements.txt | 55 +++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 src/lambdas/api/watch_histories/__init__.py create mode 100644 src/lambdas/api/watch_histories/requirements.in create mode 100644 src/lambdas/api/watch_histories/requirements.txt diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py new file mode 100644 index 00000000..0ac34a96 --- /dev/null +++ b/src/lambdas/api/watch_histories/__init__.py @@ -0,0 +1,8 @@ +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/watch-histories/item") +async def root(): + return {"message": "Hello World"} diff --git a/src/lambdas/api/watch_histories/requirements.in b/src/lambdas/api/watch_histories/requirements.in new file mode 100644 index 00000000..4388ac4d --- /dev/null +++ b/src/lambdas/api/watch_histories/requirements.in @@ -0,0 +1 @@ +fastapi==0.70.0 \ No newline at end of file diff --git a/src/lambdas/api/watch_histories/requirements.txt b/src/lambdas/api/watch_histories/requirements.txt new file mode 100644 index 00000000..d5e1fe9f --- /dev/null +++ b/src/lambdas/api/watch_histories/requirements.txt @@ -0,0 +1,55 @@ +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# pip-compile --generate-hashes --output-file=src/lambdas/watch_histories/requirements.txt src/lambdas/watch_histories/requirements.in +# +anyio==3.3.4 \ + --hash=sha256:4fd09a25ab7fa01d34512b7249e366cd10358cdafc95022c7ff8c8f8a5026d66 \ + --hash=sha256:67da67b5b21f96b9d3d65daa6ea99f5d5282cb09f50eb4456f8fb51dffefc3ff + # via starlette +fastapi==0.70.0 \ + --hash=sha256:66da43cfe5185ea1df99552acffd201f1832c6b364e0f4136c0a99f933466ced \ + --hash=sha256:a36d5f2fad931aa3575c07a3472c784e81f3e664e3bb5c8b9c88d0ec1104f59c + # via -r src/lambdas/watch_histories/requirements.in +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via anyio +pydantic==1.8.2 \ + --hash=sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd \ + --hash=sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739 \ + --hash=sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f \ + --hash=sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840 \ + --hash=sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23 \ + --hash=sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287 \ + --hash=sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62 \ + --hash=sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b \ + --hash=sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb \ + --hash=sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820 \ + --hash=sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3 \ + --hash=sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b \ + --hash=sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e \ + --hash=sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3 \ + --hash=sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316 \ + --hash=sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b \ + --hash=sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4 \ + --hash=sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20 \ + --hash=sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e \ + --hash=sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505 \ + --hash=sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1 \ + --hash=sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833 + # via fastapi +sniffio==1.2.0 \ + --hash=sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663 \ + --hash=sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de + # via anyio +starlette==0.16.0 \ + --hash=sha256:38eb24bf705a2c317e15868e384c1b8a12ca396e5a3c3a003db7e667c43f939f \ + --hash=sha256:e1904b5d0007aee24bdd3c43994be9b3b729f4f58e740200de1d623f8c3a8870 + # via fastapi +typing-extensions==3.10.0.2 \ + --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ + --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ + --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 + # via pydantic From d6e8cb0af8a1b120853d573dea1f6a15faa81945 Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 23:00:54 +0200 Subject: [PATCH 08/75] Rename cron lambda and quickfix fastapi requirement --- .github/workflows/deploy.yml | 5 +++++ deploy/lib/watch_history.py | 2 +- src/lambdas/cron/{update_eps => show_updates}/__init__.py | 0 3 files changed, 6 insertions(+), 1 deletion(-) rename src/lambdas/cron/{update_eps => show_updates}/__init__.py (100%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9c2380d8..95cc66aa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -11,6 +11,11 @@ jobs: steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + + - run: | + pip install -r src/lambdas/api/watch_histories/requirements.txt --target src/lambdas/api/watch_histories/ + - name: cdk deploy uses: youyo/aws-cdk-github-actions@v2.0.2 with: diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index e3ca08d6..f2f0b2fe 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -275,7 +275,7 @@ def _create_lambdas_config(self): ], "timeout": 10 }, - "cron-update_eps": { + "cron-show_updates": { "layers": ["utils", "databases", "api"], "variables": { "DATABASE_NAME": self.watch_history_table.table_name, diff --git a/src/lambdas/cron/update_eps/__init__.py b/src/lambdas/cron/show_updates/__init__.py similarity index 100% rename from src/lambdas/cron/update_eps/__init__.py rename to src/lambdas/cron/show_updates/__init__.py From 2f7da832ee217abb2d40a764d005fc951375bdcc Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 23:05:08 +0200 Subject: [PATCH 09/75] Ignore temp installed requirements --- deploy/lib/watch_history.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index f2f0b2fe..640de247 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -362,6 +362,9 @@ def _create_lambdas(self): if f != "__init__.py": continue + if "watch_histories/" in root: + continue + parent_folder = os.path.basename(os.path.dirname(root)) lambda_folder = os.path.basename(root) name = f"{parent_folder}-{lambda_folder}" From f7198ae3d6ef9fed4796db218fdb5613e79665d1 Mon Sep 17 00:00:00 2001 From: fulder Date: Thu, 28 Oct 2021 23:28:37 +0200 Subject: [PATCH 10/75] Use mangum in fastapi and change method to any --- Makefile | 2 +- deploy/lib/watch_history.py | 2 +- src/lambdas/api/watch_histories/__init__.py | 8 ++++++-- src/lambdas/api/watch_histories/requirements.in | 3 ++- src/lambdas/api/watch_histories/requirements.txt | 12 +++++++++--- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 697852ce..168c4c2b 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,4 @@ generate-hashes: pip-compile --generate-hashes src/layers/databases/requirements.in --output-file src/layers/databases/requirements.txt --allow-unsafe pip-compile --generate-hashes src/layers/utils/requirements.in --output-file src/layers/utils/requirements.txt --allow-unsafe pip-compile --generate-hashes deploy/requirements.in --output-file deploy/requirements.txt --allow-unsafe - pip-compile --generate-hashes src/lambdas/watch_histories/requirements.in --output-file src/lambdas/watch_histories/requirements.txt \ No newline at end of file + pip-compile --generate-hashes src/lambdas/api/watch_histories/requirements.in --output-file src/lambdas/api/watch_histories/requirements.txt \ No newline at end of file diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 640de247..70b08b29 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -454,7 +454,7 @@ def _create_gateway(self): routes = { "watch_histories": { - "method": ["GET"], + "method": ["ANY"], "route": "/{proxy+}", "target_lambda": self.lambdas["api-watch_histories"] }, diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 0ac34a96..57437ebb 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,8 +1,12 @@ from fastapi import FastAPI +from mangum import Mangum app = FastAPI() @app.get("/watch-histories/item") -async def root(): - return {"message": "Hello World"} +def item(api_name: str, api_id: str): + return {"message": f"Got {api_name}_{api_id}"} + + +handler = Mangum(app) diff --git a/src/lambdas/api/watch_histories/requirements.in b/src/lambdas/api/watch_histories/requirements.in index 4388ac4d..24556cc5 100644 --- a/src/lambdas/api/watch_histories/requirements.in +++ b/src/lambdas/api/watch_histories/requirements.in @@ -1 +1,2 @@ -fastapi==0.70.0 \ No newline at end of file +fastapi==0.70.0 +mangum==0.12.3 \ No newline at end of file diff --git a/src/lambdas/api/watch_histories/requirements.txt b/src/lambdas/api/watch_histories/requirements.txt index d5e1fe9f..5a9d682f 100644 --- a/src/lambdas/api/watch_histories/requirements.txt +++ b/src/lambdas/api/watch_histories/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --generate-hashes --output-file=src/lambdas/watch_histories/requirements.txt src/lambdas/watch_histories/requirements.in +# pip-compile --generate-hashes --output-file=src/lambdas/api/watch_histories/requirements.txt src/lambdas/api/watch_histories/requirements.in # anyio==3.3.4 \ --hash=sha256:4fd09a25ab7fa01d34512b7249e366cd10358cdafc95022c7ff8c8f8a5026d66 \ @@ -11,11 +11,15 @@ anyio==3.3.4 \ fastapi==0.70.0 \ --hash=sha256:66da43cfe5185ea1df99552acffd201f1832c6b364e0f4136c0a99f933466ced \ --hash=sha256:a36d5f2fad931aa3575c07a3472c784e81f3e664e3bb5c8b9c88d0ec1104f59c - # via -r src/lambdas/watch_histories/requirements.in + # via -r src/lambdas/api/watch_histories/requirements.in idna==3.3 \ --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d # via anyio +mangum==0.12.3 \ + --hash=sha256:128a8c1aa2eef13fc94ac5344210df8bd71dfba584b658bec9473e784bd86301 \ + --hash=sha256:a45cc1c57736e7044318e65216eea83a397cb709e39fe6810a46faf0e15d2664 + # via -r src/lambdas/api/watch_histories/requirements.in pydantic==1.8.2 \ --hash=sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd \ --hash=sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739 \ @@ -52,4 +56,6 @@ typing-extensions==3.10.0.2 \ --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 - # via pydantic + # via + # mangum + # pydantic From aab38c20cdb8a50b64666bfd66ce8685fb65231a Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 08:30:41 +0200 Subject: [PATCH 11/75] Add needed base_path to mangum --- src/lambdas/api/watch_histories/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 57437ebb..577873b9 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -9,4 +9,4 @@ def item(api_name: str, api_id: str): return {"message": f"Got {api_name}_{api_id}"} -handler = Mangum(app) +handle = Mangum(app, api_gateway_base_path="/prod/") From 295218ec860ac500a9d4c6253844e3bf64d5fa6b Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 08:44:46 +0200 Subject: [PATCH 12/75] Add middelware for getting username from JWT --- src/lambdas/api/watch_histories/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 577873b9..c81d9381 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,12 +1,22 @@ -from fastapi import FastAPI +from fastapi import FastAPI, Request from mangum import Mangum +from . import api +import jwt_utils + app = FastAPI() @app.get("/watch-histories/item") def item(api_name: str, api_id: str): - return {"message": f"Got {api_name}_{api_id}"} + return api.get_item(api_name, api_id) + + +@app.middleware("http") +def parse_token(request: Request, call_next): + username = jwt_utils.get_username(request.headers.get("authorization")) + request.state.username = username + return call_next handle = Mangum(app, api_gateway_base_path="/prod/") From 69d60e3e9bc49557f1acfd9815c9c400a8169531 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 08:47:01 +0200 Subject: [PATCH 13/75] Implement first try of api.get_item --- src/lambdas/api/watch_histories/__init__.py | 4 +-- src/lambdas/api/watch_histories/api.py | 30 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 src/lambdas/api/watch_histories/api.py diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index c81d9381..10a8b601 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -8,8 +8,8 @@ @app.get("/watch-histories/item") -def item(api_name: str, api_id: str): - return api.get_item(api_name, api_id) +def item(request: Request, api_name: str, api_id: str): + return api.get_item(request.state.username, api_name, api_id) @app.middleware("http") diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py new file mode 100644 index 00000000..faebc38b --- /dev/null +++ b/src/lambdas/api/watch_histories/api.py @@ -0,0 +1,30 @@ +import json + +from fastapi import HTTPException + +import decimal_encoder +import logger +import tvmaze +import watch_history_db + +tvmaze_api = tvmaze.TvMazeApi() +log = logger.get_logger(__name__) + + +def get_item(username, api_name, api_id): + try: + if api_name == "tvmaze": + api_ret = tvmaze_api.get_show(api_id) + else: + raise HTTPException(status_code=501) + except tvmaze.HTTPError as e: + err_msg = f"Could not get item from {api_name} api with id: {api_id}" + log.error(f"{err_msg}. Error: {str(e)}") + raise HTTPException(status_code=e.code) + + try: + w_ret = watch_history_db.get_item_by_api_id(username, api_id) + ret = {**w_ret, api_name: {**api_ret}} + return {json.dumps(ret, cls=decimal_encoder.DecimalEncoder)} + except watch_history_db.NotFoundError: + raise HTTPException(status_code=404) From 9b07163e8f1685dd977e7b13c03a62a86c3cebfd Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 17:38:24 +0200 Subject: [PATCH 14/75] Change relative import --- src/lambdas/api/watch_histories/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 10a8b601..8709bd8b 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,7 +1,7 @@ from fastapi import FastAPI, Request from mangum import Mangum -from . import api +import api import jwt_utils app = FastAPI() From 0f395375e22d716874e5fba102d305d3c8a70801 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 17:42:23 +0200 Subject: [PATCH 15/75] Remove Bearer string from jwt data --- src/lambdas/api/watch_histories/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 8709bd8b..7bc37e93 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -14,8 +14,9 @@ def item(request: Request, api_name: str, api_id: str): @app.middleware("http") def parse_token(request: Request, call_next): - username = jwt_utils.get_username(request.headers.get("authorization")) - request.state.username = username + auth_header = request.headers.get("authorization") + jwt_str = auth_header.split("Bearer ")[1] + request.state.username = jwt_utils.get_username(jwt_str) return call_next From 52c6f6a3d8f2434437d5b2cba7e4fc446b3ec188 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 17:49:17 +0200 Subject: [PATCH 16/75] Forgotten to add request param to call_next --- src/lambdas/api/watch_histories/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 7bc37e93..e0efa6ca 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -17,7 +17,7 @@ def parse_token(request: Request, call_next): auth_header = request.headers.get("authorization") jwt_str = auth_header.split("Bearer ")[1] request.state.username = jwt_utils.get_username(jwt_str) - return call_next + return call_next(request) handle = Mangum(app, api_gateway_base_path="/prod/") From 654eb689b846a2f8ee55fb2f798526b9f1e40111 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 17:54:49 +0200 Subject: [PATCH 17/75] Fix api_info in get_item_by_api_id --- src/lambdas/api/watch_histories/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py index faebc38b..d3782ef1 100644 --- a/src/lambdas/api/watch_histories/api.py +++ b/src/lambdas/api/watch_histories/api.py @@ -23,7 +23,7 @@ def get_item(username, api_name, api_id): raise HTTPException(status_code=e.code) try: - w_ret = watch_history_db.get_item_by_api_id(username, api_id) + w_ret = watch_history_db.get_item_by_api_id(username, f"{api_name}_{api_id}") ret = {**w_ret, api_name: {**api_ret}} return {json.dumps(ret, cls=decimal_encoder.DecimalEncoder)} except watch_history_db.NotFoundError: From 326ff0e189f6221973df2f143accb8c32739dde5 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 17:58:16 +0200 Subject: [PATCH 18/75] Remove duplicated dict --- src/lambdas/api/watch_histories/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py index d3782ef1..52cd0779 100644 --- a/src/lambdas/api/watch_histories/api.py +++ b/src/lambdas/api/watch_histories/api.py @@ -25,6 +25,6 @@ def get_item(username, api_name, api_id): try: w_ret = watch_history_db.get_item_by_api_id(username, f"{api_name}_{api_id}") ret = {**w_ret, api_name: {**api_ret}} - return {json.dumps(ret, cls=decimal_encoder.DecimalEncoder)} + return json.dumps(ret, cls=decimal_encoder.DecimalEncoder) except watch_history_db.NotFoundError: raise HTTPException(status_code=404) From d57dcf81815737ab2e09a971d3ab771a6f214f82 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 18:00:40 +0200 Subject: [PATCH 19/75] Try returing dict directly --- src/lambdas/api/watch_histories/api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py index 52cd0779..e92d1353 100644 --- a/src/lambdas/api/watch_histories/api.py +++ b/src/lambdas/api/watch_histories/api.py @@ -24,7 +24,6 @@ def get_item(username, api_name, api_id): try: w_ret = watch_history_db.get_item_by_api_id(username, f"{api_name}_{api_id}") - ret = {**w_ret, api_name: {**api_ret}} - return json.dumps(ret, cls=decimal_encoder.DecimalEncoder) + return {**w_ret, api_name: {**api_ret}} except watch_history_db.NotFoundError: raise HTTPException(status_code=404) From bffc74216732bb6b30aa9a79a0adc5ad808b5673 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 19:36:06 +0200 Subject: [PATCH 20/75] Change ANY to real verbs This will collide with the OPTIONS CORS settings otherwise --- deploy/lib/watch_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 70b08b29..0aeb0249 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -454,7 +454,7 @@ def _create_gateway(self): routes = { "watch_histories": { - "method": ["ANY"], + "method": ["GET", "POST", "DELETE"], "route": "/{proxy+}", "target_lambda": self.lambdas["api-watch_histories"] }, From e075728cd28f23e7a064b4eba991270996453c3c Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 22:54:11 +0200 Subject: [PATCH 21/75] Implement new add_item with show support --- src/lambdas/api/watch_histories/__init__.py | 8 +++- src/lambdas/api/watch_histories/api.py | 35 ++++++++++++----- .../api/watch_histories/requirements.in | 3 +- .../api/watch_histories/requirements.txt | 4 +- .../databases/python/watch_history_db.py | 39 ++++++++++++++++++- 5 files changed, 74 insertions(+), 15 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index e0efa6ca..1aff3f97 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -3,15 +3,21 @@ import api import jwt_utils +from models import AddItem app = FastAPI() @app.get("/watch-histories/item") -def item(request: Request, api_name: str, api_id: str): +def get_item(request: Request, api_name: str, api_id: str): return api.get_item(request.state.username, api_name, api_id) +@app.post("/watch-histories/item") +def add_item(request: Request, item: AddItem): + return api.add_item(request.state.username, item.api_name, item.api_id) + + @app.middleware("http") def parse_token(request: Request, call_next): auth_header = request.headers.get("authorization") diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py index e92d1353..e3425375 100644 --- a/src/lambdas/api/watch_histories/api.py +++ b/src/lambdas/api/watch_histories/api.py @@ -1,8 +1,5 @@ -import json - from fastapi import HTTPException -import decimal_encoder import logger import tvmaze import watch_history_db @@ -12,18 +9,36 @@ def get_item(username, api_name, api_id): + api_ret = _get_api_res(api_name, api_id) + + try: + w_ret = watch_history_db.get_item_by_api_id( + username, + f"{api_name}_{api_id}", + ) + return {**w_ret, api_name: {**api_ret}} + except watch_history_db.NotFoundError: + raise HTTPException(status_code=404) + + +def add_item(username, api_name, api_id): + api_ret = _get_api_res(api_name, api_id) + + watch_history_db.add_item_v2( + username, + api_name, + api_id, + ) + return api_ret + + +def _get_api_res(api_name, api_id): try: if api_name == "tvmaze": - api_ret = tvmaze_api.get_show(api_id) + return tvmaze_api.get_show(api_id) else: raise HTTPException(status_code=501) except tvmaze.HTTPError as e: err_msg = f"Could not get item from {api_name} api with id: {api_id}" log.error(f"{err_msg}. Error: {str(e)}") raise HTTPException(status_code=e.code) - - try: - w_ret = watch_history_db.get_item_by_api_id(username, f"{api_name}_{api_id}") - return {**w_ret, api_name: {**api_ret}} - except watch_history_db.NotFoundError: - raise HTTPException(status_code=404) diff --git a/src/lambdas/api/watch_histories/requirements.in b/src/lambdas/api/watch_histories/requirements.in index 24556cc5..afa7c47a 100644 --- a/src/lambdas/api/watch_histories/requirements.in +++ b/src/lambdas/api/watch_histories/requirements.in @@ -1,2 +1,3 @@ fastapi==0.70.0 -mangum==0.12.3 \ No newline at end of file +mangum==0.12.3 +pydantic==1.8.2 \ No newline at end of file diff --git a/src/lambdas/api/watch_histories/requirements.txt b/src/lambdas/api/watch_histories/requirements.txt index 5a9d682f..f2c77d4d 100644 --- a/src/lambdas/api/watch_histories/requirements.txt +++ b/src/lambdas/api/watch_histories/requirements.txt @@ -43,7 +43,9 @@ pydantic==1.8.2 \ --hash=sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505 \ --hash=sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1 \ --hash=sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833 - # via fastapi + # via + # -r src/lambdas/api/watch_histories/requirements.in + # fastapi sniffio==1.2.0 \ --hash=sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663 \ --hash=sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 4276f7e0..c4c6b2a3 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -1,5 +1,6 @@ import os import time +import uuid from datetime import datetime import boto3 @@ -51,6 +52,34 @@ def _get_client(): return client +def add_item_v2(username, api_name, api_id): + api_info = f"{api_name}_{api_id}" + data = { + "api_info": api_info, + } + + if "dates_watched" not in data: + data["latest_watch_date"] = "0" + try: + get_item_by_api_id( + username, + api_info, + include_deleted=True, + ) + except NotFoundError: + data["created_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + item_id = _get_show_item_id(api_name, api_id) + update_item(username, None, item_id, data, + clean_whitelist=["deleted_at"]) + + +def _get_show_item_id(api_name, api_id): + show_namespace = uuid.UUID("6045673a-9dd2-451c-aa58-d94a217b993a") + api_uuid = uuid.uuid5(show_namespace, api_name) + return str(uuid.uuid5(api_uuid, api_id)) + + def add_item(username, collection_name, item_id, data=None): if data is None: data = {} @@ -89,11 +118,16 @@ def get_item(username, collection_name, item_id, include_deleted=False): return res["Items"][0] -def get_item_by_api_id(username, api_info): +def get_item_by_api_id(username, api_info, include_deleted=False): + filter_exp = None + if not include_deleted: + filter_exp = Attr("deleted_at").not_exists() + res = _get_table().query( IndexName="api_info", KeyConditionExpression=Key("username").eq(username) & Key("api_info").eq(api_info), + FilterExpression=filter_exp, ) if not res["Items"]: @@ -152,7 +186,8 @@ def change_watched_eps(username, collection_name, item_id, change, if item[f"{field_name}_count"] == 0: ep_progress = 0 else: - ep_progress = (item[f"watched_{field_name}s"] + (change)) / item[f"{field_name}_count"] + ep_progress = (item[f"watched_{field_name}s"] + (change)) / item[ + f"{field_name}_count"] ep_progress = round(ep_progress * 100, 2) _get_table().update_item( From 505bedafb499120002b87ffd946f218c42cd9124 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 22:56:12 +0200 Subject: [PATCH 22/75] Return collection_name temporary --- .../databases/python/watch_history_db.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index c4c6b2a3..9d4a1140 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -69,15 +69,23 @@ def add_item_v2(username, api_name, api_id): except NotFoundError: data["created_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - item_id = _get_show_item_id(api_name, api_id) - update_item(username, None, item_id, data, - clean_whitelist=["deleted_at"]) + # create legacy item properties + collection_name, item_id = _get_collection_and_item_id(api_name, api_id) + update_item( + username, + collection_name, + item_id, + data, + clean_whitelist=["deleted_at"], + ) + +def _get_collection_and_item_id(api_name, api_id): + if api_name == "tvmaze": + show_namespace = uuid.UUID("6045673a-9dd2-451c-aa58-d94a217b993a") + api_uuid = uuid.uuid5(show_namespace, api_name) + return "show", str(uuid.uuid5(api_uuid, api_id)) -def _get_show_item_id(api_name, api_id): - show_namespace = uuid.UUID("6045673a-9dd2-451c-aa58-d94a217b993a") - api_uuid = uuid.uuid5(show_namespace, api_name) - return str(uuid.uuid5(api_uuid, api_id)) def add_item(username, collection_name, item_id, data=None): From 1a03f8dd611a19cef76c429858ee4eeccb29ea40 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 22:58:28 +0200 Subject: [PATCH 23/75] Add pydantic models file --- src/lambdas/api/watch_histories/models.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/lambdas/api/watch_histories/models.py diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py new file mode 100644 index 00000000..f1e8d89b --- /dev/null +++ b/src/lambdas/api/watch_histories/models.py @@ -0,0 +1,15 @@ +from enum import Enum + +from pydantic import BaseModel + + +class ApiName(str, Enum): + tmdb = "tmdb" + tvmaze = "tvmaze" + anidb = "anidb" + + +class AddItem(BaseModel): + api_id: str + api_name: ApiName + From 50708af618e7a7857c946d6a1084d3bef398468c Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 23:00:57 +0200 Subject: [PATCH 24/75] Return 204 code for post item --- src/lambdas/api/watch_histories/__init__.py | 2 +- src/lambdas/api/watch_histories/api.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 1aff3f97..db411539 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -13,7 +13,7 @@ def get_item(request: Request, api_name: str, api_id: str): return api.get_item(request.state.username, api_name, api_id) -@app.post("/watch-histories/item") +@app.post("/watch-histories/item", status_code=204) def add_item(request: Request, item: AddItem): return api.add_item(request.state.username, item.api_name, item.api_id) diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py index e3425375..167424c0 100644 --- a/src/lambdas/api/watch_histories/api.py +++ b/src/lambdas/api/watch_histories/api.py @@ -22,14 +22,13 @@ def get_item(username, api_name, api_id): def add_item(username, api_name, api_id): - api_ret = _get_api_res(api_name, api_id) + _get_api_res(api_name, api_id) watch_history_db.add_item_v2( username, api_name, api_id, ) - return api_ret def _get_api_res(api_name, api_id): From 92a05acd4de166e15b156089da0b12c26a80ccec Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 23:17:25 +0200 Subject: [PATCH 25/75] Convert add_item data into watch histories items --- src/lambdas/api/watch_histories/__init__.py | 7 +- src/lambdas/api/watch_histories/api.py | 3 +- src/lambdas/api/watch_histories/models.py | 30 ++++-- .../api/watch_histories/requirements.in | 1 + .../api/watch_histories/requirements.txt | 97 +++++++++++++++++++ .../databases/python/watch_history_db.py | 9 +- 6 files changed, 132 insertions(+), 15 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index db411539..1fbd5a2f 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -3,7 +3,7 @@ import api import jwt_utils -from models import AddItem +from models import Item app = FastAPI() @@ -14,8 +14,9 @@ def get_item(request: Request, api_name: str, api_id: str): @app.post("/watch-histories/item", status_code=204) -def add_item(request: Request, item: AddItem): - return api.add_item(request.state.username, item.api_name, item.api_id) +def add_item(request: Request, item: Item): + d = item.dict(exclude={"api_name", "api_id"}) + return api.add_item(request.state.username, item.api_name, item.api_id, d) @app.middleware("http") diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py index 167424c0..b3e40504 100644 --- a/src/lambdas/api/watch_histories/api.py +++ b/src/lambdas/api/watch_histories/api.py @@ -21,13 +21,14 @@ def get_item(username, api_name, api_id): raise HTTPException(status_code=404) -def add_item(username, api_name, api_id): +def add_item(username, api_name, api_id, data): _get_api_res(api_name, api_id) watch_history_db.add_item_v2( username, api_name, api_id, + data ) diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index f1e8d89b..33c5332b 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -1,15 +1,31 @@ -from enum import Enum +from datetime import datetime +from enum import auto +from typing import Optional + +from fastapi_utils.enums import StrEnum from pydantic import BaseModel -class ApiName(str, Enum): - tmdb = "tmdb" - tvmaze = "tvmaze" - anidb = "anidb" +class ApiName(StrEnum): + tmdb = auto() + tvmaze = auto() + anidb = auto() + +class Status(StrEnum): + finished = auto() + following = auto() + watching = auto() + dropped = auto() + backlog = auto() -class AddItem(BaseModel): + +class Item(BaseModel): api_id: str api_name: ApiName - + rating: Optional[int] + overview: Optional[str] + review: Optional[str] + dates_watched: Optional[list[datetime]] + status: Optional[Status] diff --git a/src/lambdas/api/watch_histories/requirements.in b/src/lambdas/api/watch_histories/requirements.in index afa7c47a..0905d32f 100644 --- a/src/lambdas/api/watch_histories/requirements.in +++ b/src/lambdas/api/watch_histories/requirements.in @@ -1,3 +1,4 @@ fastapi==0.70.0 +fastapi-utils==0.2.1 mangum==0.12.3 pydantic==1.8.2 \ No newline at end of file diff --git a/src/lambdas/api/watch_histories/requirements.txt b/src/lambdas/api/watch_histories/requirements.txt index f2c77d4d..75931485 100644 --- a/src/lambdas/api/watch_histories/requirements.txt +++ b/src/lambdas/api/watch_histories/requirements.txt @@ -11,7 +11,65 @@ anyio==3.3.4 \ fastapi==0.70.0 \ --hash=sha256:66da43cfe5185ea1df99552acffd201f1832c6b364e0f4136c0a99f933466ced \ --hash=sha256:a36d5f2fad931aa3575c07a3472c784e81f3e664e3bb5c8b9c88d0ec1104f59c + # via + # -r src/lambdas/api/watch_histories/requirements.in + # fastapi-utils +fastapi-utils==0.2.1 \ + --hash=sha256:0e6c7fc1870b80e681494957abf65d4f4f42f4c7f70005918e9181b22f1bd759 \ + --hash=sha256:dd0be7dc7f03fa681b25487a206651d99f2330d5a567fb8ab6cb5f8a06a29360 # via -r src/lambdas/api/watch_histories/requirements.in +greenlet==1.1.2 \ + --hash=sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711 \ + --hash=sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd \ + --hash=sha256:049fe7579230e44daef03a259faa24511d10ebfa44f69411d99e6a184fe68073 \ + --hash=sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708 \ + --hash=sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67 \ + --hash=sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23 \ + --hash=sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1 \ + --hash=sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08 \ + --hash=sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd \ + --hash=sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa \ + --hash=sha256:356b3576ad078c89a6107caa9c50cc14e98e3a6c4874a37c3e0273e4baf33de8 \ + --hash=sha256:40b951f601af999a8bf2ce8c71e8aaa4e8c6f78ff8afae7b808aae2dc50d4c40 \ + --hash=sha256:572e1787d1460da79590bf44304abbc0a2da944ea64ec549188fa84d89bba7ab \ + --hash=sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6 \ + --hash=sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc \ + --hash=sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b \ + --hash=sha256:7418b6bfc7fe3331541b84bb2141c9baf1ec7132a7ecd9f375912eca810e714e \ + --hash=sha256:7cbd7574ce8e138bda9df4efc6bf2ab8572c9aff640d8ecfece1b006b68da963 \ + --hash=sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3 \ + --hash=sha256:833e1551925ed51e6b44c800e71e77dacd7e49181fdc9ac9a0bf3714d515785d \ + --hash=sha256:8639cadfda96737427330a094476d4c7a56ac03de7265622fcf4cfe57c8ae18d \ + --hash=sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28 \ + --hash=sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3 \ + --hash=sha256:903bbd302a2378f984aef528f76d4c9b1748f318fe1294961c072bdc7f2ffa3e \ + --hash=sha256:93f81b134a165cc17123626ab8da2e30c0455441d4ab5576eed73a64c025b25c \ + --hash=sha256:95e69877983ea39b7303570fa6760f81a3eec23d0e3ab2021b7144b94d06202d \ + --hash=sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0 \ + --hash=sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497 \ + --hash=sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee \ + --hash=sha256:aa5b467f15e78b82257319aebc78dd2915e4c1436c3c0d1ad6f53e47ba6e2713 \ + --hash=sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58 \ + --hash=sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a \ + --hash=sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06 \ + --hash=sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88 \ + --hash=sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4 \ + --hash=sha256:be5f425ff1f5f4b3c1e33ad64ab994eed12fc284a6ea71c5243fd564502ecbe5 \ + --hash=sha256:dd0b1e9e891f69e7675ba5c92e28b90eaa045f6ab134ffe70b52e948aa175b3c \ + --hash=sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a \ + --hash=sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1 \ + --hash=sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43 \ + --hash=sha256:eb6ea6da4c787111adf40f697b4e58732ee0942b5d3bd8f435277643329ba627 \ + --hash=sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b \ + --hash=sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168 \ + --hash=sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d \ + --hash=sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5 \ + --hash=sha256:f3acda1924472472ddd60c29e5b9db0cec629fbe3c5c5accb74d6d6d14773478 \ + --hash=sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf \ + --hash=sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce \ + --hash=sha256:fa877ca7f6b48054f847b61d6fa7bed5cebb663ebc55e018fda12db09dcc664c \ + --hash=sha256:fdcec0b8399108577ec290f55551d926d9a1fa6cad45882093a7a07ac5ec147b + # via sqlalchemy idna==3.3 \ --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d @@ -46,10 +104,49 @@ pydantic==1.8.2 \ # via # -r src/lambdas/api/watch_histories/requirements.in # fastapi + # fastapi-utils sniffio==1.2.0 \ --hash=sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663 \ --hash=sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de # via anyio +sqlalchemy==1.4.26 \ + --hash=sha256:07ac4461a1116b317519ddf6f34bcb00b011b5c1370ebeaaf56595504ffc7e84 \ + --hash=sha256:090536fd23bf49077ee94ff97142bc5ee8bad24294c3d7c8d5284267c885dde7 \ + --hash=sha256:1dee515578d04bc80c4f9a8c8cfe93f455db725059e885f1b1da174d91c4d077 \ + --hash=sha256:1ef37c9ec2015ce2f0dc1084514e197f2f199d3dc3514190db7620b78e6004c8 \ + --hash=sha256:295b90efef1278f27fe27d94a45460ae3c17f5c5c2b32c163e29c359740a1599 \ + --hash=sha256:2ce42ad1f59eb85c55c44fb505f8854081ee23748f76b62a7f569cfa9b6d0604 \ + --hash=sha256:2feb028dc75e13ba93456a42ac042b255bf94dbd692bf80b47b22653bb25ccf8 \ + --hash=sha256:31f4426cfad19b5a50d07153146b2bcb372a279975d5fa39f98883c0ef0f3313 \ + --hash=sha256:3c0c5f54560a92691d54b0768d67b4d3159e514b426cfcb1258af8c195577e8f \ + --hash=sha256:463ef692259ff8189be42223e433542347ae17e33f91c1013e9c5c64e2798088 \ + --hash=sha256:4a882dedb9dfa6f33524953c3e3d72bcf518a5defd6d5863150a821928b19ad3 \ + --hash=sha256:4c185c928e2638af9bae13acc3f70e0096eac76471a1101a10f96b80666b8270 \ + --hash=sha256:5039faa365e7522a8eb4736a54afd24a7e75dcc33b81ab2f0e6c456140f1ad64 \ + --hash=sha256:5c6774b34782116ad9bdec61c2dbce9faaca4b166a0bc8e7b03c2b870b121d94 \ + --hash=sha256:6bc7f9d7d90ef55e8c6db1308a8619cd8f40e24a34f759119b95e7284dca351a \ + --hash=sha256:7e8ef103eaa72a857746fd57dda5b8b5961e8e82a528a3f8b7e2884d8506f0b7 \ + --hash=sha256:7ef421c3887b39c6f352e5022a53ac18de8387de331130481cb956b2d029cad6 \ + --hash=sha256:908fad32c53b17aad12d722379150c3c5317c422437e44032256a77df1746292 \ + --hash=sha256:91efbda4e6d311812f23996242bad7665c1392209554f8a31ec6db757456db5c \ + --hash=sha256:a6506c17b0b6016656783232d0bdd03fd333f1f654d51a14d93223f953903646 \ + --hash=sha256:a95bf9c725012dcd7ea3cac16bf647054e0d62b31d67467d228338e6a163e4ff \ + --hash=sha256:ad7e403fc1e3cb76e802872694e30d6ca6129b9bc6ad4e7caa48ca35f8a144f8 \ + --hash=sha256:b86f762cee3709722ab4691981958cbec475ea43406a6916a7ec375db9cbd9e9 \ + --hash=sha256:ba84026e84379326bbf2f0c50792f2ae56ab9c01937df5597b6893810b8ca369 \ + --hash=sha256:bca660b76672e15d70a7dba5e703e1ce451a0257b6bd2028e62b0487885e8ae9 \ + --hash=sha256:c24c01dcd03426a5fe5ee7af735906bec6084977b9027a3605d11d949a565c01 \ + --hash=sha256:c2f2114b0968a280f94deeeaa31cfbac9175e6ac7bd3058b3ce6e054ecd762b3 \ + --hash=sha256:c46f013ff31b80cbe36410281675e1fb4eaf3e25c284fd8a69981c73f6fa4cb4 \ + --hash=sha256:c757ba1279b85b3460e72e8b92239dae6f8b060a75fb24b3d9be984dd78cfa55 \ + --hash=sha256:cc6b21f19bc9d4cd77cbcba5f3b260436ce033f1053cea225b6efea2603d201e \ + --hash=sha256:dbf588ab09e522ac2cbd010919a592c6aae2f15ccc3cd9a96d01c42fbc13f63e \ + --hash=sha256:de996756d894a2d52c132742e3b6d64ecd37e0919ddadf4dc3981818777c7e67 \ + --hash=sha256:e700d48056475d077f867e6a36e58546de71bdb6fdc3d34b879e3240827fefab \ + --hash=sha256:f1e97c5f36b94542f72917b62f3a2f92be914b2cf33b80fa69cede7529241d2a \ + --hash=sha256:fb2aa74a6e3c2cebea38dd21633671841fbe70ea486053cba33d68e3e22ccc0a \ + --hash=sha256:ff8f91a7b1c4a1c7772caa9efe640f2768828897044748f2458b708f1026e2d4 + # via fastapi-utils starlette==0.16.0 \ --hash=sha256:38eb24bf705a2c317e15868e384c1b8a12ca396e5a3c3a003db7e667c43f939f \ --hash=sha256:e1904b5d0007aee24bdd3c43994be9b3b729f4f58e740200de1d623f8c3a8870 diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 9d4a1140..c0dedca4 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -52,11 +52,12 @@ def _get_client(): return client -def add_item_v2(username, api_name, api_id): +def add_item_v2(username, api_name, api_id, data=None): + if data is None: + data = {} + api_info = f"{api_name}_{api_id}" - data = { - "api_info": api_info, - } + data["api_info"] = api_info if "dates_watched" not in data: data["latest_watch_date"] = "0" From b10bdf93642c786d0748de53990e49127dfc83e3 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 23:27:18 +0200 Subject: [PATCH 26/75] Get show episodes count during add_item --- src/lambdas/api/watch_histories/api.py | 53 +++++++++++++------ .../databases/python/watch_history_db.py | 10 ++-- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/api.py index b3e40504..0de733f5 100644 --- a/src/lambdas/api/watch_histories/api.py +++ b/src/lambdas/api/watch_histories/api.py @@ -9,12 +9,21 @@ def get_item(username, api_name, api_id): - api_ret = _get_api_res(api_name, api_id) + try: + if api_name == "tvmaze": + api_ret = tvmaze_api.get_show(api_id) + else: + raise HTTPException(status_code=501) + except tvmaze.HTTPError as e: + err_msg = f"Could not get item from {api_name} api with id: {api_id}" + log.error(f"{err_msg}. Error: {str(e)}") + raise HTTPException(status_code=e.code) try: w_ret = watch_history_db.get_item_by_api_id( username, - f"{api_name}_{api_id}", + api_name, + api_id, ) return {**w_ret, api_name: {**api_ret}} except watch_history_db.NotFoundError: @@ -22,23 +31,37 @@ def get_item(username, api_name, api_id): def add_item(username, api_name, api_id, data): - _get_api_res(api_name, api_id) - - watch_history_db.add_item_v2( - username, - api_name, - api_id, - data - ) - - -def _get_api_res(api_name, api_id): try: if api_name == "tvmaze": - return tvmaze_api.get_show(api_id) + res = tvmaze_api.get_show_episodes_count(api_id) else: raise HTTPException(status_code=501) except tvmaze.HTTPError as e: - err_msg = f"Could not get item from {api_name} api with id: {api_id}" + err_msg = f"Could not get show episodes in add_item func" \ + f" from {api_name} api with id: {api_id}" log.error(f"{err_msg}. Error: {str(e)}") raise HTTPException(status_code=e.code) + + try: + current_item = watch_history_db.get_item_by_api_id( + username, + api_name, + api_id, + include_deleted=True, + ) + except watch_history_db.NotFoundError: + current_item = {} + + data["ep_count"] = res.get("ep_count", 0) + data["special_count"] = res.get("special_count", 0) + data["ep_progress"] = current_item.get("ep_progress", 0) + data["special_progress"] = current_item.get("special_progress", 0) + data["watched_eps"] = current_item.get("watched_eps", 0) + data["watched_special"] = current_item.get("watched_special", 0) + + watch_history_db.add_item_v2( + username, + api_name, + api_id, + data + ) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index c0dedca4..fcbe5490 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -55,16 +55,15 @@ def _get_client(): def add_item_v2(username, api_name, api_id, data=None): if data is None: data = {} - - api_info = f"{api_name}_{api_id}" - data["api_info"] = api_info + data["api_info"] = f"{api_name}_{api_id}" if "dates_watched" not in data: data["latest_watch_date"] = "0" try: get_item_by_api_id( username, - api_info, + api_name, + api_id, include_deleted=True, ) except NotFoundError: @@ -127,7 +126,8 @@ def get_item(username, collection_name, item_id, include_deleted=False): return res["Items"][0] -def get_item_by_api_id(username, api_info, include_deleted=False): +def get_item_by_api_id(username, api_name, api_id, include_deleted=False): + api_info = f"{api_name}_{api_id}" filter_exp = None if not include_deleted: filter_exp = Attr("deleted_at").not_exists() From 1687b61b95f0c3b93c1a4354904f7e740e88bce8 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 23:35:36 +0200 Subject: [PATCH 27/75] Change reserved list keyword to List from typing --- src/lambdas/api/watch_histories/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index 33c5332b..7981f74d 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import auto -from typing import Optional +from typing import Optional, List from fastapi_utils.enums import StrEnum @@ -27,5 +27,5 @@ class Item(BaseModel): rating: Optional[int] overview: Optional[str] review: Optional[str] - dates_watched: Optional[list[datetime]] + dates_watched: Optional[List[datetime]] status: Optional[Status] From f716e74f022664917563a266f31534c0610b13bf Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 23:42:09 +0200 Subject: [PATCH 28/75] Let FastAPI handle CORS --- deploy/lib/watch_history.py | 2 +- src/lambdas/api/watch_histories/__init__.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 0aeb0249..70b08b29 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -454,7 +454,7 @@ def _create_gateway(self): routes = { "watch_histories": { - "method": ["GET", "POST", "DELETE"], + "method": ["ANY"], "route": "/{proxy+}", "target_lambda": self.lambdas["api-watch_histories"] }, diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 1fbd5a2f..88fdcafc 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,3 +1,5 @@ +from starlette.middleware.cors import CORSMiddleware + from fastapi import FastAPI, Request from mangum import Mangum @@ -7,6 +9,19 @@ app = FastAPI() +origins = [ + "https://moshan.tv", + "https://beta.moshan.tv", +] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["GET", "POST", "PUT", "DELETE"], + allow_headers=["authorization", "content-type"], +) + @app.get("/watch-histories/item") def get_item(request: Request, api_name: str, api_id: str): From de9f168074c82c206f48576c8ef721f7c18fa7e7 Mon Sep 17 00:00:00 2001 From: fulder Date: Fri, 29 Oct 2021 23:43:11 +0200 Subject: [PATCH 29/75] Fix invalid import --- src/lambdas/api/watch_histories/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 88fdcafc..43440163 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,6 +1,5 @@ -from starlette.middleware.cors import CORSMiddleware - from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware from mangum import Mangum import api From f9052cdda7acea50c10c746735f06de15e522001 Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 00:37:49 +0200 Subject: [PATCH 30/75] Revert CORS and fix base_path --- deploy/lib/watch_history.py | 2 +- src/lambdas/api/watch_histories/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 70b08b29..fad09fa1 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -454,7 +454,7 @@ def _create_gateway(self): routes = { "watch_histories": { - "method": ["ANY"], + "method": ["GET", "POST", "PUT", "DELETE"], "route": "/{proxy+}", "target_lambda": self.lambdas["api-watch_histories"] }, diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 43440163..38169c4d 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -41,4 +41,4 @@ def parse_token(request: Request, call_next): return call_next(request) -handle = Mangum(app, api_gateway_base_path="/prod/") +handle = Mangum(app, api_gateway_base_path="/prod") From 2054d55ad713fc1a3ae7de73d3b2684c7bbcdc76 Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 11:28:05 +0200 Subject: [PATCH 31/75] Remove Bearer from auth header --- src/lambdas/api/watch_histories/__init__.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 38169c4d..c7cb2243 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,5 +1,4 @@ from fastapi import FastAPI, Request -from fastapi.middleware.cors import CORSMiddleware from mangum import Mangum import api @@ -8,19 +7,6 @@ app = FastAPI() -origins = [ - "https://moshan.tv", - "https://beta.moshan.tv", -] - -app.add_middleware( - CORSMiddleware, - allow_origins=origins, - allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "DELETE"], - allow_headers=["authorization", "content-type"], -) - @app.get("/watch-histories/item") def get_item(request: Request, api_name: str, api_id: str): @@ -36,8 +22,7 @@ def add_item(request: Request, item: Item): @app.middleware("http") def parse_token(request: Request, call_next): auth_header = request.headers.get("authorization") - jwt_str = auth_header.split("Bearer ")[1] - request.state.username = jwt_utils.get_username(jwt_str) + request.state.username = jwt_utils.get_username(auth_header) return call_next(request) From 4f8449766a08255385591e17804167db9bdcceff Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 11:56:59 +0200 Subject: [PATCH 32/75] Don't set None filterexpression --- src/layers/databases/python/watch_history_db.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index fcbe5490..ed746bd6 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -87,7 +87,6 @@ def _get_collection_and_item_id(api_name, api_id): return "show", str(uuid.uuid5(api_uuid, api_id)) - def add_item(username, collection_name, item_id, data=None): if data is None: data = {} @@ -128,16 +127,16 @@ def get_item(username, collection_name, item_id, include_deleted=False): def get_item_by_api_id(username, api_name, api_id, include_deleted=False): api_info = f"{api_name}_{api_id}" - filter_exp = None + + kwargs = { + "IndexName": "api_info", + "KeyConditionExpression": Key("username").eq(username) & + Key("api_info").eq(api_info) + } if not include_deleted: - filter_exp = Attr("deleted_at").not_exists() + kwargs["FilterExpression"] = Attr("deleted_at").not_exists() - res = _get_table().query( - IndexName="api_info", - KeyConditionExpression=Key("username").eq(username) & - Key("api_info").eq(api_info), - FilterExpression=filter_exp, - ) + res = _get_table().query(**kwargs) if not res["Items"]: raise NotFoundError(f"Item with api_info: {api_info} not found") From 0e1a9645d87fd9112bbc3846be4e62807f47cc4d Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 12:01:26 +0200 Subject: [PATCH 33/75] Check if dates_watched is none --- src/layers/databases/python/watch_history_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index ed746bd6..689417e6 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -57,7 +57,7 @@ def add_item_v2(username, api_name, api_id, data=None): data = {} data["api_info"] = f"{api_name}_{api_id}" - if "dates_watched" not in data: + if data.get("dates_watched") is None: data["latest_watch_date"] = "0" try: get_item_by_api_id( @@ -149,7 +149,7 @@ def update_item(username, collection_name, item_id, data, data["collection_name"] = collection_name data["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if "dates_watched" in data: + if data.get("dates_watched") is None: m_d = max([dateutil.parser.parse(d) for d in data["dates_watched"]]) m_d = m_d.strftime("%Y-%m-%dT%H:%M:%S.%fZ") data["latest_watch_date"] = m_d.replace("000Z", "Z") From 835b188655bdddf94e51cee788e9fa7f8ba2a13b Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 12:04:43 +0200 Subject: [PATCH 34/75] Make dates_watched get consistent --- src/lambdas/api/episode_by_collection_item/__init__.py | 2 +- src/lambdas/api/episode_by_id/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambdas/api/episode_by_collection_item/__init__.py b/src/lambdas/api/episode_by_collection_item/__init__.py index 3e27c8eb..fb7a1f08 100644 --- a/src/lambdas/api/episode_by_collection_item/__init__.py +++ b/src/lambdas/api/episode_by_collection_item/__init__.py @@ -161,7 +161,7 @@ def _post_episode(username, collection_name, item_id, body, token): special=res["is_special"] ) - if "dates_watched" not in body: + if body.get("dates_watched") is None: return { "statusCode": 200, "body": json.dumps({"id": episode_id}) diff --git a/src/lambdas/api/episode_by_id/__init__.py b/src/lambdas/api/episode_by_id/__init__.py index 84644dba..0c1730ec 100644 --- a/src/lambdas/api/episode_by_id/__init__.py +++ b/src/lambdas/api/episode_by_id/__init__.py @@ -109,7 +109,7 @@ def _put_episode(username, collection_name, item_id, episode_id, body, token, ap body ) - if "dates_watched" not in body: + if body.get("dates_watched") is None: return {"statusCode": 204} # If episode watch date is changed check if its larger than current From 3594d2044179c38912c5761b6d7aa6eb5101de9b Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 12:05:54 +0200 Subject: [PATCH 35/75] Fix invalid dates_watched condition check --- src/layers/databases/python/watch_history_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 689417e6..48bbe67f 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -149,7 +149,7 @@ def update_item(username, collection_name, item_id, data, data["collection_name"] = collection_name data["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if data.get("dates_watched") is None: + if data.get("dates_watched") is not None: m_d = max([dateutil.parser.parse(d) for d in data["dates_watched"]]) m_d = m_d.strftime("%Y-%m-%dT%H:%M:%S.%fZ") data["latest_watch_date"] = m_d.replace("000Z", "Z") From e2830437622d785823d0cfca5f16868643bde9a1 Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 12:09:38 +0200 Subject: [PATCH 36/75] Allow fastAPI lambda to update items --- deploy/lib/watch_history.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index fad09fa1..3e78afd3 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -129,6 +129,10 @@ def _create_lambdas_config(self): f"{self.watch_history_table.table_arn}/index/*", ] ), + PolicyStatement( + actions=["dynamodb:UpdateItem"], + resources=[self.watch_history_table.table_arn], + ), ], "timeout": 60 }, From 44a0715bf253f641c44e8e8ddc7287c9a1246850 Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 13:00:52 +0200 Subject: [PATCH 37/75] Remove api file to routes --- src/lambdas/api/watch_histories/__init__.py | 6 +++--- src/lambdas/api/watch_histories/{api.py => routes.py} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename src/lambdas/api/watch_histories/{api.py => routes.py} (100%) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index c7cb2243..25891000 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,7 +1,7 @@ from fastapi import FastAPI, Request from mangum import Mangum -import api +import routes import jwt_utils from models import Item @@ -10,13 +10,13 @@ @app.get("/watch-histories/item") def get_item(request: Request, api_name: str, api_id: str): - return api.get_item(request.state.username, api_name, api_id) + return routes.get_item(request.state.username, api_name, api_id) @app.post("/watch-histories/item", status_code=204) def add_item(request: Request, item: Item): d = item.dict(exclude={"api_name", "api_id"}) - return api.add_item(request.state.username, item.api_name, item.api_id, d) + return routes.add_item(request.state.username, item.api_name, item.api_id, d) @app.middleware("http") diff --git a/src/lambdas/api/watch_histories/api.py b/src/lambdas/api/watch_histories/routes.py similarity index 100% rename from src/lambdas/api/watch_histories/api.py rename to src/lambdas/api/watch_histories/routes.py From 4c43c53371a9e2c925933f479da95c1e135bc879 Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 13:01:06 +0200 Subject: [PATCH 38/75] Fix issues with None data values in updat_item --- .../databases/python/watch_history_db.py | 17 +++++++--- test/unittest/test_watch_history_db.py | 32 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 48bbe67f..2a7a6417 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -154,10 +154,19 @@ def update_item(username, collection_name, item_id, data, m_d = m_d.strftime("%Y-%m-%dT%H:%M:%S.%fZ") data["latest_watch_date"] = m_d.replace("000Z", "Z") - items = ','.join(f'#{k}=:{k}' for k in data) - update_expression = f"SET {items}" - expression_attribute_names = {f'#{k}': k for k in data} - expression_attribute_values = {f':{k}': v for k, v in data.items()} + update_expression = "SET " + expression_attribute_names = {} + expression_attribute_values = {} + for k, v in data.items(): + if v is None: + continue + + update_expression += f"#{k}=:{k}," + expression_attribute_names[f"#{k}"] = k + expression_attribute_values[f":{k}"] = v + + # remove last comma + update_expression = update_expression[:-1] remove_names = [] for o in OPTIONAL_FIELDS: diff --git a/test/unittest/test_watch_history_db.py b/test/unittest/test_watch_history_db.py index fc8a16cf..3f98f051 100644 --- a/test/unittest/test_watch_history_db.py +++ b/test/unittest/test_watch_history_db.py @@ -258,6 +258,38 @@ def test_update_item_dates_watched_one_date(mocked_watch_history_db): } +def test_update_item_none_data(mocked_watch_history_db): + mock_func = MockFunc() + mocked_watch_history_db.table.update_item = mock_func.f + + mocked_watch_history_db.update_item(TEST_USERNAME, "MOVIE", "123", + {"review": "review_text", "ignore": None}) + + assert mock_func.update_values == { + 'ExpressionAttributeNames': { + '#collection_name': 'collection_name', + '#dates_watched': 'dates_watched', + '#deleted_at': 'deleted_at', + '#overview': 'overview', + '#rating': 'rating', + '#review': 'review', + '#status': 'status', + '#updated_at': 'updated_at' + }, + 'ExpressionAttributeValues': { + ':collection_name': 'MOVIE', + ":updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + ":review": "review_text" + }, + 'Key': { + 'username': TEST_USERNAME, + 'item_id': '123'}, + 'UpdateExpression': 'SET #review=:review,#collection_name=:collection_name,' + '#updated_at=:updated_at ' + 'REMOVE #deleted_at,#overview,#status,#rating,#dates_watched' + } + + def test_delete_item(mocked_watch_history_db): mock_func = MockFunc() mocked_watch_history_db.table.update_item = mock_func.f From a4cd8e9bc701cc716f50ea9086b0a15ca108b852 Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 13:30:00 +0200 Subject: [PATCH 39/75] Change query params to path variabels and start adding delete route --- src/lambdas/api/watch_histories/__init__.py | 14 ++++++++++++-- src/lambdas/api/watch_histories/routes.py | 3 +++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 25891000..88e307f9 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -8,15 +8,25 @@ app = FastAPI() -@app.get("/watch-histories/item") +@app.get("/watch-histories/item/{api_name}/{api_id}") def get_item(request: Request, api_name: str, api_id: str): return routes.get_item(request.state.username, api_name, api_id) +@app.delete("/watch-histories/item/{api_name}/{api_id}") +def delete_item(request: Request, api_name: str, api_id: str): + return routes.delete_item(request.state.username, api_name, api_id) + + @app.post("/watch-histories/item", status_code=204) def add_item(request: Request, item: Item): d = item.dict(exclude={"api_name", "api_id"}) - return routes.add_item(request.state.username, item.api_name, item.api_id, d) + return routes.add_item( + request.state.username, + item.api_name, + item.api_id, + d, + ) @app.middleware("http") diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 0de733f5..7e5d1384 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -65,3 +65,6 @@ def add_item(username, api_name, api_id, data): api_id, data ) + +def delete_item(username, api_name, api_id, data): + pass From d2c6bacee16181ba5a43158de1945609f9f84aac Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 13:34:19 +0200 Subject: [PATCH 40/75] Implement delete_item route --- src/lambdas/api/watch_histories/routes.py | 10 ++++++++-- src/layers/databases/python/watch_history_db.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 7e5d1384..967e8594 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -66,5 +66,11 @@ def add_item(username, api_name, api_id, data): data ) -def delete_item(username, api_name, api_id, data): - pass + +def delete_item(username, api_name, api_id): + collection_name, item_id = watch_history_db.get_collection_and_item_id( + api_name, + api_id, + ) + watch_history_db.delete_item(username, collection_name, item_id) + return {"statusCode": 204} diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 2a7a6417..7bf7c8cf 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -80,7 +80,7 @@ def add_item_v2(username, api_name, api_id, data=None): ) -def _get_collection_and_item_id(api_name, api_id): +def get_collection_and_item_id(api_name, api_id): if api_name == "tvmaze": show_namespace = uuid.UUID("6045673a-9dd2-451c-aa58-d94a217b993a") api_uuid = uuid.uuid5(show_namespace, api_name) From 170105802f0378de632a346af84d7b33d66dda52 Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 13:41:26 +0200 Subject: [PATCH 41/75] Fix invalid name to repo func --- src/layers/databases/python/watch_history_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 7bf7c8cf..8a6998c9 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -70,7 +70,7 @@ def add_item_v2(username, api_name, api_id, data=None): data["created_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") # create legacy item properties - collection_name, item_id = _get_collection_and_item_id(api_name, api_id) + collection_name, item_id = get_collection_and_item_id(api_name, api_id) update_item( username, collection_name, From 4fc8ede5813c87a8181c7ba2ad0c37934156c87b Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 13:54:01 +0200 Subject: [PATCH 42/75] Skip returning apidata in get item route --- src/lambdas/api/watch_histories/__init__.py | 2 +- src/lambdas/api/watch_histories/routes.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 88e307f9..cf829be7 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -13,7 +13,7 @@ def get_item(request: Request, api_name: str, api_id: str): return routes.get_item(request.state.username, api_name, api_id) -@app.delete("/watch-histories/item/{api_name}/{api_id}") +@app.delete("/watch-histories/item/{api_name}/{api_id}", status_code=204) def delete_item(request: Request, api_name: str, api_id: str): return routes.delete_item(request.state.username, api_name, api_id) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 967e8594..c9325389 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -11,7 +11,7 @@ def get_item(username, api_name, api_id): try: if api_name == "tvmaze": - api_ret = tvmaze_api.get_show(api_id) + tvmaze_api.get_show(api_id) else: raise HTTPException(status_code=501) except tvmaze.HTTPError as e: @@ -25,7 +25,7 @@ def get_item(username, api_name, api_id): api_name, api_id, ) - return {**w_ret, api_name: {**api_ret}} + return w_ret except watch_history_db.NotFoundError: raise HTTPException(status_code=404) @@ -73,4 +73,3 @@ def delete_item(username, api_name, api_id): api_id, ) watch_history_db.delete_item(username, collection_name, item_id) - return {"statusCode": 204} From 7e48dc55097a37dba81dd0a1b2b9df369d5e4cbb Mon Sep 17 00:00:00 2001 From: fulder Date: Sat, 30 Oct 2021 23:58:25 +0200 Subject: [PATCH 43/75] Implement put item route --- src/lambdas/api/watch_histories/__init__.py | 14 ++++++++++++-- src/lambdas/api/watch_histories/models.py | 7 +++++-- src/lambdas/api/watch_histories/routes.py | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index cf829be7..36610381 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -3,7 +3,7 @@ import routes import jwt_utils -from models import Item +from models import Item, PostItem app = FastAPI() @@ -18,8 +18,18 @@ def delete_item(request: Request, api_name: str, api_id: str): return routes.delete_item(request.state.username, api_name, api_id) +@app.put("/watch-histories/item/{api_name}/{api_id}", status_code=204) +def add_item(request: Request, api_name: str, api_id: str, item: Item): + return routes.update_item( + request.state.username, + api_name, + api_id, + item.dict(), + ) + + @app.post("/watch-histories/item", status_code=204) -def add_item(request: Request, item: Item): +def add_item(request: Request, item: PostItem): d = item.dict(exclude={"api_name", "api_id"}) return routes.add_item( request.state.username, diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index 7981f74d..64dea49f 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -22,10 +22,13 @@ class Status(StrEnum): class Item(BaseModel): - api_id: str - api_name: ApiName rating: Optional[int] overview: Optional[str] review: Optional[str] dates_watched: Optional[List[datetime]] status: Optional[Status] + + +class PostItem(Item): + api_id: str + api_name: ApiName diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index c9325389..2c7b8f75 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -67,6 +67,25 @@ def add_item(username, api_name, api_id, data): ) +def update_item(username, api_name, api_id, data): + all_none = True + for v in data.values(): + if v is not None: + all_none = False + if all_none: + raise HTTPException( + status_code=400, + detail="Please specify at least one of the optional fields" + ) + + watch_history_db.add_item_v2( + username, + api_name, + api_id, + data + ) + + def delete_item(username, api_name, api_id): collection_name, item_id = watch_history_db.get_collection_and_item_id( api_name, From ce8058d9744f73ca7fbb564fe4fe6561295cfdc6 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 00:02:03 +0200 Subject: [PATCH 44/75] Set ep data only for episode collection items --- src/lambdas/api/watch_histories/routes.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 2c7b8f75..b297bbb5 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -31,9 +31,10 @@ def get_item(username, api_name, api_id): def add_item(username, api_name, api_id, data): + ep_count_res = None try: if api_name == "tvmaze": - res = tvmaze_api.get_show_episodes_count(api_id) + ep_count_res = tvmaze_api.get_show_episodes_count(api_id) else: raise HTTPException(status_code=501) except tvmaze.HTTPError as e: @@ -52,12 +53,13 @@ def add_item(username, api_name, api_id, data): except watch_history_db.NotFoundError: current_item = {} - data["ep_count"] = res.get("ep_count", 0) - data["special_count"] = res.get("special_count", 0) - data["ep_progress"] = current_item.get("ep_progress", 0) - data["special_progress"] = current_item.get("special_progress", 0) - data["watched_eps"] = current_item.get("watched_eps", 0) - data["watched_special"] = current_item.get("watched_special", 0) + if ep_count_res is not None: + data["ep_count"] = ep_count_res.get("ep_count", 0) + data["special_count"] = ep_count_res.get("special_count", 0) + data["ep_progress"] = current_item.get("ep_progress", 0) + data["special_progress"] = current_item.get("special_progress", 0) + data["watched_eps"] = current_item.get("watched_eps", 0) + data["watched_special"] = current_item.get("watched_special", 0) watch_history_db.add_item_v2( username, From f91d137d84abd63be5d37229dda34a55fd0cedf6 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 00:18:14 +0200 Subject: [PATCH 45/75] Use update_item in put route --- src/lambdas/api/watch_histories/routes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index b297bbb5..ccb3b5f3 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -80,10 +80,14 @@ def update_item(username, api_name, api_id, data): detail="Please specify at least one of the optional fields" ) - watch_history_db.add_item_v2( - username, + collection_name, item_id = watch_history_db.get_collection_and_item_id( api_name, api_id, + ) + watch_history_db.update_item( + username, + collection_name, + item_id, data ) From 053f92c6d200ae8ec11712857c80f4bf7d5c5938 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 00:21:01 +0200 Subject: [PATCH 46/75] Rename update item func --- src/lambdas/api/watch_histories/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 36610381..f05d03e2 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -19,7 +19,7 @@ def delete_item(request: Request, api_name: str, api_id: str): @app.put("/watch-histories/item/{api_name}/{api_id}", status_code=204) -def add_item(request: Request, api_name: str, api_id: str, item: Item): +def update_item(request: Request, api_name: str, api_id: str, item: Item): return routes.update_item( request.state.username, api_name, From 8bfe4a54aead55b1e5edaffd02e8e410ce52f0ff Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 11:20:18 +0100 Subject: [PATCH 47/75] Add new indexes for episode table and api_info fields --- deploy/lib/watch_history.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 3e78afd3..32d7378a 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -108,6 +108,16 @@ def _create_tables(self): type=AttributeType.STRING), index_name="latest_watch_date" ) + self.episodes_table.add_global_secondary_index( + partition_key=Attribute(name="username", type=AttributeType.STRING), + sort_key=Attribute(name="item_api_info", type=AttributeType.STRING), + index_name="item_api_info" + ) + self.episodes_table.add_global_secondary_index( + partition_key=Attribute(name="username", type=AttributeType.STRING), + sort_key=Attribute(name="api_info", type=AttributeType.STRING), + index_name="api_info" + ) def _create_lambdas_config(self): self.lambdas_config = { From 9be3a3b4a465c29bbd4a893760cb9464f209cb4f Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 11:40:03 +0100 Subject: [PATCH 48/75] Implement get episode by api id using new GSI and migrated data --- src/lambdas/api/watch_histories/__init__.py | 9 +++++ src/lambdas/api/watch_histories/routes.py | 39 +++++++++++++++------ src/layers/databases/python/episodes_db.py | 32 ++++++++++++----- 3 files changed, 61 insertions(+), 19 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index f05d03e2..a3fa3632 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -39,6 +39,15 @@ def add_item(request: Request, item: PostItem): ) +@app.get("/watch-histories/episodes/{api_name}/{api_id}") +def get_episode(request: Request, api_name: str, api_id: str): + return routes.get_episode( + request.state.username, + api_name, + api_id, + ) + + @app.middleware("http") def parse_token(request: Request, call_next): auth_header = request.headers.get("authorization") diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index ccb3b5f3..456145f1 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -1,3 +1,4 @@ +import episodes_db from fastapi import HTTPException import logger @@ -9,15 +10,15 @@ def get_item(username, api_name, api_id): - try: - if api_name == "tvmaze": - tvmaze_api.get_show(api_id) - else: - raise HTTPException(status_code=501) - except tvmaze.HTTPError as e: - err_msg = f"Could not get item from {api_name} api with id: {api_id}" - log.error(f"{err_msg}. Error: {str(e)}") - raise HTTPException(status_code=e.code) + # try: + # if api_name == "tvmaze": + # tvmaze_api.get_show(api_id) + # else: + # raise HTTPException(status_code=501) + # except tvmaze.HTTPError as e: + # err_msg = f"Could not get item from {api_name} api with id: {api_id}" + # log.error(f"{err_msg}. Error: {str(e)}") + # raise HTTPException(status_code=e.code) try: w_ret = watch_history_db.get_item_by_api_id( @@ -31,7 +32,6 @@ def get_item(username, api_name, api_id): def add_item(username, api_name, api_id, data): - ep_count_res = None try: if api_name == "tvmaze": ep_count_res = tvmaze_api.get_show_episodes_count(api_id) @@ -98,3 +98,22 @@ def delete_item(username, api_name, api_id): api_id, ) watch_history_db.delete_item(username, collection_name, item_id) + + +# def get_episodes(username, api_name, api_id): +# episodes = episodes_db.get_episodes( +# username, +# collection_name, +# item_id +# ) + +def get_episode(username, api_name, api_id): + try: + w_ret = episodes_db.get_episode_by_api_id( + username, + api_name, + api_id + ) + return w_ret + except episodes_db.NotFoundError: + raise HTTPException(status_code=404) diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index 91f9bfa0..ad9a44fe 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -60,7 +60,8 @@ def add_episode(username, collection_name, item_id, episode_id, data=None): except NotFoundError: data["created_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - update_episode(username, collection_name, episode_id, data, clean_whitelist=["deleted_at"]) + update_episode(username, collection_name, episode_id, data, + clean_whitelist=["deleted_at"]) def delete_episode(username, collection_name, episode_id): @@ -87,6 +88,26 @@ def get_episode(username, collection_name, episode_id, include_deleted=False): return res["Items"][0] +def get_episode_by_api_id(username, api_name, api_id, include_deleted=False): + api_info = f"{api_name}_{api_id}" + + kwargs = { + "IndexName": "api_info", + "KeyConditionExpression": Key("username").eq(username) & + Key("api_info").eq(api_info) + } + if not include_deleted: + kwargs["FilterExpression"] = Attr("deleted_at").not_exists() + + res = _get_table().query(**kwargs) + + if not res["Items"]: + raise NotFoundError( + f"Episode with api id: {api_id} not found. Api name: {api_name}") + + return res["Items"][0] + + def update_episode(username, collection_name, episode_id, data, clean_whitelist=OPTIONAL_FIELDS): data["collection_name"] = collection_name @@ -125,14 +146,7 @@ def update_episode(username, collection_name, episode_id, data, ) -def get_episodes(username, collection_name, item_id, limit=100, start=1): - start_page = 0 - res = [] - - if start <= 0: - raise InvalidStartOffset - - total_pages = 0 +def get_episodes(username, api_name, api_id): for p in _episodes_generator(username, collection_name, item_id, limit=limit): total_pages += 1 From 6c026911edf8e7cfc2854d71e722278ca7078e32 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 11:58:48 +0100 Subject: [PATCH 49/75] Add route for getting all item episodes --- src/lambdas/api/watch_histories/__init__.py | 9 +++++ src/lambdas/api/watch_histories/routes.py | 12 +++--- src/layers/databases/python/episodes_db.py | 41 +++++---------------- 3 files changed, 24 insertions(+), 38 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index a3fa3632..8ac5a9a3 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -39,6 +39,15 @@ def add_item(request: Request, item: PostItem): ) +@app.get("/watch-histories/item/{api_name}/{api_id}/episodes") +def get_episodes(request: Request, api_name: str, api_id: str): + return routes.get_episodes( + request.state.username, + api_name, + api_id, + ) + + @app.get("/watch-histories/episodes/{api_name}/{api_id}") def get_episode(request: Request, api_name: str, api_id: str): return routes.get_episode( diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 456145f1..e8acf09b 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -100,12 +100,12 @@ def delete_item(username, api_name, api_id): watch_history_db.delete_item(username, collection_name, item_id) -# def get_episodes(username, api_name, api_id): -# episodes = episodes_db.get_episodes( -# username, -# collection_name, -# item_id -# ) +def get_episodes(username, api_name, api_id): + episodes = episodes_db.get_episodes( + username, + collection_name, + item_id + ) def get_episode(username, api_name, api_id): try: diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index ad9a44fe..58b4e134 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -146,53 +146,30 @@ def update_episode(username, collection_name, episode_id, data, ) -def get_episodes(username, api_name, api_id): - for p in _episodes_generator(username, collection_name, item_id, - limit=limit): - total_pages += 1 - start_page += 1 - if start_page == start: - res = p +def get_episodes(username, api_name, item_api_id): + item_api_info = f"{api_name}_{item_api_id}" - if start > start_page: - raise InvalidStartOffset - - log.debug(f"get_episodes response: {res}") - - if not res: - raise NotFoundError( - f"episodes for client with username: {username} and collection: {collection_name} not found") - - return { - "episodes": res, - "total_pages": total_pages - } - - -def _episodes_generator(username, collection_name, item_id, limit): paginator = _get_client().get_paginator('query') query_kwargs = { "TableName": DATABASE_NAME, + "IndexName": "item_api_info", "KeyConditionExpression": "username = :username", "ExpressionAttributeValues": { ":username": {"S": username}, - ":item_id": {"S": item_id}, - ":collection_name": {"S": collection_name}, + ":item_api_info": {"S": item_api_info}, }, - "Limit": limit, "ScanIndexForward": False, - "FilterExpression": "attribute_not_exists(deleted_at) and item_id = :item_id and collection_name = :collection_name", + "FilterExpression": "attribute_not_exists(deleted_at) and item_api_info = :item_api_info", } log.debug(f"Query kwargs: {query_kwargs}") page_iterator = paginator.paginate(**query_kwargs) + res = [] for p in page_iterator: - items = {} for i in p["Items"]: - item = json_util.loads(i) - episode_id = item.pop("id") - items[episode_id] = item - yield items + i = json_util.loads(i) + res.append(i) + return res From 2b0cae211e5ae4ad87835fde9208c4496bd16841 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 12:05:59 +0100 Subject: [PATCH 50/75] Add permissions to episode_table for fastAPI lambda --- deploy/lib/watch_history.py | 7 ++++++- src/lambdas/api/watch_histories/routes.py | 7 ++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 32d7378a..e06942dd 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -137,11 +137,16 @@ def _create_lambdas_config(self): resources=[ self.watch_history_table.table_arn, f"{self.watch_history_table.table_arn}/index/*", + self.episodes_table.table_arn, + f"{self.episodes_table.table_arn}/index/*", ] ), PolicyStatement( actions=["dynamodb:UpdateItem"], - resources=[self.watch_history_table.table_arn], + resources=[ + self.watch_history_table.table_arn, + self.episodes_table.table_arn + ], ), ], "timeout": 60 diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index e8acf09b..0a9768bc 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -101,12 +101,13 @@ def delete_item(username, api_name, api_id): def get_episodes(username, api_name, api_id): - episodes = episodes_db.get_episodes( + return episodes_db.get_episodes( username, - collection_name, - item_id + api_name, + api_id, ) + def get_episode(username, api_name, api_id): try: w_ret = episodes_db.get_episode_by_api_id( From 87e1fa43a4ef13ab91b070bdcab885dbb5da8094 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 12:14:35 +0100 Subject: [PATCH 51/75] Remove item_api_id from episode table --- deploy/lib/watch_history.py | 5 ----- src/layers/databases/python/episodes_db.py | 6 +++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index e06942dd..7a6caee7 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -108,11 +108,6 @@ def _create_tables(self): type=AttributeType.STRING), index_name="latest_watch_date" ) - self.episodes_table.add_global_secondary_index( - partition_key=Attribute(name="username", type=AttributeType.STRING), - sort_key=Attribute(name="item_api_info", type=AttributeType.STRING), - index_name="item_api_info" - ) self.episodes_table.add_global_secondary_index( partition_key=Attribute(name="username", type=AttributeType.STRING), sort_key=Attribute(name="api_info", type=AttributeType.STRING), diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index 58b4e134..6a61fb34 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -153,14 +153,14 @@ def get_episodes(username, api_name, item_api_id): query_kwargs = { "TableName": DATABASE_NAME, - "IndexName": "item_api_info", + "IndexName": "api_info", "KeyConditionExpression": "username = :username", "ExpressionAttributeValues": { ":username": {"S": username}, - ":item_api_info": {"S": item_api_info}, + ":api_info": {"S": item_api_info}, }, "ScanIndexForward": False, - "FilterExpression": "attribute_not_exists(deleted_at) and item_api_info = :item_api_info", + "FilterExpression": "attribute_not_exists(deleted_at) and begins_with(api_info, :api_info)", } log.debug(f"Query kwargs: {query_kwargs}") From 3b14635d3d48adf6873cc7f9a51873eb5efabb9c Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 12:20:47 +0100 Subject: [PATCH 52/75] Include item api_id in one episode get --- src/lambdas/api/watch_histories/__init__.py | 9 ++++++--- src/lambdas/api/watch_histories/routes.py | 5 +++-- src/layers/databases/python/episodes_db.py | 5 +++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 8ac5a9a3..e666bdcf 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -48,12 +48,15 @@ def get_episodes(request: Request, api_name: str, api_id: str): ) -@app.get("/watch-histories/episodes/{api_name}/{api_id}") -def get_episode(request: Request, api_name: str, api_id: str): +@app.get( + "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}") +def get_episode(request: Request, api_name: str, item_api_id: str, + episode_api_id: str): return routes.get_episode( request.state.username, api_name, - api_id, + item_api_id, + episode_api_id, ) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 0a9768bc..f227cbee 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -108,12 +108,13 @@ def get_episodes(username, api_name, api_id): ) -def get_episode(username, api_name, api_id): +def get_episode(username, api_name, item_api_id, episode_api_id): try: w_ret = episodes_db.get_episode_by_api_id( username, api_name, - api_id + item_api_id, + episode_api_id, ) return w_ret except episodes_db.NotFoundError: diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index 6a61fb34..8d598f28 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -88,8 +88,9 @@ def get_episode(username, collection_name, episode_id, include_deleted=False): return res["Items"][0] -def get_episode_by_api_id(username, api_name, api_id, include_deleted=False): - api_info = f"{api_name}_{api_id}" +def get_episode_by_api_id(username, api_name, api_id, episode_api_id, + include_deleted=False): + api_info = f"{api_name}_{api_id}_{episode_api_id}" kwargs = { "IndexName": "api_info", From 0f18bf2cf573eebd0ead72b6279b3e421a1699e2 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 12:28:51 +0100 Subject: [PATCH 53/75] Rename api_id to item_api_id in all routes --- src/lambdas/api/watch_histories/__init__.py | 28 ++++++++++----------- src/lambdas/api/watch_histories/models.py | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index e666bdcf..385db0bb 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -8,43 +8,43 @@ app = FastAPI() -@app.get("/watch-histories/item/{api_name}/{api_id}") -def get_item(request: Request, api_name: str, api_id: str): - return routes.get_item(request.state.username, api_name, api_id) +@app.get("/watch-histories/item/{api_name}/{item_api_id}") +def get_item(request: Request, api_name: str, item_api_id: str): + return routes.get_item(request.state.username, api_name, item_api_id) -@app.delete("/watch-histories/item/{api_name}/{api_id}", status_code=204) -def delete_item(request: Request, api_name: str, api_id: str): - return routes.delete_item(request.state.username, api_name, api_id) +@app.delete("/watch-histories/item/{api_name}/{item_api_id}", status_code=204) +def delete_item(request: Request, api_name: str, item_api_id: str): + return routes.delete_item(request.state.username, api_name, item_api_id) -@app.put("/watch-histories/item/{api_name}/{api_id}", status_code=204) -def update_item(request: Request, api_name: str, api_id: str, item: Item): +@app.put("/watch-histories/item/{api_name}/{item_api_id}", status_code=204) +def update_item(request: Request, api_name: str, item_api_id: str, item: Item): return routes.update_item( request.state.username, api_name, - api_id, + item_api_id, item.dict(), ) @app.post("/watch-histories/item", status_code=204) def add_item(request: Request, item: PostItem): - d = item.dict(exclude={"api_name", "api_id"}) + d = item.dict(exclude={"api_name", "item_api_id"}) return routes.add_item( request.state.username, item.api_name, - item.api_id, + item.item_api_id, d, ) -@app.get("/watch-histories/item/{api_name}/{api_id}/episodes") -def get_episodes(request: Request, api_name: str, api_id: str): +@app.get("/watch-histories/item/{api_name}/{item_api_id}/episodes") +def get_episodes(request: Request, api_name: str, item_api_id: str): return routes.get_episodes( request.state.username, api_name, - api_id, + item_api_id, ) diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index 64dea49f..a899eb54 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -30,5 +30,5 @@ class Item(BaseModel): class PostItem(Item): - api_id: str + item_api_id: str api_name: ApiName From 556acc4a2199212b8108a0d0f43922c5642e2e8c Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 12:57:14 +0100 Subject: [PATCH 54/75] Add first implementation of post episode --- src/lambdas/api/watch_histories/__init__.py | 14 ++++- src/lambdas/api/watch_histories/models.py | 4 ++ src/lambdas/api/watch_histories/routes.py | 64 +++++++++++++++++++++ src/layers/databases/python/episodes_db.py | 62 ++++++++++++++++++-- 4 files changed, 137 insertions(+), 7 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 385db0bb..07132915 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -3,7 +3,7 @@ import routes import jwt_utils -from models import Item, PostItem +from models import Item, PostItem, PostEpisode app = FastAPI() @@ -48,6 +48,18 @@ def get_episodes(request: Request, api_name: str, item_api_id: str): ) +@app.post("/watch-histories/item/{api_name}/{item_api_id}/episodes") +def add_episode(request: Request, api_name, item_api_id, episode: PostEpisode): + d = episode.dict(exclude={"episode_api_id"}) + return routes.add_episode( + request.state.username, + api_name, + item_api_id, + episode.episode_api_id, + d + ) + + @app.get( "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}") def get_episode(request: Request, api_name: str, item_api_id: str, diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index a899eb54..fea6d48e 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -32,3 +32,7 @@ class Item(BaseModel): class PostItem(Item): item_api_id: str api_name: ApiName + + +class PostEpisode(Item): + episode_api_id: str diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index f227cbee..fe0a729f 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -1,5 +1,6 @@ import episodes_db from fastapi import HTTPException +import dateutil.parser import logger import tvmaze @@ -119,3 +120,66 @@ def get_episode(username, api_name, item_api_id, episode_api_id): return w_ret except episodes_db.NotFoundError: raise HTTPException(status_code=404) + + +def add_episode(username, api_name, item_api_id, episode_api_id, data): + try: + if api_name == "tvmaze": + api_res = tvmaze_api.get_episode(episode_api_id) + is_special = api_res["type"] != "regular" + else: + raise HTTPException(status_code=501) + except tvmaze.HTTPError as e: + err_msg = f"Could not get show episode in add_episode func" \ + f" from {api_name} api with id: {episode_api_id}" + log.error(f"{err_msg}. Error: {str(e)}") + raise HTTPException(status_code=e.code) + + try: + item = watch_history_db.get_item_by_api_id( + username, + api_name, + item_api_id, + ) + except watch_history_db.NotFoundError: + err_msg = f"Item with api_id: {item_api_id} not found. " \ + f"Please add it to the watch-history before posting episode" + raise HTTPException(status_code=404, detail=err_msg) + + episodes_db.add_episode_v2( + username, + api_name, + item_api_id, + episode_api_id, + data + ) + + collection_name, item_id = watch_history_db.get_collection_and_item_id( + api_name, + item_api_id, + ) + watch_history_db.change_watched_eps( + username, + collection_name, + item_id, + 1, + special=is_special + ) + + if data.get("dates_watched") is None: + return + + # If episode watch date is changed check if its larger than current + # item latest date and update item if that's the case + ep_date = max([dateutil.parser.parse(d) for d in data["dates_watched"]]) + + if (item["latest_watch_date"] == "0" or + ep_date > dateutil.parser.parse(item["latest_watch_date"])): + ep_date = ep_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ").replace("000Z", "Z") + watch_history_db.update_item( + username, + collection_name, + item_id, + {"latest_watch_date": f"{ep_date}"}, + clean_whitelist=[], + ) diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index 8d598f28..2a5499e3 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -1,5 +1,6 @@ import os import time +import uuid from datetime import datetime import boto3 @@ -8,6 +9,7 @@ from dynamodb_json import json_util import logger +import watch_history_db DATABASE_NAME = os.getenv("EPISODES_DATABASE_NAME") OPTIONAL_FIELDS = [ @@ -114,13 +116,22 @@ def update_episode(username, collection_name, episode_id, data, data["collection_name"] = collection_name data["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if "dates_watched" in data: + if data.get("dates_watched") is not None: data["latest_watch_date"] = max(data["dates_watched"]) - items = ','.join(f'#{k}=:{k}' for k in data) - update_expression = f"SET {items}" - expression_attribute_names = {f'#{k}': k for k in data} - expression_attribute_values = {f':{k}': v for k, v in data.items()} + update_expression = "SET " + expression_attribute_names = {} + expression_attribute_values = {} + for k, v in data.items(): + if v is None: + continue + + update_expression += f"#{k}=:{k}," + expression_attribute_names[f"#{k}"] = k + expression_attribute_values[f":{k}"] = v + + # remove last comma + update_expression = update_expression[:-1] remove_names = [] for o in OPTIONAL_FIELDS: @@ -130,7 +141,7 @@ def update_episode(username, collection_name, episode_id, data, if len(remove_names) > 0: update_expression += f" REMOVE {','.join(remove_names)}" - log.debug("Running update_item") + log.debug("Running update_episode") log.debug(f"Update expression: {update_expression}") log.debug(f"Expression attribute names: {expression_attribute_names}") log.debug(f"Expression attribute values: {expression_attribute_values}") @@ -174,3 +185,42 @@ def get_episodes(username, api_name, item_api_id): i = json_util.loads(i) res.append(i) return res + + +def add_episode_v2(username, api_name, item_api_id, episode_api_id, data=None): + if data is None: + data = {} + data["api_info"] = f"{api_name}_{item_api_id}_{episode_api_id}" + + if data.get("dates_watched") is None: + data["latest_watch_date"] = "0" + try: + get_episode_by_api_id( + username, + api_name, + item_api_id, + episode_api_id, + include_deleted=True, + ) + except NotFoundError: + data["created_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # create legacy episode properties + collection_name, item_id = watch_history_db.get_collection_and_item_id( + api_name, + item_api_id, + ) + episode_id = get_episode_uuid(item_id, episode_api_id) + # --------------- + + update_episode( + username, + collection_name, + episode_id, + data, + clean_whitelist=["deleted_at"], + ) + + +def get_episode_uuid(show_uuid, api_id): + return str(uuid.uuid5(uuid.UUID(show_uuid), str(api_id))) From 2b3075f63db63934b5ec41a56e8ceeb67302b4d0 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 20:12:43 +0100 Subject: [PATCH 55/75] Implement delete_episode route and add db wrappers for v2 operations --- src/lambdas/api/watch_histories/__init__.py | 20 ++++++- src/lambdas/api/watch_histories/routes.py | 59 ++++++++++++------- src/layers/databases/python/episodes_db.py | 12 ++++ .../databases/python/watch_history_db.py | 34 ++++++++++- 4 files changed, 101 insertions(+), 24 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 07132915..54c84145 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -31,7 +31,7 @@ def update_item(request: Request, api_name: str, item_api_id: str, item: Item): @app.post("/watch-histories/item", status_code=204) def add_item(request: Request, item: PostItem): d = item.dict(exclude={"api_name", "item_api_id"}) - return routes.add_item( + routes.add_item( request.state.username, item.api_name, item.item_api_id, @@ -48,10 +48,11 @@ def get_episodes(request: Request, api_name: str, item_api_id: str): ) -@app.post("/watch-histories/item/{api_name}/{item_api_id}/episodes") +@app.post("/watch-histories/item/{api_name}/{item_api_id}/episodes", + status_code=204) def add_episode(request: Request, api_name, item_api_id, episode: PostEpisode): d = episode.dict(exclude={"episode_api_id"}) - return routes.add_episode( + routes.add_episode( request.state.username, api_name, item_api_id, @@ -72,6 +73,19 @@ def get_episode(request: Request, api_name: str, item_api_id: str, ) +@app.delete( + "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}", + status_code=204) +def delete_episode(request: Request, api_name: str, item_api_id: str, + episode_api_id: str): + routes.delete_episode( + request.state.username, + api_name, + item_api_id, + episode_api_id, + ) + + @app.middleware("http") def parse_token(request: Request, call_next): auth_header = request.headers.get("authorization") diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index fe0a729f..40fcdf66 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -81,24 +81,16 @@ def update_item(username, api_name, api_id, data): detail="Please specify at least one of the optional fields" ) - collection_name, item_id = watch_history_db.get_collection_and_item_id( + watch_history_db.update_item_v2( + username, api_name, api_id, - ) - watch_history_db.update_item( - username, - collection_name, - item_id, data ) def delete_item(username, api_name, api_id): - collection_name, item_id = watch_history_db.get_collection_and_item_id( - api_name, - api_id, - ) - watch_history_db.delete_item(username, collection_name, item_id) + watch_history_db.delete_item_v2(username, api_name, api_id) def get_episodes(username, api_name, api_id): @@ -154,14 +146,10 @@ def add_episode(username, api_name, item_api_id, episode_api_id, data): data ) - collection_name, item_id = watch_history_db.get_collection_and_item_id( + watch_history_db.change_watched_eps_v2( + username, api_name, item_api_id, - ) - watch_history_db.change_watched_eps( - username, - collection_name, - item_id, 1, special=is_special ) @@ -176,10 +164,41 @@ def add_episode(username, api_name, item_api_id, episode_api_id, data): if (item["latest_watch_date"] == "0" or ep_date > dateutil.parser.parse(item["latest_watch_date"])): ep_date = ep_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ").replace("000Z", "Z") - watch_history_db.update_item( + watch_history_db.update_item_v2( username, - collection_name, - item_id, + api_name, + item_api_id, {"latest_watch_date": f"{ep_date}"}, clean_whitelist=[], ) + + +def delete_episode(username, api_name, item_api_id, episode_api_id): + try: + if api_name == "tvmaze": + api_res = tvmaze_api.get_episode(episode_api_id) + is_special = api_res["type"] != "regular" + else: + raise HTTPException(status_code=501) + except tvmaze.HTTPError as e: + err_msg = f"Could not get show episode in add_episode func" \ + f" from {api_name} api with id: {episode_api_id}" + log.error(f"{err_msg}. Error: {str(e)}") + raise HTTPException(status_code=e.code) + + episodes_db.delete_episode_v2( + username, + api_name, + item_api_id, + episode_api_id, + ) + + watch_history_db.change_watched_eps_v2( + username, + api_name, + item_api_id, + -1, + special=is_special + ) + + diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index 2a5499e3..8cb4e9e5 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -72,6 +72,18 @@ def delete_episode(username, collection_name, episode_id): clean_whitelist=[]) +def delete_episode_v2(username, api_name, item_api_id, episode_api_id): + # create legacy episode properties + collection_name, item_id = watch_history_db.get_collection_and_item_id( + api_name, + item_api_id, + ) + episode_id = get_episode_uuid(item_id, episode_api_id) + # --------------- + + delete_episode(username, collection_name, episode_id) + + def get_episode(username, collection_name, episode_id, include_deleted=False): filter_exp = Attr("collection_name").eq(collection_name) if not include_deleted: diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 8a6998c9..88260c65 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -80,6 +80,18 @@ def add_item_v2(username, api_name, api_id, data=None): ) +def update_item_v2(username, api_name, api_id, data, + clean_whitelist=OPTIONAL_FIELDS): + collection_name, item_id = get_collection_and_item_id(api_name, api_id) + update_item( + username, + collection_name, + item_id, + data, + clean_whitelist=clean_whitelist, + ) + + def get_collection_and_item_id(api_name, api_id): if api_name == "tvmaze": show_namespace = uuid.UUID("6045673a-9dd2-451c-aa58-d94a217b993a") @@ -107,6 +119,15 @@ def delete_item(username, collection_name, item_id): update_item(username, collection_name, item_id, data, clean_whitelist=[]) +def delete_item_v2(username, api_name, api_id): + collection_name, item_id = get_collection_and_item_id(api_name, api_id) + delete_item( + username, + collection_name, + item_id, + ) + + def get_item(username, collection_name, item_id, include_deleted=False): filter_exp = Attr("collection_name").eq(collection_name) if not include_deleted: @@ -200,7 +221,7 @@ def change_watched_eps(username, collection_name, item_id, change, field_name = "special" item = get_item(username, collection_name, item_id) - if item[f"{field_name}_count"] == 0: + if f"{field_name}_count" not in item or item[f"{field_name}_count"] == 0: ep_progress = 0 else: ep_progress = (item[f"watched_{field_name}s"] + (change)) / item[ @@ -224,6 +245,17 @@ def change_watched_eps(username, collection_name, item_id, change, ) +def change_watched_eps_v2(username, api_name, api_id, change, special=False): + collection_name, item_id = get_collection_and_item_id(api_name, api_id) + change_watched_eps( + username, + collection_name, + item_id, + change, + special=special + ) + + def get_watch_history(username, collection_name=None, index_name=None, status_filter=None): paginator = _get_client().get_paginator('query') From 77301bda4fdd45918fe07ebf4b499d090063f385 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 20:23:32 +0100 Subject: [PATCH 56/75] Remove dependency to show service in cron/subscribers --- deploy/lib/watch_history.py | 4 ++++ src/lambdas/cron/show_updates/__init__.py | 2 +- src/lambdas/subscribers/show_updates/__init__.py | 14 ++++++++------ src/layers/databases/python/watch_history_db.py | 13 +++++++++++++ 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/deploy/lib/watch_history.py b/deploy/lib/watch_history.py index 7a6caee7..95c94fc4 100644 --- a/deploy/lib/watch_history.py +++ b/deploy/lib/watch_history.py @@ -113,6 +113,10 @@ def _create_tables(self): sort_key=Attribute(name="api_info", type=AttributeType.STRING), index_name="api_info" ) + self.episodes_table.add_global_secondary_index( + partition_key=Attribute(name="api_info", type=AttributeType.STRING), + index_name="all_api_info" + ) def _create_lambdas_config(self): self.lambdas_config = { diff --git a/src/lambdas/cron/show_updates/__init__.py b/src/lambdas/cron/show_updates/__init__.py index 174d9a17..e1d2dcae 100644 --- a/src/lambdas/cron/show_updates/__init__.py +++ b/src/lambdas/cron/show_updates/__init__.py @@ -10,7 +10,7 @@ def handle(event, context): for tvmaze_id in tvmaze_updates: try: - watch_history_db.get_items_by_id(tvmaze_id) + watch_history_db.get_items_by_api_id("tvmaze", tvmaze_id) except HTTPError: # Show not present in db, exclude it from updates continue diff --git a/src/lambdas/subscribers/show_updates/__init__.py b/src/lambdas/subscribers/show_updates/__init__.py index 5a371dc4..f4918276 100644 --- a/src/lambdas/subscribers/show_updates/__init__.py +++ b/src/lambdas/subscribers/show_updates/__init__.py @@ -1,18 +1,20 @@ import json -import shows_api +import tvmaze import watch_history_db +tvmaze_api = tvmaze.TvMazeApi() + def handle(event, context): message = json.loads(event["Records"][0]["Sns"]["Message"]) - show = shows_api.get_show_by_api_id( + items = watch_history_db.get_items_by_api_id( message["api_name"], - message["api_id"], + message["api_id"] ) + tvmaze_item = tvmaze_api.get_show_episodes_count(message["api_id"]) - items = watch_history_db.get_items_by_id(show["id"]) for item in items: print(f"Updating item: {item}") @@ -21,14 +23,14 @@ def handle(event, context): if "watched_specials" not in item: item["watched_specials"] = 0 - item["ep_count"] = show["ep_count"] + item["ep_count"] = tvmaze_item["ep_count"] if item["ep_count"] == 0: ep_progress = 0 else: ep_progress = item["watched_eps"] / item["ep_count"] item["ep_progress"] = round(ep_progress * 100, 2) - item["special_count"] = show["special_count"] + item["special_count"] = tvmaze_item["special_count"] if item["special_count"] == 0: special_progress = 0 else: diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 88260c65..92f9e936 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -165,6 +165,19 @@ def get_item_by_api_id(username, api_name, api_id, include_deleted=False): return res["Items"][0] +def get_items_by_api_id(api_name, api_id): + api_info = f"{api_name}_{api_id}" + res = _get_table().query( + IndexName="all_api_info", + KeyConditionExpression=Key("api_info").eq(api_info), + ) + + if not res["Items"]: + raise NotFoundError(f"Item with api_info: {api_info} not found.") + + return res["Items"] + + def update_item(username, collection_name, item_id, data, clean_whitelist=OPTIONAL_FIELDS): data["collection_name"] = collection_name From 108c27162260b0a4c191fe3a64ea7362323f1a05 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 20:46:14 +0100 Subject: [PATCH 57/75] Implement route for updating episode --- src/lambdas/api/watch_histories/__init__.py | 17 ++++++++++- src/lambdas/api/watch_histories/routes.py | 32 +++++++++++++++++++-- src/layers/databases/python/episodes_db.py | 17 +++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 54c84145..2f1423b0 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -73,11 +73,26 @@ def get_episode(request: Request, api_name: str, item_api_id: str, ) +@app.put( + "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}", + status_code=204) +def update_episode(request: Request, api_name: str, item_api_id: str, + episode_api_id: str, episode: PostEpisode): + d = episode.dict(exclude={"episode_api_id"}) + return routes.update_episode( + request.state.username, + api_name, + item_api_id, + episode_api_id, + d, + ) + + @app.delete( "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}", status_code=204) def delete_episode(request: Request, api_name: str, item_api_id: str, - episode_api_id: str): + episode_api_id: str): routes.delete_episode( request.state.username, api_name, diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 40fcdf66..5f24c56f 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -157,6 +157,36 @@ def add_episode(username, api_name, item_api_id, episode_api_id, data): if data.get("dates_watched") is None: return + _update_latest_watch_date(item, data, username, api_name, item_api_id) + + +def update_episode(username, api_name, item_api_id, episode_api_id, data): + try: + item = watch_history_db.get_item_by_api_id( + username, + api_name, + item_api_id, + ) + except watch_history_db.NotFoundError: + err_msg = f"Item with api_id: {item_api_id} not found. " \ + f"Please add it to the watch-history before posting episode" + raise HTTPException(status_code=404, detail=err_msg) + + episodes_db.update_episode_v2( + username, + api_name, + item_api_id, + episode_api_id, + data + ) + + if data.get("dates_watched") is None: + return + + _update_latest_watch_date(item, data, username, api_name, item_api_id) + + +def _update_latest_watch_date(item, data, username, api_name, item_api_id): # If episode watch date is changed check if its larger than current # item latest date and update item if that's the case ep_date = max([dateutil.parser.parse(d) for d in data["dates_watched"]]) @@ -200,5 +230,3 @@ def delete_episode(username, api_name, item_api_id, episode_api_id): -1, special=is_special ) - - diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index 8cb4e9e5..c88a44de 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -170,6 +170,23 @@ def update_episode(username, collection_name, episode_id, data, ) +def update_episode_v2(username, api_name, item_api_id, episode_api_id, data): + # create legacy episode properties + collection_name, item_id = watch_history_db.get_collection_and_item_id( + api_name, + item_api_id, + ) + episode_id = get_episode_uuid(item_id, episode_api_id) + # --------------- + + update_episode( + username, + collection_name, + episode_id, + data, + ) + + def get_episodes(username, api_name, item_api_id): item_api_info = f"{api_name}_{item_api_id}" From 68b4359a7681ac755d034ae29cac0a1d497be895 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 20:48:06 +0100 Subject: [PATCH 58/75] Check if episode really exists in tvmaze --- src/lambdas/api/watch_histories/routes.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 5f24c56f..143aaa55 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -161,6 +161,17 @@ def add_episode(username, api_name, item_api_id, episode_api_id, data): def update_episode(username, api_name, item_api_id, episode_api_id, data): + try: + if api_name == "tvmaze": + tvmaze_api.get_episode(episode_api_id) + else: + raise HTTPException(status_code=501) + except tvmaze.HTTPError as e: + err_msg = f"Could not get show episode in add_episode func" \ + f" from {api_name} api with id: {episode_api_id}" + log.error(f"{err_msg}. Error: {str(e)}") + raise HTTPException(status_code=e.code) + try: item = watch_history_db.get_item_by_api_id( username, From eb1cd7314775feb6cd62ef7fcaf0ef4d09338374 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 20:52:02 +0100 Subject: [PATCH 59/75] Rename Item model to ReviewData --- src/lambdas/api/watch_histories/__init__.py | 12 ++++++------ src/lambdas/api/watch_histories/models.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 2f1423b0..bb44384c 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -3,7 +3,7 @@ import routes import jwt_utils -from models import Item, PostItem, PostEpisode +from models import PostItem, PostEpisode, ReviewData app = FastAPI() @@ -19,12 +19,13 @@ def delete_item(request: Request, api_name: str, item_api_id: str): @app.put("/watch-histories/item/{api_name}/{item_api_id}", status_code=204) -def update_item(request: Request, api_name: str, item_api_id: str, item: Item): +def update_item(request: Request, api_name: str, item_api_id: str, + data: ReviewData): return routes.update_item( request.state.username, api_name, item_api_id, - item.dict(), + data.dict(), ) @@ -77,14 +78,13 @@ def get_episode(request: Request, api_name: str, item_api_id: str, "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}", status_code=204) def update_episode(request: Request, api_name: str, item_api_id: str, - episode_api_id: str, episode: PostEpisode): - d = episode.dict(exclude={"episode_api_id"}) + episode_api_id: str, data: ReviewData): return routes.update_episode( request.state.username, api_name, item_api_id, episode_api_id, - d, + data.dict(), ) diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index fea6d48e..b4d769e5 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -21,7 +21,7 @@ class Status(StrEnum): backlog = auto() -class Item(BaseModel): +class ReviewData(BaseModel): rating: Optional[int] overview: Optional[str] review: Optional[str] @@ -29,10 +29,10 @@ class Item(BaseModel): status: Optional[Status] -class PostItem(Item): +class PostItem(ReviewData): item_api_id: str api_name: ApiName -class PostEpisode(Item): +class PostEpisode(ReviewData): episode_api_id: str From 1901540fd1c9d65b3faef31119c6b6322b0f8827 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:03:59 +0100 Subject: [PATCH 60/75] Use fastapi jsonable_encoder for boto data --- src/lambdas/api/watch_histories/__init__.py | 10 ++++++---- src/lambdas/api/watch_histories/models.py | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index bb44384c..fb97b5ae 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,3 +1,5 @@ +from fastapi.encoders import jsonable_encoder + from fastapi import FastAPI, Request from mangum import Mangum @@ -25,7 +27,7 @@ def update_item(request: Request, api_name: str, item_api_id: str, request.state.username, api_name, item_api_id, - data.dict(), + jsonable_encoder(data), ) @@ -36,7 +38,7 @@ def add_item(request: Request, item: PostItem): request.state.username, item.api_name, item.item_api_id, - d, + jsonable_encoder(d), ) @@ -58,7 +60,7 @@ def add_episode(request: Request, api_name, item_api_id, episode: PostEpisode): api_name, item_api_id, episode.episode_api_id, - d + jsonable_encoder(d), ) @@ -84,7 +86,7 @@ def update_episode(request: Request, api_name: str, item_api_id: str, api_name, item_api_id, episode_api_id, - data.dict(), + jsonable_encoder(data), ) diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index b4d769e5..675ee3ea 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -2,6 +2,8 @@ from enum import auto from typing import Optional, List +from fastapi.encoders import jsonable_encoder + from fastapi_utils.enums import StrEnum from pydantic import BaseModel From 72574292055cef5fdf21950243c007ee2595a60d Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:32:36 +0100 Subject: [PATCH 61/75] Add helper func to convert ReviewData into dict --- src/lambdas/api/watch_histories/__init__.py | 14 +++++--------- src/lambdas/api/watch_histories/models.py | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index fb97b5ae..16785fa3 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -1,11 +1,9 @@ -from fastapi.encoders import jsonable_encoder - from fastapi import FastAPI, Request from mangum import Mangum import routes import jwt_utils -from models import PostItem, PostEpisode, ReviewData +from models import PostItem, PostEpisode, ReviewData, review_data_to_dict app = FastAPI() @@ -27,18 +25,17 @@ def update_item(request: Request, api_name: str, item_api_id: str, request.state.username, api_name, item_api_id, - jsonable_encoder(data), + review_data_to_dict(data), ) @app.post("/watch-histories/item", status_code=204) def add_item(request: Request, item: PostItem): - d = item.dict(exclude={"api_name", "item_api_id"}) routes.add_item( request.state.username, item.api_name, item.item_api_id, - jsonable_encoder(d), + review_data_to_dict(item), ) @@ -54,13 +51,12 @@ def get_episodes(request: Request, api_name: str, item_api_id: str): @app.post("/watch-histories/item/{api_name}/{item_api_id}/episodes", status_code=204) def add_episode(request: Request, api_name, item_api_id, episode: PostEpisode): - d = episode.dict(exclude={"episode_api_id"}) routes.add_episode( request.state.username, api_name, item_api_id, episode.episode_api_id, - jsonable_encoder(d), + review_data_to_dict(episode), ) @@ -86,7 +82,7 @@ def update_episode(request: Request, api_name: str, item_api_id: str, api_name, item_api_id, episode_api_id, - jsonable_encoder(data), + review_data_to_dict(data), ) diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index 675ee3ea..1a096c72 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -2,8 +2,6 @@ from enum import auto from typing import Optional, List -from fastapi.encoders import jsonable_encoder - from fastapi_utils.enums import StrEnum from pydantic import BaseModel @@ -38,3 +36,15 @@ class PostItem(ReviewData): class PostEpisode(ReviewData): episode_api_id: str + + +def review_data_to_dict(data: ReviewData): + d = data.dict(exclude={"api_name", "item_api_id", "episode_api_id"}) + dates = d.get("dates_watched") + if dates: + parsed_dates = [] + for d in dates: + new_d = d.strftime("%Y-%m-%dT%H:%M:%S.%fZ").replace("000Z", "Z") + parsed_dates.append(new_d) + d["dates_watched"] = parsed_dates + return d From 78b56f40cccae3135ce442a6e1e7313172868324 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:33:55 +0100 Subject: [PATCH 62/75] Ignore empty dates_watched properties --- src/layers/databases/python/episodes_db.py | 2 +- src/layers/databases/python/watch_history_db.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index c88a44de..2b24f710 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -128,7 +128,7 @@ def update_episode(username, collection_name, episode_id, data, data["collection_name"] = collection_name data["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if data.get("dates_watched") is not None: + if data.get("dates_watched"): data["latest_watch_date"] = max(data["dates_watched"]) update_expression = "SET " diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 92f9e936..9a2824cd 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -183,7 +183,7 @@ def update_item(username, collection_name, item_id, data, data["collection_name"] = collection_name data["updated_at"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if data.get("dates_watched") is not None: + if data.get("dates_watched"): m_d = max([dateutil.parser.parse(d) for d in data["dates_watched"]]) m_d = m_d.strftime("%Y-%m-%dT%H:%M:%S.%fZ") data["latest_watch_date"] = m_d.replace("000Z", "Z") From 24c8a7e840759026c41e67c8815851d60f33e91d Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:38:09 +0100 Subject: [PATCH 63/75] Fix colliding var names --- src/lambdas/api/watch_histories/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lambdas/api/watch_histories/models.py b/src/lambdas/api/watch_histories/models.py index 1a096c72..72e14a32 100644 --- a/src/lambdas/api/watch_histories/models.py +++ b/src/lambdas/api/watch_histories/models.py @@ -39,12 +39,12 @@ class PostEpisode(ReviewData): def review_data_to_dict(data: ReviewData): - d = data.dict(exclude={"api_name", "item_api_id", "episode_api_id"}) - dates = d.get("dates_watched") + data = data.dict(exclude={"api_name", "item_api_id", "episode_api_id"}) + dates = data.get("dates_watched") if dates: parsed_dates = [] for d in dates: new_d = d.strftime("%Y-%m-%dT%H:%M:%S.%fZ").replace("000Z", "Z") parsed_dates.append(new_d) - d["dates_watched"] = parsed_dates - return d + data["dates_watched"] = parsed_dates + return data From 33656c121c7966d1b379324819a06506e4030126 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:43:43 +0100 Subject: [PATCH 64/75] Check for empty dates_watched --- src/lambdas/api/watch_histories/routes.py | 4 ++-- src/layers/databases/python/episodes_db.py | 2 +- src/layers/databases/python/watch_history_db.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index 143aaa55..c397ae11 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -154,7 +154,7 @@ def add_episode(username, api_name, item_api_id, episode_api_id, data): special=is_special ) - if data.get("dates_watched") is None: + if data.get("dates_watched"): return _update_latest_watch_date(item, data, username, api_name, item_api_id) @@ -191,7 +191,7 @@ def update_episode(username, api_name, item_api_id, episode_api_id, data): data ) - if data.get("dates_watched") is None: + if data.get("dates_watched"): return _update_latest_watch_date(item, data, username, api_name, item_api_id) diff --git a/src/layers/databases/python/episodes_db.py b/src/layers/databases/python/episodes_db.py index 2b24f710..02ce0d45 100644 --- a/src/layers/databases/python/episodes_db.py +++ b/src/layers/databases/python/episodes_db.py @@ -221,7 +221,7 @@ def add_episode_v2(username, api_name, item_api_id, episode_api_id, data=None): data = {} data["api_info"] = f"{api_name}_{item_api_id}_{episode_api_id}" - if data.get("dates_watched") is None: + if data.get("dates_watched"): data["latest_watch_date"] = "0" try: get_episode_by_api_id( diff --git a/src/layers/databases/python/watch_history_db.py b/src/layers/databases/python/watch_history_db.py index 9a2824cd..f7f0078a 100644 --- a/src/layers/databases/python/watch_history_db.py +++ b/src/layers/databases/python/watch_history_db.py @@ -57,7 +57,7 @@ def add_item_v2(username, api_name, api_id, data=None): data = {} data["api_info"] = f"{api_name}_{api_id}" - if data.get("dates_watched") is None: + if data.get("dates_watched"): data["latest_watch_date"] = "0" try: get_item_by_api_id( From b688f5174857ba04ffe75da8aec54a1a3d27dde4 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:47:23 +0100 Subject: [PATCH 65/75] Fix invalid if conditions --- src/lambdas/api/watch_histories/routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index c397ae11..e427a328 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -154,7 +154,7 @@ def add_episode(username, api_name, item_api_id, episode_api_id, data): special=is_special ) - if data.get("dates_watched"): + if not data.get("dates_watched"): return _update_latest_watch_date(item, data, username, api_name, item_api_id) @@ -191,7 +191,7 @@ def update_episode(username, api_name, item_api_id, episode_api_id, data): data ) - if data.get("dates_watched"): + if not data.get("dates_watched"): return _update_latest_watch_date(item, data, username, api_name, item_api_id) From eacbdd276ed5af76e19f913d5f7c077421554118 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:48:54 +0100 Subject: [PATCH 66/75] Clear latest_watch_date on empty dates_watched put --- src/lambdas/api/watch_histories/routes.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index e427a328..d0f9caee 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -192,6 +192,13 @@ def update_episode(username, api_name, item_api_id, episode_api_id, data): ) if not data.get("dates_watched"): + watch_history_db.update_item_v2( + username, + api_name, + item_api_id, + {"latest_watch_date": "0"}, + clean_whitelist=[], + ) return _update_latest_watch_date(item, data, username, api_name, item_api_id) From 6316c3640c4a544adae79ff5a1e0943ddeb7c9a1 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 21:55:18 +0100 Subject: [PATCH 67/75] Revert clearing latest_watch_date of item on episode update --- src/lambdas/api/watch_histories/routes.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/lambdas/api/watch_histories/routes.py b/src/lambdas/api/watch_histories/routes.py index d0f9caee..e427a328 100644 --- a/src/lambdas/api/watch_histories/routes.py +++ b/src/lambdas/api/watch_histories/routes.py @@ -192,13 +192,6 @@ def update_episode(username, api_name, item_api_id, episode_api_id, data): ) if not data.get("dates_watched"): - watch_history_db.update_item_v2( - username, - api_name, - item_api_id, - {"latest_watch_date": "0"}, - clean_whitelist=[], - ) return _update_latest_watch_date(item, data, username, api_name, item_api_id) From 525a1ba7fc5c8629b3d6a7912ac983fafc1a1c79 Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 23:27:35 +0100 Subject: [PATCH 68/75] Shortcircut all show api calls --- src/lambdas/api/episode_by_collection_item/__init__.py | 6 ++++-- src/lambdas/api/episode_by_id/__init__.py | 9 ++++++--- src/lambdas/api/item_by_collection/__init__.py | 6 ++++-- src/lambdas/api/watch_history_by_collection/__init__.py | 6 ++++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/lambdas/api/episode_by_collection_item/__init__.py b/src/lambdas/api/episode_by_collection_item/__init__.py index fb7a1f08..b8880b81 100644 --- a/src/lambdas/api/episode_by_collection_item/__init__.py +++ b/src/lambdas/api/episode_by_collection_item/__init__.py @@ -65,7 +65,8 @@ def _get_episode_by_api_id(collection_name, item_id, api_name, api_id, s_ret = anime_api.get_episode_by_api_id(item_id, api_name, api_id, token) elif collection_name == "show": - s_ret = shows_api.get_episode_by_api_id(api_name, api_id) + raise utils.HttpError("", 501) + # s_ret = shows_api.get_episode_by_api_id(api_name, api_id) except utils.HttpError as e: err_msg = f"Could not get {collection_name} episode" log.error(f"{err_msg}. Error: {str(e)}") @@ -139,7 +140,8 @@ def _post_episode(username, collection_name, item_id, body, token): if collection_name == "anime": res = anime_api.post_episode(item_id, api_body, token) elif collection_name == "show": - res = shows_api.post_episode(item_id, api_body) + raise utils.HttpError("", 501) + # res = shows_api.post_episode(item_id, api_body) except utils.HttpError as e: err_msg = f"Could not post {collection_name}" log.error(f"{err_msg}. Error: {str(e)}") diff --git a/src/lambdas/api/episode_by_id/__init__.py b/src/lambdas/api/episode_by_id/__init__.py index 0c1730ec..fc52a26a 100644 --- a/src/lambdas/api/episode_by_id/__init__.py +++ b/src/lambdas/api/episode_by_id/__init__.py @@ -56,7 +56,8 @@ def _get_episode(username, collection_name, item_id, episode_id, token, api_name if collection_name == "anime": s_ret = anime_api.get_episode(item_id, episode_id, token) elif collection_name == "show": - s_ret = shows_api.get_episode(item_id, episode_id, api_name=api_name) + raise utils.HttpError("", 501) + # s_ret = shows_api.get_episode(item_id, episode_id, api_name=api_name) except utils.HttpError as e: err_msg = f"Could not get {collection_name} episode for " \ f"item: {item_id} and episode_id: {episode_id}" @@ -94,7 +95,8 @@ def _put_episode(username, collection_name, item_id, episode_id, body, token, ap if collection_name == "anime": anime_api.get_episode(item_id, episode_id, token) elif collection_name == "show": - shows_api.get_episode(item_id, episode_id, api_name=api_name) + raise utils.HttpError("", 501) + # shows_api.get_episode(item_id, episode_id, api_name=api_name) except utils.HttpError as e: err_msg = f"Could not get {collection_name} episode for " \ f"item: {item_id} and episode_id: {episode_id}" @@ -137,7 +139,8 @@ def _delete_episode(username, collection_name, item_id, episode_id, token, api_n if collection_name == "anime": res = anime_api.get_episode(item_id, episode_id, token) elif collection_name == "show": - res = shows_api.get_episode(item_id, episode_id, api_name=api_name) + raise utils.HttpError("", 501) + # res = shows_api.get_episode(item_id, episode_id, api_name=api_name) except utils.HttpError as e: err_msg = f"Could not get {collection_name} episode for " \ f"item: {item_id} and episode_id: {episode_id}" diff --git a/src/lambdas/api/item_by_collection/__init__.py b/src/lambdas/api/item_by_collection/__init__.py index d0fe0b6d..6f109260 100644 --- a/src/lambdas/api/item_by_collection/__init__.py +++ b/src/lambdas/api/item_by_collection/__init__.py @@ -51,7 +51,8 @@ def _get_item(username, collection_name, item_id, token, show_api): if collection_name == "anime": s_ret = anime_api.get_anime(item_id, token) elif collection_name == "show": - s_ret = shows_api.get_show(item_id, show_api) + raise utils.HttpError("", 501) + #s_ret = shows_api.get_show(item_id, show_api) elif collection_name == "movie": s_ret = movie_api.get_movie(item_id, token) except utils.HttpError as e: @@ -92,7 +93,8 @@ def _put_item(username, collection_name, item_id, body, token, show_api): if collection_name == "anime": anime_api.get_anime(item_id, token) elif collection_name == "show": - shows_api.get_show(item_id, show_api) + raise utils.HttpError("", 501) + #shows_api.get_show(item_id, show_api) elif collection_name == "movie": movie_api.get_movie(item_id, token) except utils.HttpError as e: diff --git a/src/lambdas/api/watch_history_by_collection/__init__.py b/src/lambdas/api/watch_history_by_collection/__init__.py index e8cadd11..2a0229c9 100644 --- a/src/lambdas/api/watch_history_by_collection/__init__.py +++ b/src/lambdas/api/watch_history_by_collection/__init__.py @@ -60,7 +60,8 @@ def _get_by_api_id(collection_name, api_name, api_id, username, token): if collection_name == "anime": s_ret = anime_api.get_anime_by_api_id(api_name, api_id, token) elif collection_name == "show": - s_ret = shows_api.get_show_by_api_id(api_name, api_id) + raise utils.HttpError("", 501) + # s_ret = shows_api.get_show_by_api_id(api_name, api_id) elif collection_name == "movie": s_ret = movie_api.get_movie_by_api_id(api_name, api_id, token) except utils.HttpError as e: @@ -147,7 +148,8 @@ def _post_collection_item(username, collection_name, body, token): if collection_name == "anime": res = anime_api.post_anime(api_body, token) elif collection_name == "show": - res = shows_api.post_show(api_body) + raise utils.HttpError("", 501) + # res = shows_api.post_show(api_body) elif collection_name == "movie": res = movie_api.post_movie(api_body, token) except utils.HttpError as e: From 3535fcb78e70937a535879f4f0da422a1c92426f Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 23:35:36 +0100 Subject: [PATCH 69/75] Add missing 's' to items routes --- src/lambdas/api/watch_histories/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 16785fa3..4b118764 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -8,17 +8,17 @@ app = FastAPI() -@app.get("/watch-histories/item/{api_name}/{item_api_id}") +@app.get("/watch-histories/items/{api_name}/{item_api_id}") def get_item(request: Request, api_name: str, item_api_id: str): return routes.get_item(request.state.username, api_name, item_api_id) -@app.delete("/watch-histories/item/{api_name}/{item_api_id}", status_code=204) +@app.delete("/watch-histories/items/{api_name}/{item_api_id}", status_code=204) def delete_item(request: Request, api_name: str, item_api_id: str): return routes.delete_item(request.state.username, api_name, item_api_id) -@app.put("/watch-histories/item/{api_name}/{item_api_id}", status_code=204) +@app.put("/watch-histories/items/{api_name}/{item_api_id}", status_code=204) def update_item(request: Request, api_name: str, item_api_id: str, data: ReviewData): return routes.update_item( @@ -29,7 +29,7 @@ def update_item(request: Request, api_name: str, item_api_id: str, ) -@app.post("/watch-histories/item", status_code=204) +@app.post("/watch-histories/items", status_code=204) def add_item(request: Request, item: PostItem): routes.add_item( request.state.username, @@ -39,7 +39,7 @@ def add_item(request: Request, item: PostItem): ) -@app.get("/watch-histories/item/{api_name}/{item_api_id}/episodes") +@app.get("/watch-histories/items/{api_name}/{item_api_id}/episodes") def get_episodes(request: Request, api_name: str, item_api_id: str): return routes.get_episodes( request.state.username, @@ -48,7 +48,7 @@ def get_episodes(request: Request, api_name: str, item_api_id: str): ) -@app.post("/watch-histories/item/{api_name}/{item_api_id}/episodes", +@app.post("/watch-histories/items/{api_name}/{item_api_id}/episodes", status_code=204) def add_episode(request: Request, api_name, item_api_id, episode: PostEpisode): routes.add_episode( @@ -61,7 +61,7 @@ def add_episode(request: Request, api_name, item_api_id, episode: PostEpisode): @app.get( - "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}") + "/watch-histories/items/{api_name}/{item_api_id}/episodes/{episode_api_id}") def get_episode(request: Request, api_name: str, item_api_id: str, episode_api_id: str): return routes.get_episode( @@ -73,7 +73,7 @@ def get_episode(request: Request, api_name: str, item_api_id: str, @app.put( - "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}", + "/watch-histories/items/{api_name}/{item_api_id}/episodes/{episode_api_id}", status_code=204) def update_episode(request: Request, api_name: str, item_api_id: str, episode_api_id: str, data: ReviewData): @@ -87,7 +87,7 @@ def update_episode(request: Request, api_name: str, item_api_id: str, @app.delete( - "/watch-histories/item/{api_name}/{item_api_id}/episodes/{episode_api_id}", + "/watch-histories/items/{api_name}/{item_api_id}/episodes/{episode_api_id}", status_code=204) def delete_episode(request: Request, api_name: str, item_api_id: str, episode_api_id: str): From 4a389902ed60303e2ffb59a89d0635cdfe6d927c Mon Sep 17 00:00:00 2001 From: fulder Date: Sun, 31 Oct 2021 23:44:44 +0100 Subject: [PATCH 70/75] Remove shows_api from api layer --- src/layers/api/python/shows_api.py | 89 ------------------ test/unittest/test_shows_api.py | 143 ----------------------------- 2 files changed, 232 deletions(-) delete mode 100644 src/layers/api/python/shows_api.py delete mode 100644 test/unittest/test_shows_api.py diff --git a/src/layers/api/python/shows_api.py b/src/layers/api/python/shows_api.py deleted file mode 100644 index 48f7e671..00000000 --- a/src/layers/api/python/shows_api.py +++ /dev/null @@ -1,89 +0,0 @@ -import os -import requests - -import utils - -SHOWS_API_URL = os.getenv("SHOWS_API_URL") - - -def get_show(show_id, api_name=None): - url = f"{SHOWS_API_URL}/shows/{show_id}" - if api_name is not None: - url += f"?api_name={api_name}" - - res = requests.get( - url, auth=utils.get_v4_signature_auth() - ) - if res.status_code != 200: - raise utils.HttpError("Invalid response in get_show", res.status_code) - - return res.json() - - -def get_show_by_api_id(api_name, api_id): - res = requests.get( - f"{SHOWS_API_URL}/shows?api_name={api_name}&api_id={api_id}", - auth=utils.get_v4_signature_auth(), - ) - if res.status_code != 200: - raise utils.HttpError( - "Invalid response in get_shows_by_api_id", res.status_code - ) - - return res.json() - - -def post_show(body): - res = requests.post( - f"{SHOWS_API_URL}/shows", json=body, auth=utils.get_v4_signature_auth() - ) - if res.status_code != 200: - raise utils.HttpError( - "Invalid response during show post", res.status_code - ) - - return res.json() - - -def get_episode(item_id, episode_id, api_name=None): - url = f"{SHOWS_API_URL}/shows/{item_id}/episodes/{episode_id}" - if api_name is not None: - url += f"?api_name={api_name}" - - res = requests.get( - url, - auth=utils.get_v4_signature_auth(), - ) - if res.status_code != 200: - raise utils.HttpError( - "Invalid response during show post", res.status_code - ) - - return res.json() - - -def get_episode_by_api_id(api_name, api_id): - res = requests.get( - f"{SHOWS_API_URL}/episodes?api_name={api_name}&api_id={api_id}", - auth=utils.get_v4_signature_auth(), - ) - if res.status_code != 200: - raise utils.HttpError( - "Invalid response in get_shows_by_api_id", res.status_code - ) - - return res.json() - - -def post_episode(item_id, body): - res = requests.post( - f"{SHOWS_API_URL}/shows/{item_id}/episodes", - json=body, - auth=utils.get_v4_signature_auth(), - ) - if res.status_code != 200: - raise utils.HttpError( - "Invalid response during show post", res.status_code - ) - - return res.json() diff --git a/test/unittest/test_shows_api.py b/test/unittest/test_shows_api.py deleted file mode 100644 index ea37743b..00000000 --- a/test/unittest/test_shows_api.py +++ /dev/null @@ -1,143 +0,0 @@ -from unittest.mock import patch, MagicMock - -import pytest - -import utils - - -@patch("shows_api.requests.get") -def test_get_show(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 200 - m.json.return_value = {"show_id": "123"} - mocked_get.return_value = m - - ret = mocked_show_api.get_show("123") - - assert ret == {"show_id": "123"} - - -@patch("shows_api.requests.get") -def test_get_show_invalid_code(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 404 - mocked_get.return_value = m - - with pytest.raises(utils.HttpError): - mocked_show_api.get_show("123") - - -@patch("shows_api.requests.get") -def test_get_show_by_api_id(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 200 - m.json.return_value = {"show_id": "123"} - mocked_get.return_value = m - - ret = mocked_show_api.get_show_by_api_id("tvdb", "123") - - assert ret == {"show_id": "123"} - - -@patch("shows_api.requests.get") -def test_get_show_by_api_id_invalid_code(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 404 - mocked_get.return_value = m - - with pytest.raises(utils.HttpError): - mocked_show_api.get_show_by_api_id("tvdb", "123") - - -@patch("shows_api.requests.post") -def test_post_show(mocked_post, mocked_show_api): - m = MagicMock() - m.status_code = 200 - m.json.return_value = {"show_id": "123"} - mocked_post.return_value = m - - ret = mocked_show_api.post_show({"api_id": "123", "api_name": "tvmaze"}) - - assert ret == {"show_id": "123"} - - -@patch("shows_api.requests.post") -def test_post_show_invalid_code(mocked_post, mocked_show_api): - m = MagicMock() - m.status_code = 404 - mocked_post.return_value = m - - with pytest.raises(utils.HttpError): - mocked_show_api.post_show({"api_id": "123", "api_name": "tvmaze"}) - - -@patch("shows_api.requests.get") -def test_get_episode(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 200 - m.json.return_value = {"show_id": "123"} - mocked_get.return_value = m - - ret = mocked_show_api.get_episode("123", "ep_id") - - assert ret == {"show_id": "123"} - - -@patch("shows_api.requests.get") -def test_get_episode_invalid_code(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 404 - mocked_get.return_value = m - - with pytest.raises(utils.HttpError): - mocked_show_api.get_episode("item_id", "ep_id") - - -@patch("shows_api.requests.get") -def test_get_episode_by_api_id(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 200 - m.json.return_value = {"show_id": "123"} - mocked_get.return_value = m - - ret = mocked_show_api.get_episode_by_api_id("tvdb", "123") - - assert ret == {"show_id": "123"} - - -@patch("shows_api.requests.get") -def test_get_episode_by_api_id_invalid_code(mocked_get, mocked_show_api): - m = MagicMock() - m.status_code = 404 - mocked_get.return_value = m - - with pytest.raises(utils.HttpError): - mocked_show_api.get_episode_by_api_id("tvdb", "123") - - -@patch("shows_api.requests.post") -def test_post_episode(mocked_post, mocked_show_api): - m = MagicMock() - m.status_code = 200 - m.json.return_value = {"show_id": "123"} - mocked_post.return_value = m - - ret = mocked_show_api.post_episode( - "123", - {"api_id": "123", "api_name": "tvmaze"}, - ) - - assert ret == {"show_id": "123"} - - -@patch("shows_api.requests.post") -def test_post_episode_invalid_code(mocked_post, mocked_show_api): - m = MagicMock() - m.status_code = 404 - mocked_post.return_value = m - - with pytest.raises(utils.HttpError): - mocked_show_api.post_episode( - "123", - {"api_id": "123", "api_name": "tvmaze"}, - ) From e0ac99e62d10e87619ea468e10c4dd613550a321 Mon Sep 17 00:00:00 2001 From: fulder Date: Mon, 1 Nov 2021 00:01:22 +0100 Subject: [PATCH 71/75] Add first fastapi unittest for get item --- .../episode_by_collection_item/__init__.py | 1 - src/lambdas/api/episode_by_id/__init__.py | 1 - .../api/item_by_collection/__init__.py | 1 - .../watch_history_by_collection/__init__.py | 1 - test/unittest/test_fastapi.py | 40 +++++++++++++++++++ 5 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 test/unittest/test_fastapi.py diff --git a/src/lambdas/api/episode_by_collection_item/__init__.py b/src/lambdas/api/episode_by_collection_item/__init__.py index b8880b81..45905b42 100644 --- a/src/lambdas/api/episode_by_collection_item/__init__.py +++ b/src/lambdas/api/episode_by_collection_item/__init__.py @@ -10,7 +10,6 @@ import jwt_utils import schema import episodes_db -import shows_api import watch_history_db log = logger.get_logger("episode_by_collection_item") diff --git a/src/lambdas/api/episode_by_id/__init__.py b/src/lambdas/api/episode_by_id/__init__.py index fc52a26a..672428ff 100644 --- a/src/lambdas/api/episode_by_id/__init__.py +++ b/src/lambdas/api/episode_by_id/__init__.py @@ -11,7 +11,6 @@ import schema import episodes_db import watch_history_db -import shows_api log = logger.get_logger("episodes_by_id") diff --git a/src/lambdas/api/item_by_collection/__init__.py b/src/lambdas/api/item_by_collection/__init__.py index 6f109260..97b45563 100644 --- a/src/lambdas/api/item_by_collection/__init__.py +++ b/src/lambdas/api/item_by_collection/__init__.py @@ -9,7 +9,6 @@ import jwt_utils import movie_api import schema -import shows_api import watch_history_db log = logger.get_logger("watch_history") diff --git a/src/lambdas/api/watch_history_by_collection/__init__.py b/src/lambdas/api/watch_history_by_collection/__init__.py index 2a0229c9..8e5e72ff 100644 --- a/src/lambdas/api/watch_history_by_collection/__init__.py +++ b/src/lambdas/api/watch_history_by_collection/__init__.py @@ -8,7 +8,6 @@ import jwt_utils import movie_api import schema -import shows_api import watch_history_db import anime_api diff --git a/test/unittest/test_fastapi.py b/test/unittest/test_fastapi.py new file mode 100644 index 00000000..81d8a984 --- /dev/null +++ b/test/unittest/test_fastapi.py @@ -0,0 +1,40 @@ +import jwt +from boto3.dynamodb.conditions import Key, Attr + +from starlette.testclient import TestClient +from api.watch_histories import app + +client = TestClient(app) + +TEST_USER = "TEST_USER" +TEST_SHOW_ID = "123123" + + +def test_get_item(mocked_watch_history_db): + mocked_watch_history_db.table.query.return_value = { + "Items": [ + { + "api_info": f"tvmaze_{TEST_SHOW_ID}" + } + ] + } + token = jwt.encode( + {"username": TEST_USER}, + "secret", + algorithm="HS256" + ).decode("utf-8") + + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", + headers={"Authorization": token} + ) + + exp_kwargs = { + "IndexName": "api_info", + "KeyConditionExpression": Key("username").eq(TEST_USER) & + Key("api_info").eq(f"tvmaze_{TEST_SHOW_ID}"), + "FilterExpression": Attr("deleted_at").not_exists(), + } + mocked_watch_history_db.table.query.assert_called_with(**exp_kwargs) + assert response.status_code == 200 + assert response.json() == {"api_info": f"tvmaze_{TEST_SHOW_ID}"} From e994ae550adf82f4238ddca1e9af1dc0dea3b7eb Mon Sep 17 00:00:00 2001 From: fulder Date: Mon, 1 Nov 2021 20:34:17 +0100 Subject: [PATCH 72/75] Add happy test case for post item --- src/layers/api/python/utils.py | 10 +- test/unittest/conftest.py | 10 -- test/unittest/test_episode_by_collection.py | 45 ------ test/unittest/test_fastapi.py | 148 +++++++++++++++++--- test/unittest/test_item_by_collection.py | 15 -- 5 files changed, 133 insertions(+), 95 deletions(-) diff --git a/src/layers/api/python/utils.py b/src/layers/api/python/utils.py index 214454ec..0f4f7bdb 100644 --- a/src/layers/api/python/utils.py +++ b/src/layers/api/python/utils.py @@ -4,9 +4,13 @@ from requests_aws4auth import AWS4Auth from threading import Lock, Thread +import tvmaze + items_lock = Lock() merged_items = [] +tvmaze_api = tvmaze.TvMazeApi() + class Error(Exception): pass @@ -46,13 +50,15 @@ def __init__(self, item, token, remove_status, show_api=None): def run(self): import anime_api import movie_api - import shows_api s_ret = None if self.collection_name == "movie": s_ret = movie_api.get_movie(self.item_id, self.token) if self.collection_name == "show": - s_ret = shows_api.get_show(self.item_id, self.show_api) + tvmaze_id = self.item["api_info"].split("_")[1] + s_ret = { + "tvmaze": tvmaze_api.get_show(tvmaze_id) + } elif self.collection_name == "anime": s_ret = anime_api.get_anime(self.item_id, self.token) diff --git a/test/unittest/conftest.py b/test/unittest/conftest.py index f25693e7..cb09a1db 100644 --- a/test/unittest/conftest.py +++ b/test/unittest/conftest.py @@ -35,16 +35,6 @@ def mocked_episodes_db(): return episodes_db -@pytest.fixture(scope='function') -def mocked_show_api(): - import shows_api - - shows_api.SHOW_API_URL = "https://mocked" - shows_api.utils.get_v4_signature_auth = MagicMock() - - return shows_api - - @pytest.fixture(scope='function') def mocked_movie_api(): import movie_api diff --git a/test/unittest/test_episode_by_collection.py b/test/unittest/test_episode_by_collection.py index 3f74a62a..484a734f 100644 --- a/test/unittest/test_episode_by_collection.py +++ b/test/unittest/test_episode_by_collection.py @@ -116,51 +116,6 @@ def test_anime_by_api_id_success(self, mocked_get_anime, "statusCode": 200 } - @patch("api.episode_by_collection_item.episodes_db.get_episode") - @patch("api.episode_by_collection_item.shows_api.get_episode_by_api_id") - def test_show_by_api_id_success(self, mocked_get_shows, - mocked_get_episode): - w_ret = { - "collection_name": "show", - "item_id": 123, - "episode_id": 345, - } - s_ret = { - "id": 123 - } - mocked_get_shows.return_value = s_ret - mocked_get_episode.return_value = w_ret - event = copy.deepcopy(self.event) - event["queryStringParameters"] = { - "api_id": "123", - "api_name": "tvdb", - } - event["pathParameters"]["collection_name"] = "show" - - ret = handle(event, None) - exp_data = {**w_ret, **s_ret} - assert ret == { - "body": json.dumps(exp_data), - "statusCode": 200 - } - - @patch("api.episode_by_collection_item.shows_api.get_episode_by_api_id") - def test_show_by_api_id_http_error(self, mocked_get_shows): - mocked_get_shows.side_effect = HttpError("test-error", 409) - event = copy.deepcopy(self.event) - event["queryStringParameters"] = { - "api_id": "123", - "api_name": "tvdb", - } - event["pathParameters"]["collection_name"] = "show" - - ret = handle(event, None) - assert ret == { - "body": '{"message": "Could not get show episode"}', - "error": "test-error", - "statusCode": 409 - } - @patch("api.episode_by_collection_item.episodes_db.get_episode") @patch("api.episode_by_collection_item.anime_api.get_episode_by_api_id") def test_anime_by_api_id_not_found(self, mocked_get_anime, diff --git a/test/unittest/test_fastapi.py b/test/unittest/test_fastapi.py index 81d8a984..bff8a45e 100644 --- a/test/unittest/test_fastapi.py +++ b/test/unittest/test_fastapi.py @@ -1,8 +1,14 @@ +from datetime import datetime +from unittest import mock +from unittest.mock import patch + import jwt +import pytest from boto3.dynamodb.conditions import Key, Attr from starlette.testclient import TestClient from api.watch_histories import app +from tvmaze import TvMazeApi client = TestClient(app) @@ -10,31 +16,127 @@ TEST_SHOW_ID = "123123" -def test_get_item(mocked_watch_history_db): - mocked_watch_history_db.table.query.return_value = { - "Items": [ - { - "api_info": f"tvmaze_{TEST_SHOW_ID}" - } - ] - } - token = jwt.encode( +@pytest.fixture +def test_token(): + return jwt.encode( {"username": TEST_USER}, "secret", algorithm="HS256" ).decode("utf-8") - response = client.get( - f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", - headers={"Authorization": token} - ) - - exp_kwargs = { - "IndexName": "api_info", - "KeyConditionExpression": Key("username").eq(TEST_USER) & - Key("api_info").eq(f"tvmaze_{TEST_SHOW_ID}"), - "FilterExpression": Attr("deleted_at").not_exists(), - } - mocked_watch_history_db.table.query.assert_called_with(**exp_kwargs) - assert response.status_code == 200 - assert response.json() == {"api_info": f"tvmaze_{TEST_SHOW_ID}"} + +class TestGetItem: + def test_get_item(self, mocked_watch_history_db, test_token): + mocked_watch_history_db.table.query.return_value = { + "Items": [ + { + "api_info": f"tvmaze_{TEST_SHOW_ID}" + } + ] + } + + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", + headers={"Authorization": test_token} + ) + + exp_kwargs = { + "IndexName": "api_info", + "KeyConditionExpression": + Key("username").eq(TEST_USER) & + Key("api_info").eq(f"tvmaze_{TEST_SHOW_ID}"), + "FilterExpression": Attr("deleted_at").not_exists(), + } + mocked_watch_history_db.table.query.assert_called_with(**exp_kwargs) + assert response.status_code == 200 + assert response.json() == {"api_info": f"tvmaze_{TEST_SHOW_ID}"} + + def test_get_item_not_found(self, mocked_watch_history_db, test_token): + mocked_watch_history_db.table.query.return_value = { + "Items": [] + } + + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", + headers={"Authorization": test_token} + ) + + assert response.status_code == 404 + + +@patch.object(TvMazeApi, "get_show_episodes_count") +class TestPostItem: + + def test_post_item(self, mocked_tvmaze_episodes_count, + test_token, mocked_watch_history_db): + mocked_tvmaze_episodes_count.return_value = { + "ep_count": 1, + "special_count": 2, + } + mocked_watch_history_db.table.query.return_value = { + "Items": [ + {} + ] + } + response = client.post( + "/watch-histories/items", + headers={"Authorization": test_token}, + json={ + "item_api_id": TEST_SHOW_ID, + "api_name": "tvmaze" + } + ) + + mocked_watch_history_db.table.update_item.assert_called_with( + Key={ + "username": TEST_USER, + "item_id": "d2a81cbb-9a60-52ec-a020-bc699d17790c" + }, + UpdateExpression="SET " + "#ep_count=:ep_count," + "#special_count=:special_count," + "#ep_progress=:ep_progress," + "#special_progress=:special_progress," + "#watched_eps=:watched_eps," + "#watched_special=:watched_special," + "#api_info=:api_info," + "#collection_name=:collection_name," + "#updated_at=:updated_at " + "REMOVE " + "#deleted_at", + ExpressionAttributeNames={ + "#ep_count": "ep_count", + "#special_count": "special_count", + "#ep_progress": "ep_progress", + "#special_progress": "special_progress", + "#watched_eps": "watched_eps", + "#watched_special": "watched_special", + "#api_info": "api_info", + "#collection_name": "collection_name", + "#updated_at": "updated_at", + "#deleted_at": "deleted_at" + }, + ExpressionAttributeValues={ + ":ep_count": 1, + ":special_count": 2, + ":ep_progress": 0, + ":special_progress": 0, + ":watched_eps": 0, + ":watched_special": 0, + ":api_info": f"tvmaze_{TEST_SHOW_ID}", + ":collection_name": "show", + ":updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S") + }) + assert response.status_code == 204 + + # def test_get_item_not_found(self, mocked_watch_history_db, test_token): + # mocked_watch_history_db.table.query.return_value = { + # "Items": [] + # } + # + # response = client.get( + # f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", + # headers={"Authorization": test_token} + # ) + # + # assert response.status_code == 404 diff --git a/test/unittest/test_item_by_collection.py b/test/unittest/test_item_by_collection.py index fba74544..17773c15 100644 --- a/test/unittest/test_item_by_collection.py +++ b/test/unittest/test_item_by_collection.py @@ -39,21 +39,6 @@ def test_success_anime(self, mocked_anime_get, mocked_get): "statusCode": 200 } - @patch("api.item_by_collection.watch_history_db.get_item") - @patch("api.item_by_collection.shows_api.get_show") - def test_success_show(self, mocked_show_get, mocked_get): - mocked_show_get.return_value = {"tvdb_id": 564} - mocked_get.return_value = {"collection_name": "show", "item_id": 123} - event = copy.deepcopy(self.event) - event["pathParameters"]["collection_name"] = "show" - - ret = handle(event, None) - assert ret == { - "body": '{"collection_name": "show", ' - '"item_id": 123, "tvdb_id": 564}', - "statusCode": 200 - } - @patch("api.item_by_collection.watch_history_db.get_item") @patch("api.item_by_collection.movie_api.get_movie") def test_success_movie(self, mocked_movie_get, mocked_get): From 36f589487387e024fbefba199119350f8047a37b Mon Sep 17 00:00:00 2001 From: fulder Date: Tue, 2 Nov 2021 00:10:38 +0100 Subject: [PATCH 73/75] Cleanup old show tests and add new more simple route tests --- src/lambdas/api/watch_histories/__init__.py | 6 +- test/unittest/conftest.py | 22 +++ test/unittest/test_episode_by_collection.py | 25 --- test/unittest/test_episode_by_id.py | 45 ------ test/unittest/test_episodes_db.py | 113 ------------- test/unittest/test_fastapi.py | 142 ---------------- test/unittest/test_item_by_collection.py | 5 +- test/unittest/test_routes.py | 151 ++++++++++++++++++ .../test_watch_history_by_collection.py | 64 -------- 9 files changed, 178 insertions(+), 395 deletions(-) delete mode 100644 test/unittest/test_fastapi.py create mode 100644 test/unittest/test_routes.py diff --git a/src/lambdas/api/watch_histories/__init__.py b/src/lambdas/api/watch_histories/__init__.py index 4b118764..4c31de8c 100644 --- a/src/lambdas/api/watch_histories/__init__.py +++ b/src/lambdas/api/watch_histories/__init__.py @@ -15,13 +15,13 @@ def get_item(request: Request, api_name: str, item_api_id: str): @app.delete("/watch-histories/items/{api_name}/{item_api_id}", status_code=204) def delete_item(request: Request, api_name: str, item_api_id: str): - return routes.delete_item(request.state.username, api_name, item_api_id) + routes.delete_item(request.state.username, api_name, item_api_id) @app.put("/watch-histories/items/{api_name}/{item_api_id}", status_code=204) def update_item(request: Request, api_name: str, item_api_id: str, data: ReviewData): - return routes.update_item( + routes.update_item( request.state.username, api_name, item_api_id, @@ -77,7 +77,7 @@ def get_episode(request: Request, api_name: str, item_api_id: str, status_code=204) def update_episode(request: Request, api_name: str, item_api_id: str, episode_api_id: str, data: ReviewData): - return routes.update_episode( + routes.update_episode( request.state.username, api_name, item_api_id, diff --git a/test/unittest/conftest.py b/test/unittest/conftest.py index cb09a1db..fd448106 100644 --- a/test/unittest/conftest.py +++ b/test/unittest/conftest.py @@ -1,7 +1,10 @@ import os from unittest.mock import MagicMock +import jwt import pytest +from starlette.testclient import TestClient +from api.watch_histories import app os.environ["LOG_LEVEL"] = "DEBUG" @@ -42,3 +45,22 @@ def mocked_movie_api(): movie_api.MOVIE_API_URL = "https://mocked" return movie_api + + +@pytest.fixture(scope='session') +def client(): + return TestClient(app) + + +@pytest.fixture(scope='session') +def username(): + return "TEST_USER" + + +@pytest.fixture(scope='session') +def token(username): + return jwt.encode( + {"username": username}, + "secret", + algorithm="HS256" + ).decode("utf-8") diff --git a/test/unittest/test_episode_by_collection.py b/test/unittest/test_episode_by_collection.py index 484a734f..7d32a52f 100644 --- a/test/unittest/test_episode_by_collection.py +++ b/test/unittest/test_episode_by_collection.py @@ -175,31 +175,6 @@ def test_success_anime(self, mocked_change_watched_eps, mocked_update_episode, m "statusCode": 200 } - @patch("api.episode_by_collection_item.episodes_db.add_episode") - @patch("api.episode_by_collection_item.shows_api.post_episode") - @patch("api.episode_by_collection_item.watch_history_db.get_item") - @patch("api.episode_by_collection_item.watch_history_db.update_item") - @patch("api.episode_by_collection_item.episodes_db.update_episode") - @patch("api.episode_by_collection_item.watch_history_db.change_watched_eps") - def test_success_show(self, mocked_change_watched_eps, mocked_update_episode, mocked_update_item, mocked_get_item, mocked_post_episode, mocked_post): - mocked_post.return_value = True - mocked_post_episode.return_value = { - "id": "123", - "is_special": False, - } - mocked_get_item.return_value = { - "latest_watch_date": "2030-01-01 10:00:00" - } - - event = copy.deepcopy(self.event) - event["pathParameters"]["collection_name"] = "show" - - ret = handle(event, None) - assert ret == { - "body": '{"id": "123"}', - "statusCode": 200 - } - @patch("api.episode_by_collection_item.episodes_db.add_episode") @patch("api.episode_by_collection_item.anime_api.post_episode") def test_api_error(self, mocked_post_episode, mocked_post): diff --git a/test/unittest/test_episode_by_id.py b/test/unittest/test_episode_by_id.py index 759d77de..7b924f73 100644 --- a/test/unittest/test_episode_by_id.py +++ b/test/unittest/test_episode_by_id.py @@ -37,19 +37,6 @@ def test_success_anime(self, mocked_anime_ep, mocked_get): '"item_id": 123, "mal_id": 345}', 'statusCode': 200} - @patch("api.episode_by_id.episodes_db.get_episode") - @patch("api.episode_by_id.shows_api.get_episode") - def test_success_show(self, mocked_shows_ep, mocked_get): - mocked_shows_ep.return_value = {"tvmaze_id": 345} - mocked_get.return_value = {"collection_name": "show", "item_id": 123} - event = copy.deepcopy(self.event) - event["pathParameters"]["collection_name"] = "show" - - ret = handle(event, None) - assert ret == {'body': '{"collection_name": "show", ' - '"item_id": 123, "tvmaze_id": 345}', - 'statusCode': 200} - @patch("api.episode_by_id.episodes_db.get_episode") @patch("api.episode_by_id.anime_api.get_episode") def test_anime_http_error(self, mocked_anime_ep, mocked_get): @@ -143,38 +130,6 @@ def test_success_anime(self, mocked_update_item, mocked_get_item, ret = handle(self.event, None) assert ret == {'statusCode': 204} - @patch("api.episode_by_id.episodes_db.update_episode") - @patch("api.episode_by_collection_item.shows_api.get_episode") - @patch("api.episode_by_id.watch_history_db.get_item") - def test_success_show(self, mocked_get_item, mocked_get_episode, - mocked_update_episode): - mocked_get_item.return_value = { - "latest_watch_date": "2021-01-01" - } - mocked_update_episode.return_value = "2020-01-01" - - event = copy.deepcopy(self.event) - event["pathParameters"]["collection_name"] = "show" - - ret = handle(event, None) - assert ret == {'statusCode': 204} - - @patch("api.episode_by_id.episodes_db.update_episode") - @patch("api.episode_by_collection_item.shows_api.get_episode") - @patch("api.episode_by_id.watch_history_db.get_item") - def test_missing_last_watch_date(self, mocked_get_item, mocked_get_episode, - mocked_update_episode): - mocked_get_item.return_value = { - "latest_watch_date": "2021-01-01" - } - mocked_update_episode.return_value = None - - event = copy.deepcopy(self.event) - event["pathParameters"]["collection_name"] = "show" - - ret = handle(event, None) - assert ret == {'statusCode': 204} - @patch("api.episode_by_id.episodes_db.update_episode") @patch("api.episode_by_collection_item.anime_api.get_episode") def test_api_error(self, mocked_get_episode, mocked_post): diff --git a/test/unittest/test_episodes_db.py b/test/unittest/test_episodes_db.py index 59f80fbb..75b419b5 100644 --- a/test/unittest/test_episodes_db.py +++ b/test/unittest/test_episodes_db.py @@ -16,119 +16,6 @@ def mock_func(**kwargs): return MOCK_RETURN -def test_get_episodes(mocked_episodes_db): - global MOCK_RETURN - MOCK_RETURN = [ - {"Items": [{"collection_name": "ANIME", "item_id": 123, "id": 345}]}, - {"Items": [{"collection_name": "MOVIE", "item_id": 123, "id": 345}]} - ] - m = MagicMock() - mocked_episodes_db.client.get_paginator.return_value = m - m.paginate = mock_func - - mocked_episodes_db.get_episodes(TEST_USERNAME, "anime", 123) - - assert UPDATE_VALUES == { - 'ExpressionAttributeValues': { - ':collection_name': {'S': "anime"}, - ':item_id': {'S': 123}, - ':username': {'S': 'TEST_USERNAME'} - }, - 'FilterExpression': 'attribute_not_exists(deleted_at) and item_id = :item_id ' - 'and collection_name = :collection_name', - 'KeyConditionExpression': 'username = :username', - 'Limit': 100, - 'ScanIndexForward': False, - 'TableName': None - } - - -def test_get_episodes_changed_limit(mocked_episodes_db): - global MOCK_RETURN - MOCK_RETURN = [ - {"Items": [{"collection_name": "ANIME", "item_id": 123, "id": 345}]}, - {"Items": [{"collection_name": "MOVIE", "item_id": 123, "id": 345}]} - ] - m = MagicMock() - mocked_episodes_db.client.get_paginator.return_value = m - m.paginate = mock_func - - mocked_episodes_db.get_episodes(TEST_USERNAME, "anime", 123, limit=10) - - assert UPDATE_VALUES == { - 'ExpressionAttributeValues': { - ':collection_name': {'S': 'anime'}, - ':item_id': {'S': 123}, - ':username': {'S': 'TEST_USERNAME'} - }, - 'FilterExpression': 'attribute_not_exists(deleted_at) and item_id = :item_id ' - 'and collection_name = :collection_name', - 'KeyConditionExpression': 'username = :username', - 'Limit': 10, - 'ScanIndexForward': False, - 'TableName': None - } - - -def test_get_episodes_by_with_start(mocked_episodes_db): - global MOCK_RETURN - MOCK_RETURN = [ - {"Items": [{"collection_name": "ANIME", "item_id": 123, "id": 345}]}, - {"Items": [{"collection_name": "ANIME", "item_id": 123, "id": 567}]} - ] - m = MagicMock() - mocked_episodes_db.client.get_paginator.return_value = m - m.paginate = mock_func - - ret = mocked_episodes_db.get_episodes(TEST_USERNAME, "ANIME", 123, limit=1, - start=2) - - assert UPDATE_VALUES == { - 'ExpressionAttributeValues': { - ':collection_name': {'S': 'ANIME'}, - ':item_id': {'S': 123}, - ':username': {'S': 'TEST_USERNAME'} - }, - 'FilterExpression': 'attribute_not_exists(deleted_at) and item_id = :item_id ' - 'and collection_name = :collection_name', - 'KeyConditionExpression': 'username = :username', - 'Limit': 1, - 'ScanIndexForward': False, - 'TableName': None - } - assert ret == { - 'episodes': {567: {"item_id": 123, 'collection_name': 'ANIME'}}, - "total_pages": 2 - } - - -def test_get_episodes_too_small_start_index(mocked_episodes_db): - with pytest.raises(mocked_episodes_db.InvalidStartOffset): - mocked_episodes_db.get_episodes(TEST_USERNAME, 123, "anime", start=0) - - -def test_get_episodes_too_large_start_index(mocked_episodes_db): - global MOCK_RETURN - MOCK_RETURN = [ - {"Items": [{"collection_name": "ANIME", "item_id": 123, "id": 345}]}, - {"Items": [{"collection_name": "MOVIE", "item_id": 123, "id": 567}]} - ] - m = MagicMock() - mocked_episodes_db.client.get_paginator.return_value = m - m.paginate = mock_func - with pytest.raises(mocked_episodes_db.InvalidStartOffset): - mocked_episodes_db.get_episodes(TEST_USERNAME, 123, "ANIME", start=10) - - -def test_get_episodes_not_found(mocked_episodes_db): - m = MagicMock() - mocked_episodes_db.client.get_paginator.return_value = m - m.paginate.return_value = [{"Items": []}] - - with pytest.raises(mocked_episodes_db.NotFoundError): - mocked_episodes_db.get_episodes(TEST_USERNAME, 123, "ANIME") - - def test_add_episode(mocked_episodes_db): global UPDATE_VALUES UPDATE_VALUES = {} diff --git a/test/unittest/test_fastapi.py b/test/unittest/test_fastapi.py deleted file mode 100644 index bff8a45e..00000000 --- a/test/unittest/test_fastapi.py +++ /dev/null @@ -1,142 +0,0 @@ -from datetime import datetime -from unittest import mock -from unittest.mock import patch - -import jwt -import pytest -from boto3.dynamodb.conditions import Key, Attr - -from starlette.testclient import TestClient -from api.watch_histories import app -from tvmaze import TvMazeApi - -client = TestClient(app) - -TEST_USER = "TEST_USER" -TEST_SHOW_ID = "123123" - - -@pytest.fixture -def test_token(): - return jwt.encode( - {"username": TEST_USER}, - "secret", - algorithm="HS256" - ).decode("utf-8") - - -class TestGetItem: - def test_get_item(self, mocked_watch_history_db, test_token): - mocked_watch_history_db.table.query.return_value = { - "Items": [ - { - "api_info": f"tvmaze_{TEST_SHOW_ID}" - } - ] - } - - response = client.get( - f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", - headers={"Authorization": test_token} - ) - - exp_kwargs = { - "IndexName": "api_info", - "KeyConditionExpression": - Key("username").eq(TEST_USER) & - Key("api_info").eq(f"tvmaze_{TEST_SHOW_ID}"), - "FilterExpression": Attr("deleted_at").not_exists(), - } - mocked_watch_history_db.table.query.assert_called_with(**exp_kwargs) - assert response.status_code == 200 - assert response.json() == {"api_info": f"tvmaze_{TEST_SHOW_ID}"} - - def test_get_item_not_found(self, mocked_watch_history_db, test_token): - mocked_watch_history_db.table.query.return_value = { - "Items": [] - } - - response = client.get( - f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", - headers={"Authorization": test_token} - ) - - assert response.status_code == 404 - - -@patch.object(TvMazeApi, "get_show_episodes_count") -class TestPostItem: - - def test_post_item(self, mocked_tvmaze_episodes_count, - test_token, mocked_watch_history_db): - mocked_tvmaze_episodes_count.return_value = { - "ep_count": 1, - "special_count": 2, - } - mocked_watch_history_db.table.query.return_value = { - "Items": [ - {} - ] - } - response = client.post( - "/watch-histories/items", - headers={"Authorization": test_token}, - json={ - "item_api_id": TEST_SHOW_ID, - "api_name": "tvmaze" - } - ) - - mocked_watch_history_db.table.update_item.assert_called_with( - Key={ - "username": TEST_USER, - "item_id": "d2a81cbb-9a60-52ec-a020-bc699d17790c" - }, - UpdateExpression="SET " - "#ep_count=:ep_count," - "#special_count=:special_count," - "#ep_progress=:ep_progress," - "#special_progress=:special_progress," - "#watched_eps=:watched_eps," - "#watched_special=:watched_special," - "#api_info=:api_info," - "#collection_name=:collection_name," - "#updated_at=:updated_at " - "REMOVE " - "#deleted_at", - ExpressionAttributeNames={ - "#ep_count": "ep_count", - "#special_count": "special_count", - "#ep_progress": "ep_progress", - "#special_progress": "special_progress", - "#watched_eps": "watched_eps", - "#watched_special": "watched_special", - "#api_info": "api_info", - "#collection_name": "collection_name", - "#updated_at": "updated_at", - "#deleted_at": "deleted_at" - }, - ExpressionAttributeValues={ - ":ep_count": 1, - ":special_count": 2, - ":ep_progress": 0, - ":special_progress": 0, - ":watched_eps": 0, - ":watched_special": 0, - ":api_info": f"tvmaze_{TEST_SHOW_ID}", - ":collection_name": "show", - ":updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S") - }) - assert response.status_code == 204 - - # def test_get_item_not_found(self, mocked_watch_history_db, test_token): - # mocked_watch_history_db.table.query.return_value = { - # "Items": [] - # } - # - # response = client.get( - # f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", - # headers={"Authorization": test_token} - # ) - # - # assert response.status_code == 404 diff --git a/test/unittest/test_item_by_collection.py b/test/unittest/test_item_by_collection.py index 17773c15..46888fb7 100644 --- a/test/unittest/test_item_by_collection.py +++ b/test/unittest/test_item_by_collection.py @@ -130,13 +130,12 @@ class TestPut: @pytest.mark.parametrize( "collection_name", - ["anime", "show", "movie"] + ["anime", "movie"] ) @patch("api.item_by_collection.anime_api.get_anime") - @patch("api.item_by_collection.shows_api.get_show") @patch("api.item_by_collection.movie_api.get_movie") @patch("api.item_by_collection.watch_history_db.update_item") - def test_success(self, a, s, m, mocked_post, collection_name): + def test_success(self, a, m, mocked_post, collection_name): mocked_post.return_value = True event = copy.deepcopy(self.event) event["pathParameters"]["collection_name"] = collection_name diff --git a/test/unittest/test_routes.py b/test/unittest/test_routes.py new file mode 100644 index 00000000..1a09a94c --- /dev/null +++ b/test/unittest/test_routes.py @@ -0,0 +1,151 @@ +from unittest.mock import patch + +import episodes_db +import tvmaze +import watch_history_db + +TEST_SHOW_ID = "123123" +TEST_EPISODE_ID = "456465" + + +@patch("watch_history_db.get_item_by_api_id") +def test_get_item(m_get_item, token, client, username): + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", + headers={"Authorization": token} + ) + + assert response.status_code == 200 + + +@patch("watch_history_db.get_item_by_api_id") +def test_not_found(m_get_item, client, token): + m_get_item.side_effect = watch_history_db.NotFoundError + + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}", + headers={"Authorization": token} + ) + + assert response.status_code == 404 + + +@patch.object(tvmaze.TvMazeApi, "get_show_episodes_count") +@patch("watch_history_db.get_item_by_api_id") +@patch("watch_history_db.add_item_v2") +def test_post_item(m_add_item, m_get_item, mocked_ep_count, token, client): + mocked_ep_count.return_value = { + "ep_count": 1, + "special_count": 2, + } + + m_get_item.return_value = {} + + response = client.post( + "/watch-histories/items", + headers={"Authorization": token}, + json={ + "item_api_id": TEST_SHOW_ID, + "api_name": "tvmaze" + } + ) + + assert response.status_code == 204 + + +@patch.object(tvmaze.TvMazeApi, "get_show_episodes_count") +def test_post_item_tvmaze_error(m_ep_count, token, client): + m_ep_count.side_effect = tvmaze.HTTPError(503) + + response = client.post( + "/watch-histories/items", + headers={"Authorization": token}, + json={ + "item_api_id": TEST_SHOW_ID, + "api_name": "tvmaze" + } + ) + + assert response.status_code == 503 + + +@patch.object(tvmaze.TvMazeApi, "get_show_episodes_count") +@patch("watch_history_db.get_item_by_api_id") +@patch("watch_history_db.add_item_v2") +def test_post_item_not_found(m_add_item, m_get_item, mocked_ep_count, token, + client): + mocked_ep_count.return_value = { + "ep_count": 1, + "special_count": 2, + } + m_get_item.side_effect = [ + watch_history_db.NotFoundError, {"Items": []} + ] + + response = client.post( + "/watch-histories/items", + headers={"Authorization": token}, + json={ + "item_api_id": TEST_SHOW_ID, + "api_name": "tvmaze" + } + ) + + assert response.status_code == 204 + + +@patch("episodes_db.get_episode_by_api_id") +def test_get_episode(m_get_ep, token, client, username): + api_info = f"tvmaze_{TEST_SHOW_ID}_{TEST_EPISODE_ID}" + m_get_ep.return_value = { + "api_info": api_info + } + + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}/episodes/{TEST_EPISODE_ID}", + headers={"Authorization": token} + ) + + assert response.status_code == 200 + assert response.json() == {"api_info": api_info} + + +@patch("episodes_db.get_episode_by_api_id") +def test_get_episode_not_found(m_get_ep, token, client, username): + m_get_ep.side_effect = episodes_db.NotFoundError + + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}/episodes/{TEST_EPISODE_ID}", + headers={"Authorization": token} + ) + + assert response.status_code == 404 + + +@patch.object(tvmaze.TvMazeApi, "get_episode") +@patch("watch_history_db.get_item_by_api_id") +@patch("episodes_db.update_episode_v2") +def test_put_episode(m_update_ep, m_get_item, m_get_ep, token, client, + username): + response = client.put( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}/episodes/{TEST_EPISODE_ID}", + headers={"Authorization": token}, + json={ + "review": "new_review" + } + ) + + assert response.status_code == 204 + + +@patch("episodes_db.get_episodes") +def test_get_episodes(m_get_eps, token, client, username): + m_get_eps.return_value = [1, 2, 3] + + response = client.get( + f"/watch-histories/items/tvmaze/{TEST_SHOW_ID}/episodes", + headers={"Authorization": token}, + ) + + assert response.status_code == 200 + assert response.json() == m_get_eps.return_value diff --git a/test/unittest/test_watch_history_by_collection.py b/test/unittest/test_watch_history_by_collection.py index 248d2e2a..4929e365 100644 --- a/test/unittest/test_watch_history_by_collection.py +++ b/test/unittest/test_watch_history_by_collection.py @@ -73,34 +73,6 @@ def test_success_by_api_id_anime(self, mocked_get_anime, "statusCode": 200 } - @patch("api.watch_history_by_collection.watch_history_db.get_item") - @patch("api.watch_history_by_collection.shows_api.get_show_by_api_id") - def test_success_by_api_id_shows(self, mocked_get_show, - mocked_get_watch_history): - w_ret = { - "collection_name": "anime", - "item_id": 123, - "username": "user", - } - s_ret = { - "id": 123 - } - mocked_get_show.return_value = s_ret - mocked_get_watch_history.return_value = w_ret - event = copy.deepcopy(self.event) - event["pathParameters"]["collection_name"] = "show" - event["queryStringParameters"] = { - "api_id": 123, - "api_name": "tvdb" - } - - ret = handle(event, None) - exp_data = {**w_ret, **s_ret} - assert ret == { - "body": json.dumps(exp_data), - "statusCode": 200 - } - @patch("api.watch_history_by_collection.watch_history_db.get_item") @patch("api.watch_history_by_collection.movie_api.get_movie_by_api_id") def test_success_by_api_id_movie(self, mocked_get_movie, @@ -268,24 +240,6 @@ def test_empty_body(self, mocked_post): "statusCode": 400 } - @patch("api.watch_history_by_collection.watch_history_db.add_item") - @patch("api.watch_history_by_collection.shows_api.post_show") - @patch("api.watch_history_by_collection.watch_history_db.get_item") - def test_show_success(self, mocked_get_item, mocked_post_show, mocked_post): - mocked_post_show.return_value = { - "id": "123" - } - mocked_post.return_value = True - event = copy.deepcopy(self.event) - event["pathParameters"]["collection_name"] = "show" - event["body"] = '{ "api_id": "123", "api_name": "tvmaze" }' - - ret = handle(event, None) - assert ret == { - "body": '{"id": "123"}', - "statusCode": 200 - } - @patch("api.watch_history_by_collection.watch_history_db.add_item") @patch("api.watch_history_by_collection.movie_api.post_movie") @patch("api.watch_history_by_collection.watch_history_db.get_item") @@ -355,21 +309,3 @@ def test_invalid_body_schema(self, mocked_post): "body": json.dumps(body), "statusCode": 400 } - - @patch("api.watch_history_by_collection.watch_history_db.add_item") - @patch("api.watch_history_by_collection.shows_api.get_show") - def test_missing_id(self, mocked_get_show, mocked_post): - mocked_get_show.return_value = True - mocked_post.return_value = True - event = copy.deepcopy(self.event) - event["body"] = '{"api_name": "mal"}' - - ret = handle(event, None) - body = { - "message": "Invalid post schema", - "error": "'api_id' is a required property" - } - assert ret == { - "body": json.dumps(body), - "statusCode": 400 - } From 56ba8c8ff71a0e560f3dbbb962fdc3af3bbc5a77 Mon Sep 17 00:00:00 2001 From: fulder Date: Tue, 2 Nov 2021 00:13:15 +0100 Subject: [PATCH 74/75] Install watch_histories requirements before tests --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7c912f50..840eb47c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,7 @@ jobs: pip install -r src/layers/databases/requirements.txt pip install -r src/layers/api/requirements.txt pip install -r src/layers/utils/requirements.txt + pip install -r src/lambdas/api/watch_histories/requirements.txt - name: Lint with flake8 run: | From 5422aec5a7807adf8676332ab873ccc8dd2fab4a Mon Sep 17 00:00:00 2001 From: fulder Date: Tue, 2 Nov 2021 00:15:25 +0100 Subject: [PATCH 75/75] Include new lambda in python path --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 168c4c2b..c59276a4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: test test: pip install -U -r test/unittest/requirements.txt - PYTHONPATH=./src/layers/utils/python:./src/lambdas/:./src/layers/databases/python:./src/layers/api/python \ + PYTHONPATH=./src/layers/utils/python:./src/lambdas/:./src/layers/databases/python:./src/layers/api/python:./src/lambdas/api/watch_histories \ pytest test/unittest --cov-report html --cov=src -vv .PHONY: apitest