From 48a2666da296c197e61185974410891a2134ab18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Mon, 17 Oct 2022 17:11:54 +0200 Subject: [PATCH] Add: Add async API for uploading release assets to GitHub The order of the files yielded depends on their size and the network. The tests for the upload heavily depend on the internal ordering in asyncio of the as_completed function. Therefore the tests may be flacky on different Python versions. --- pontos/github/api/release.py | 68 +++++++++++++++++++++++++++++++- tests/github/api/test_release.py | 48 ++++++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/pontos/github/api/release.py b/pontos/github/api/release.py index e24a8dd49..662b5b3f9 100644 --- a/pontos/github/api/release.py +++ b/pontos/github/api/release.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio from pathlib import Path from typing import ( AsyncContextManager, @@ -36,6 +37,7 @@ DownloadProgressIterable, download, download_async, + upload, ) @@ -204,12 +206,76 @@ async def download_asset(name, download_cm): asset_url: str = asset_json.get("browser_download_url", "") name: str = asset_json.get("name", "") - print("matches", name, Path(name), Path(name).match(match_pattern)) if match_pattern and not Path(name).match(match_pattern): continue yield name, download_async(self._client.stream(asset_url)) + async def upload_release_assets( + self, + repo: str, + tag: str, + files: Iterable[Union[Path, Tuple[Path, str]]], + ) -> AsyncIterator[Path]: + """ + Upload release assets asynchronously + + Args: + repo: GitHub repository (owner/name) to use + tag: The git tag for the release + files: An iterable of file paths or an iterable of tuples + containing a file path and content types to upload as an asset + + Returns: + yields each file after its upload is finished + + Raises: + HTTPError if an upload request was invalid + + Example: + .. code-block:: python + + files = (Path("foo.txt"), Path("bar.txt"),) + async for uploaded_file in api.upload_release_assets( + "foo/bar", "1.2.3", files + ): + print(f"Uploaded: {uploaded_file}") + + files = [ + (Path("foo.txt"), "text/ascii"), + (Path("bar.pdf"), "application/pdf"), + ] + async for uploaded_file in api.upload_release_assets( + "foo/bar", "1.2.3", files + ): + print(f"Uploaded: {uploaded_file}") + """ + github_json = await self.get(repo, tag) + asset_url = github_json["upload_url"].replace("{?name,label}", "") + + async def upload_file(file_path, content_type): + response = await self._client.post( + asset_url, + params={"name": file_path.name}, + content_type=content_type, + content=upload(file_path), + ) + return response, file_path + + tasks = [] + for file_path in files: + if isinstance(file_path, Tuple): + file_path, content_type = file_path + else: + content_type = "application/octet-stream" + + tasks.append(upload_file(file_path, content_type)) + + for coroutine in asyncio.as_completed(tasks): + response, file_path = await coroutine + response.raise_for_status() + yield file_path + class GitHubRESTReleaseMixin: def create_tag( diff --git a/tests/github/api/test_release.py b/tests/github/api/test_release.py index 164e8ace9..c9e07728f 100644 --- a/tests/github/api/test_release.py +++ b/tests/github/api/test_release.py @@ -352,6 +352,54 @@ async def test_download_release_assets_filter(self): with self.assertRaises(StopAsyncIteration): await anext(assets_it) + async def test_upload_release_assets(self): + response = create_response() + response.json.return_value = { + "upload_url": "https://uploads/assets{?name,label}", + } + post_response = create_response() + self.client.get.return_value = response + self.client.post.return_value = post_response + + file1 = MagicMock(spec=Path) + file1.name = "foo.txt" + content1 = b"foo" + file1.open.return_value.__enter__.return_value.read.side_effect = [ + content1 + ] + file2 = MagicMock(spec=Path) + file2.name = "bar.pdf" + content2 = b"bar" + file2.open.return_value.__enter__.return_value.read.side_effect = [ + content2 + ] + upload_files = [file1, (file2, "application/pdf")] + + it = aiter( + self.api.upload_release_assets("foo/bar", "v1.2.3", upload_files) + ) + f = await anext(it) + self.assertEqual(f, file1) + + f = await anext(it) + self.assertEqual(f, file2) + + args = self.client.post.await_args_list[0].args + self.assertEqual(args, ("https://uploads/assets",)) + kwargs = self.client.post.await_args_list[0].kwargs + self.assertEqual(kwargs["params"], {"name": "foo.txt"}) + self.assertEqual(kwargs["content_type"], "application/octet-stream") + + args = self.client.post.await_args_list[1].args + self.assertEqual(args, ("https://uploads/assets",)) + kwargs = self.client.post.await_args_list[1].kwargs + self.assertEqual(kwargs["params"], {"name": "bar.pdf"}) + self.assertEqual(kwargs["content_type"], "application/pdf") + + self.client.get.assert_awaited_once_with( + "/repos/foo/bar/releases/tags/v1.2.3" + ) + class GitHubReleaseTestCase(unittest.TestCase): @patch("pontos.github.api.api.httpx.post")