Skip to content

Commit

Permalink
Add: Add async API for uploading release assets to GitHub
Browse files Browse the repository at this point in the history
The order of the files yielded depends on their size and the network.
The tests for the upload heavily depend on the internal ordering in
asyncio of the as_completed function. Therefore the tests may be flacky
on different Python versions.
  • Loading branch information
bjoernricks committed Oct 25, 2022
1 parent 866a5fe commit 48a2666
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 1 deletion.
68 changes: 67 additions & 1 deletion pontos/github/api/release.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import asyncio
from pathlib import Path
from typing import (
AsyncContextManager,
Expand All @@ -36,6 +37,7 @@
DownloadProgressIterable,
download,
download_async,
upload,
)


Expand Down Expand Up @@ -204,12 +206,76 @@ async def download_asset(name, download_cm):
asset_url: str = asset_json.get("browser_download_url", "")
name: str = asset_json.get("name", "")

print("matches", name, Path(name), Path(name).match(match_pattern))
if match_pattern and not Path(name).match(match_pattern):
continue

yield name, download_async(self._client.stream(asset_url))

async def upload_release_assets(
self,
repo: str,
tag: str,
files: Iterable[Union[Path, Tuple[Path, str]]],
) -> AsyncIterator[Path]:
"""
Upload release assets asynchronously
Args:
repo: GitHub repository (owner/name) to use
tag: The git tag for the release
files: An iterable of file paths or an iterable of tuples
containing a file path and content types to upload as an asset
Returns:
yields each file after its upload is finished
Raises:
HTTPError if an upload request was invalid
Example:
.. code-block:: python
files = (Path("foo.txt"), Path("bar.txt"),)
async for uploaded_file in api.upload_release_assets(
"foo/bar", "1.2.3", files
):
print(f"Uploaded: {uploaded_file}")
files = [
(Path("foo.txt"), "text/ascii"),
(Path("bar.pdf"), "application/pdf"),
]
async for uploaded_file in api.upload_release_assets(
"foo/bar", "1.2.3", files
):
print(f"Uploaded: {uploaded_file}")
"""
github_json = await self.get(repo, tag)
asset_url = github_json["upload_url"].replace("{?name,label}", "")

async def upload_file(file_path, content_type):
response = await self._client.post(
asset_url,
params={"name": file_path.name},
content_type=content_type,
content=upload(file_path),
)
return response, file_path

tasks = []
for file_path in files:
if isinstance(file_path, Tuple):
file_path, content_type = file_path
else:
content_type = "application/octet-stream"

tasks.append(upload_file(file_path, content_type))

for coroutine in asyncio.as_completed(tasks):
response, file_path = await coroutine
response.raise_for_status()
yield file_path


class GitHubRESTReleaseMixin:
def create_tag(
Expand Down
48 changes: 48 additions & 0 deletions tests/github/api/test_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,54 @@ async def test_download_release_assets_filter(self):
with self.assertRaises(StopAsyncIteration):
await anext(assets_it)

async def test_upload_release_assets(self):
response = create_response()
response.json.return_value = {
"upload_url": "https://uploads/assets{?name,label}",
}
post_response = create_response()
self.client.get.return_value = response
self.client.post.return_value = post_response

file1 = MagicMock(spec=Path)
file1.name = "foo.txt"
content1 = b"foo"
file1.open.return_value.__enter__.return_value.read.side_effect = [
content1
]
file2 = MagicMock(spec=Path)
file2.name = "bar.pdf"
content2 = b"bar"
file2.open.return_value.__enter__.return_value.read.side_effect = [
content2
]
upload_files = [file1, (file2, "application/pdf")]

it = aiter(
self.api.upload_release_assets("foo/bar", "v1.2.3", upload_files)
)
f = await anext(it)
self.assertEqual(f, file1)

f = await anext(it)
self.assertEqual(f, file2)

args = self.client.post.await_args_list[0].args
self.assertEqual(args, ("https://uploads/assets",))
kwargs = self.client.post.await_args_list[0].kwargs
self.assertEqual(kwargs["params"], {"name": "foo.txt"})
self.assertEqual(kwargs["content_type"], "application/octet-stream")

args = self.client.post.await_args_list[1].args
self.assertEqual(args, ("https://uploads/assets",))
kwargs = self.client.post.await_args_list[1].kwargs
self.assertEqual(kwargs["params"], {"name": "bar.pdf"})
self.assertEqual(kwargs["content_type"], "application/pdf")

self.client.get.assert_awaited_once_with(
"/repos/foo/bar/releases/tags/v1.2.3"
)


class GitHubReleaseTestCase(unittest.TestCase):
@patch("pontos.github.api.api.httpx.post")
Expand Down

0 comments on commit 48a2666

Please sign in to comment.