From e3fa905ea46f03b56cde662d1d7a03c4af82773a Mon Sep 17 00:00:00 2001 From: chenhao Date: Thu, 2 Feb 2023 17:37:21 +0800 Subject: [PATCH] update python asyncio download --- .../fluent_python/chapter18-2-asyncio.py | 38 +++++++ .../fluent_python/chapter18-downloadflag.py | 102 ++++++++++++++++++ .../fluent_python/chapter18-downloadflag2.py | 101 +++++++++++++++++ pyPractice/practice/try_random.py | 19 ++++ 4 files changed, 260 insertions(+) create mode 100644 pyPractice/fluent_python/chapter18-2-asyncio.py create mode 100644 pyPractice/fluent_python/chapter18-downloadflag.py create mode 100644 pyPractice/fluent_python/chapter18-downloadflag2.py create mode 100644 pyPractice/practice/try_random.py diff --git a/pyPractice/fluent_python/chapter18-2-asyncio.py b/pyPractice/fluent_python/chapter18-2-asyncio.py new file mode 100644 index 0000000..67f74b0 --- /dev/null +++ b/pyPractice/fluent_python/chapter18-2-asyncio.py @@ -0,0 +1,38 @@ +# BEGIN SPINNER_ASYNCIO +import asyncio +import itertools + + +async def spin(msg): # <1> + for char in itertools.cycle('|/-\\'): + status = char + ' ' + msg + print(status, flush=True, end='\r') + try: + await asyncio.sleep(.1) # <2> + except asyncio.CancelledError: # <3> + break + print(' ' * len(status), end='\r') + + +async def slow_function(): # <4> + # pretend waiting a long time for I/O + await asyncio.sleep(3) # <5> + return 42 + + +async def supervisor(): # <6> + spinner = asyncio.create_task(spin('thinking!')) # <7> + print('spinner object:', spinner) # <8> + result = await slow_function() # <9> + spinner.cancel() # <10> + return result + + +def main(): + result = asyncio.run(supervisor()) # <11> + print('Answer:', result) + + +if __name__ == '__main__': + main() +# END SPINNER_ASYNCIO \ No newline at end of file diff --git a/pyPractice/fluent_python/chapter18-downloadflag.py b/pyPractice/fluent_python/chapter18-downloadflag.py new file mode 100644 index 0000000..cc421fd --- /dev/null +++ b/pyPractice/fluent_python/chapter18-downloadflag.py @@ -0,0 +1,102 @@ +"""Download flags of countries (with error handling). +asyncio async/await version +""" +# BEGIN FLAGS2_ASYNCIO_TOP +import asyncio +import collections + +import aiohttp +from aiohttp import web +import tqdm + +from flags2_common import main, HTTPStatus, Result, save_flag + +# default set low to avoid errors from remote site, such as +# 503 - Service Temporarily Unavailable +DEFAULT_CONCUR_REQ = 5 +MAX_CONCUR_REQ = 1000 + + +class FetchError(Exception): # <1> + + def __init__(self, country_code): + self.country_code = country_code + + +async def get_flag(session, base_url, cc): # <2> + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + async with session.get(url) as resp: + if resp.status == 200: + return await resp.read() + elif resp.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.HttpProcessingError(code=resp.status, message=resp.reason, headers=resp.headers) + + +async def download_one(session, cc, base_url, semaphore, verbose): # <3> + try: + async with semaphore: # <4> + image = await get_flag(session, base_url, cc) # <5> + except web.HTTPNotFound: # <6> + status = HTTPStatus.not_found + msg = 'not found' + except Exception as exc: + raise FetchError(cc) from exc # <7> + else: + save_flag(image, cc.lower() + '.gif') # <8> + status = HTTPStatus.ok + msg = 'OK' + + if verbose and msg: + print(cc, msg) + + return Result(status, cc) + + +# END FLAGS2_ASYNCIO_TOP + + +# BEGIN FLAGS2_ASYNCIO_DOWNLOAD_MANY +async def downloader_coro(cc_list, base_url, verbose, concur_req): # <1> + counter = collections.Counter() + semaphore = asyncio.Semaphore(concur_req) # <2> + async with aiohttp.ClientSession() as session: # <8> + to_do = [download_one(session, cc, base_url, semaphore, verbose) for cc in sorted(cc_list)] # <3> + + to_do_iter = asyncio.as_completed(to_do) # <4> + if not verbose: + to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> + for future in to_do_iter: # <6> + try: + res = await future # <7> + except FetchError as exc: # <8> + country_code = exc.country_code # <9> + try: + error_msg = exc.__cause__.args[0] # <10> + except IndexError: + error_msg = exc.__cause__.__class__.__name__ # <11> + if verbose and error_msg: + msg = '*** Error for {}: {}' + print(msg.format(country_code, error_msg)) + status = HTTPStatus.error + else: + status = res.status + + counter[status] += 1 # <12> + + return counter # <13> + + +def download_many(cc_list, base_url, verbose, concur_req): + loop = asyncio.get_event_loop() + coro = downloader_coro(cc_list, base_url, verbose, concur_req) + counts = loop.run_until_complete(coro) # <14> + loop.close() # <15> + + return counts + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) +# END FLAGS2_ASYNCIO_DOWNLOAD_MANY \ No newline at end of file diff --git a/pyPractice/fluent_python/chapter18-downloadflag2.py b/pyPractice/fluent_python/chapter18-downloadflag2.py new file mode 100644 index 0000000..d3aa151 --- /dev/null +++ b/pyPractice/fluent_python/chapter18-downloadflag2.py @@ -0,0 +1,101 @@ +"""Download flags of countries (with error handling). +asyncio async/await version +""" +# BEGIN FLAGS2_ASYNCIO_TOP +import asyncio +import collections + +import aiohttp +from aiohttp import web +import tqdm + +from flags2_common import main, HTTPStatus, Result, save_flag + +# default set low to avoid errors from remote site, such as +# 503 - Service Temporarily Unavailable +DEFAULT_CONCUR_REQ = 5 +MAX_CONCUR_REQ = 1000 + + +class FetchError(Exception): # <1> + + def __init__(self, country_code): + self.country_code = country_code + + +async def get_flag(session, base_url, cc): # <2> + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + async with session.get(url) as resp: + if resp.status == 200: + return await resp.read() + elif resp.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.HttpProcessingError(code=resp.status, message=resp.reason, headers=resp.headers) + + +async def download_one(session, cc, base_url, semaphore, verbose): # <3> + try: + async with semaphore: # <4> + image = await get_flag(session, base_url, cc) # <5> + except web.HTTPNotFound: # <6> + status = HTTPStatus.not_found + msg = 'not found' + except Exception as exc: + raise FetchError(cc) from exc # <7> + else: + save_flag(image, cc.lower() + '.gif') # <8> + status = HTTPStatus.ok + msg = 'OK' + + if verbose and msg: + print(cc, msg) + + return Result(status, cc) + + +# END FLAGS2_ASYNCIO_TOP + + +# BEGIN FLAGS2_ASYNCIO_DOWNLOAD_MANY +async def downloader_coro(cc_list, base_url, verbose, concur_req): # <1> + counter = collections.Counter() + semaphore = asyncio.Semaphore(concur_req) # <2> + async with aiohttp.ClientSession() as session: # <8> + to_do = [download_one(session, cc, base_url, semaphore, verbose) for cc in sorted(cc_list)] # <3> + + to_do_iter = asyncio.as_completed(to_do) # <4> + if not verbose: + to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> + for future in to_do_iter: # <6> + try: + res = await future # <7> + except FetchError as exc: # <8> + country_code = exc.country_code # <9> + try: + error_msg = exc.__cause__.args[0] # <10> + except IndexError: + error_msg = exc.__cause__.__class__.__name__ # <11> + if verbose and error_msg: + msg = '*** Error for {}: {}' + print(msg.format(country_code, error_msg)) + status = HTTPStatus.error + else: + status = res.status + + counter[status] += 1 # <12> + + return counter # <13> + + +def download_many(cc_list, base_url, verbose, concur_req): + loop = asyncio.get_event_loop() + coro = downloader_coro(cc_list, base_url, verbose, concur_req) + counts = loop.run_until_complete(coro) # <14> + loop.close() # <15> + + return counts + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) \ No newline at end of file diff --git a/pyPractice/practice/try_random.py b/pyPractice/practice/try_random.py new file mode 100644 index 0000000..2c3b289 --- /dev/null +++ b/pyPractice/practice/try_random.py @@ -0,0 +1,19 @@ +# coding:utf-8 + +import random + +count1 = 0 +count2 = 0 +times = 10000 + +for i in range(times): + aaa = random.choice(range(10)) + print(aaa) + if aaa >= 3: + count1 += 1 + else: + count2 += 1 + +print(count1 / times) +print(count2 / times) +# random. \ No newline at end of file