Created
May 19, 2016 20:38
-
-
Save manodeep/2c2dbc9e929c319ba97700f402ae0ec1 to your computer and use it in GitHub Desktop.
parallel async download from https://community.nitrous.io/tutorials/asynchronous-programming-with-python-3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# encoding=utf-8 | |
# para_async.py | |
import asyncio | |
import itertools | |
import aiohttp | |
@asyncio.coroutine | |
async def download(url, parts): | |
async def get_partial_content(u, i, start, end): | |
print(i, start, end) | |
async with aiohttp.get( | |
u, headers={ | |
"Range": "bytes={}-{}".\ | |
format(start, end - 1 if end else "")}) as _resp: | |
return i, await _resp.read() | |
async with aiohttp.head(url) as resp: | |
size = int(resp.headers["Content-Length"]) | |
ranges = list(range(0, size, size // parts)) | |
res, _ = await asyncio.wait( | |
[get_partial_content(url, i, start, end) for i, (start, end) in | |
enumerate(itertools.zip_longest(ranges, ranges[1:], fillvalue=""))]) | |
sorted_result = sorted(task.result() for task in res) | |
return b"".join(data for _, data in sorted_result) | |
if __name__ == '__main__': | |
image_url = "http://files.vladstudio.com/joy/where_tahrs_live/wall/"\ | |
"vladstudio_where_tahrs_live_2880x1800_signed.jpg" | |
loop = asyncio.get_event_loop() | |
bs = loop.run_until_complete(download(image_url, 16)) | |
with open("test_para_async.jpeg", "wb") as fi: | |
fi.write(bs) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment