Skip to content

Instantly share code, notes, and snippets.

Last active September 17, 2020 11:01
What would you like to do?
import io
import asyncio
import aiohttp
import nest_asyncio
async def fetch_dataframe(session, url):
""" Asynchronously fetches a csv file using the url """
async with session.get(url) as response:
byte_code = await
df = pd.read_csv(io.BytesIO(byte_code), encoding='utf8')
return df
async def fetch_concurrent(urls):
""" Makes concurrent calls """
loop = asyncio.get_event_loop()
async with aiohttp.ClientSession() as session:
tasks = []
for url in urls:
tasks.append(loop.create_task(fetch_dataframe(session, url)))
all_dfs = []
# Loop through each completed task
for df in asyncio.as_completed(tasks):
df = await df
# Concatenate all fetched dataframes into one
return pd.concat(all_dfs, ignore_index=True)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment