Download async from different domains - exploration

async-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import asyncio\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "waiting_times =  {\n",
    "    \"www.slow.com\": 10,\n",
    "    \"www.medium.com\": 4,\n",
    "    \"www.fast.com\": 1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = [\n",
    "    \"www.fast.com/a\",\n",
    "    \"www.medium.com/a\",\n",
    "    \"www.fast.com/b\",\n",
    "    \"www.fast.com/c\",\n",
    "    \"www.slow.com/a\",\n",
    "    \"www.medium.com/b\",\n",
    "    \"www.fast.com/d\",\n",
    "    \"www.slow.com/b\",\n",
    "    \"www.medium.com/c\",\n",
    "    \"www.medium.com/d\",\n",
    "    \"www.medium.com/e\",\n",
    "    \"www.fast.com/e\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_domain(url):\n",
    "    return url.split(\"/\")[0]\n",
    "\n",
    "\n",
    "def fetch_url_sync(url):\n",
    "    print(f\"Start fetching url {url} synchronously\")\n",
    "    time.sleep(waiting_times[get_domain(url)])\n",
    "    print(f\"End fetching url {url} synchronously\")\n",
    "\n",
    "\n",
    "async def fetch_url_async(url):\n",
    "    print(f\"Start fetching url {url} asynchronously\")\n",
    "    await asyncio.sleep(waiting_times[get_domain(url)])\n",
    "    print(f\"End fetching url {url} asynchronously\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start fetching url www.fast.com/a synchronously\n",
      "End fetching url www.fast.com/a asynchronously\n",
      "Start fetching url www.medium.com/a synchronously\n",
      "End fetching url www.medium.com/a asynchronously\n",
      "Start fetching url www.fast.com/b synchronously\n",
      "End fetching url www.fast.com/b asynchronously\n",
      "Start fetching url www.fast.com/c synchronously\n",
      "End fetching url www.fast.com/c asynchronously\n",
      "Start fetching url www.slow.com/a synchronously\n",
      "End fetching url www.slow.com/a asynchronously\n",
      "Start fetching url www.medium.com/b synchronously\n",
      "End fetching url www.medium.com/b asynchronously\n",
      "Start fetching url www.fast.com/d synchronously\n",
      "End fetching url www.fast.com/d asynchronously\n",
      "Start fetching url www.slow.com/b synchronously\n",
      "End fetching url www.slow.com/b asynchronously\n",
      "Start fetching url www.medium.com/c synchronously\n",
      "End fetching url www.medium.com/c asynchronously\n",
      "Start fetching url www.medium.com/d synchronously\n",
      "End fetching url www.medium.com/d asynchronously\n",
      "Start fetching url www.medium.com/e synchronously\n",
      "End fetching url www.medium.com/e asynchronously\n",
      "Start fetching url www.fast.com/e synchronously\n",
      "End fetching url www.fast.com/e asynchronously\n",
      "Wall time: 45.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# synchronous\n",
    "for url in urls:\n",
    "    fetch_url_sync(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "# Within each domain group, we avoid setting up concurrent connections to avoid\n",
    "# being banned from accessing the server\n",
    "grouped_urls = [set(grp[1]) for grp in itertools.groupby(sorted(urls), key=get_domain)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'www.fast.com/a',\n",
       "  'www.fast.com/b',\n",
       "  'www.fast.com/c',\n",
       "  'www.fast.com/d',\n",
       "  'www.fast.com/e'},\n",
       " {'www.medium.com/a',\n",
       "  'www.medium.com/b',\n",
       "  'www.medium.com/c',\n",
       "  'www.medium.com/d',\n",
       "  'www.medium.com/e'},\n",
       " {'www.slow.com/a', 'www.slow.com/b'}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "async def fetch_urlgroup(group):\n",
    "    return [await fetch_url_async(url) for url in group]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "async def main():\n",
    "    await asyncio.gather(*(fetch_urlgroup(group) for group in grouped_urls))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import asyncio\n",
    "\n",
    "loop = asyncio.get_event_loop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Task pending name='Task-5' coro=<main() running at <ipython-input-9-d41a52f87ede>:1>>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start fetching url www.fast.com/d asynchronously\n",
      "Start fetching url www.medium.com/e asynchronously\n",
      "Start fetching url www.slow.com/b asynchronously\n",
      "End fetching url www.fast.com/d asynchronously\n",
      "Start fetching url www.fast.com/c asynchronously\n",
      "End fetching url www.fast.com/c asynchronously\n",
      "Start fetching url www.fast.com/e asynchronously\n",
      "End fetching url www.fast.com/e asynchronously\n",
      "Start fetching url www.fast.com/b asynchronously\n",
      "End fetching url www.medium.com/e asynchronously\n",
      "Start fetching url www.medium.com/a asynchronously\n",
      "End fetching url www.fast.com/b asynchronously\n",
      "Start fetching url www.fast.com/a asynchronously\n",
      "End fetching url www.fast.com/a asynchronously\n",
      "End fetching url www.medium.com/a asynchronously\n",
      "Start fetching url www.medium.com/d asynchronously\n",
      "End fetching url www.slow.com/b asynchronously\n",
      "Start fetching url www.slow.com/a asynchronously\n",
      "End fetching url www.medium.com/d asynchronously\n",
      "Start fetching url www.medium.com/b asynchronously\n",
      "End fetching url www.medium.com/b asynchronously\n",
      "Start fetching url www.medium.com/c asynchronously\n",
      "End fetching url www.slow.com/a asynchronously\n",
      "End fetching url www.medium.com/c asynchronously\n"
     ]
    }
   ],
   "source": [
    "loop.create_task(main())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "About 20 seconds :-)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}