aiofile tests

aiofile.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import asyncio\n",
    "import aiofile\n",
    "import hashlib\n",
    "import os\n",
    "import io\n",
    "import time\n",
    "\n",
    "\n",
    "CHUNK_SIZE = 1024 * 1024\n",
    "CHUNK_COUNT = 200\n",
    "PAYLOAD = os.urandom(CHUNK_SIZE * CHUNK_COUNT)\n",
    "FNAME = \"/tmp/test.file\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "loop = asyncio.get_event_loop()\n",
    "\n",
    "\n",
    "async def test_aio_reader():\n",
    "    async with aiofile.AIOFile(FNAME, \"rb\", loop=loop) as afp:\n",
    "        reader = aiofile.Reader(afp, chunk_size=CHUNK_SIZE)\n",
    "        md5sum = hashlib.md5()\n",
    "\n",
    "        async for chunk in reader:\n",
    "            if not chunk:\n",
    "                break\n",
    "\n",
    "            md5sum.update(chunk)\n",
    "        \n",
    "        return md5sum.hexdigest()\n",
    "\n",
    "\n",
    "async def test_aio_writer():\n",
    "    async with aiofile.AIOFile(FNAME, \"wb+\", loop=loop) as afp:\n",
    "        writer = aiofile.Writer(afp)\n",
    "        payload = io.BytesIO(PAYLOAD)\n",
    "\n",
    "        for chunk in iter(lambda: payload.read(CHUNK_SIZE), b''):\n",
    "            await writer(chunk)\n",
    "            \n",
    "\n",
    "async def test_reader():\n",
    "    def blocking():\n",
    "        with open(FNAME, \"rb\") as fp:\n",
    "            md5sum = hashlib.md5()\n",
    "\n",
    "            for _ in range(CHUNK_COUNT):\n",
    "                md5sum.update(fp.read(CHUNK_SIZE))\n",
    "        \n",
    "        return md5sum.hexdigest()\n",
    "\n",
    "    return await loop.run_in_executor(None, blocking)\n",
    "\n",
    "\n",
    "async def test_writer():\n",
    "    def blocking():\n",
    "        with open(FNAME, \"wb+\") as fp:\n",
    "            payload = io.BytesIO(PAYLOAD)\n",
    "\n",
    "            for _ in range(CHUNK_COUNT):\n",
    "                fp.write(payload.read(CHUNK_SIZE))\n",
    "\n",
    "    return await loop.run_in_executor(None, blocking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "source": [
    "print(\"Running tests with chunk_size:\", CHUNK_SIZE, \"chunk_count:\", CHUNK_COUNT)\n",
    "\n",
    "print(\"==================\")\n",
    "print(\"AIO writer test\")\n",
    "%timeit loop.run_until_complete(test_aio_writer())\n",
    "print(\"==================\\n\")\n",
    "\n",
    "print(\"==================\")\n",
    "print(\"AIO reader test\")\n",
    "%timeit loop.run_until_complete(test_aio_reader())\n",
    "print(\"==================\\n\")\n",
    "\n",
    "print(\"==================\")\n",
    "print(\"Thread writer test\")\n",
    "%timeit loop.run_until_complete(test_writer())\n",
    "print(\"==================\\n\")\n",
    "\n",
    "print(\"==================\")\n",
    "print(\"Thread reader test\")\n",
    "%timeit loop.run_until_complete(test_reader())\n",
    "print(\"==================\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

results.md

chunk size	chunk count	AIO writer test	AIO reader test	Thread writer test	Thread reader test
1048576	200	1.06 s ± 133 ms per loop	1.13 s ± 132 ms per loop	1.21 s ± 1.41 s per loop	808 ms ± 33.7 ms per loop
1048576	100	558 ms ± 130 ms per loop	542 ms ± 53.4 ms per loop	513 ms ± 137 ms per loop	427 ms ± 34.4 ms per loop
32768	1000	644 ms ± 125 ms per loop	649 ms ± 33.1 ms per loop	149 ms ± 38 ms per loop	129 ms ± 6.06 ms per loop
1024	100	50.2 ms ± 1.99 ms per loop	47.6 ms ± 3.71 ms per loop	1.05 ms ± 29.3 µs per loop	926 µs ± 62 µs per loop

results.txt

====================================================================================================
Running tests with chunk_size: 1048576 chunk_count: 200
==================
AIO writer test
1.06 s ± 133 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
AIO reader test
1.13 s ± 132 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
Thread writer test
The slowest run took 8.61 times longer than the fastest. This could mean that an intermediate result is being cached.
1.21 s ± 1.41 s per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
Thread reader test
808 ms ± 33.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================
====================================================================================================
Running tests with chunk_size: 1048576 chunk_count: 100
==================
AIO writer test
558 ms ± 130 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
AIO reader test
542 ms ± 53.4 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
Thread writer test
513 ms ± 137 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
Thread reader test
427 ms ± 34.4 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================
====================================================================================================
Running tests with chunk_size: 32768 chunk_count: 1000
==================
AIO writer test
644 ms ± 125 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
AIO reader test
649 ms ± 33.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
==================

==================
Thread writer test
149 ms ± 38 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
==================

==================
Thread reader test
129 ms ± 6.06 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
==================
====================================================================================================
Running tests with chunk_size: 1024 chunk_count: 100
==================
AIO writer test
50.2 ms ± 1.99 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
==================

==================
AIO reader test
47.6 ms ± 3.71 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
==================

==================
Thread writer test
1.05 ms ± 29.3 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)
==================

==================
Thread reader test
926 µs ± 62 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)
==================