smmap_memoryview_experiments

smmap_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import smmap\n",
    "import mmap\n",
    "import os, gc, random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 27\n",
      "drwxrwxr-x+ 1 ankostis ankostis    0 Oct 29 17:36 doc\n",
      "drwxrwxr-x+ 1 ankostis ankostis    0 Oct  1 21:36 etc\n",
      "-rwxrw-r--+ 1 ankostis ankostis 1519 Oct 29 17:36 LICENSE\n",
      "-rwxrw-r--+ 1 ankostis ankostis  486 Oct 29 17:36 Makefile\n",
      "-rwxrw-r--+ 1 ankostis ankostis 3826 Oct 30 00:28 README.md\n",
      "-rwxrw-r--+ 1 ankostis ankostis   74 Oct 29 17:36 setup.cfg\n",
      "-rwxrwxr-x+ 1 ankostis ankostis 1953 Oct 29 17:36 setup.py\n",
      "drwxrwxr-x+ 1 ankostis ankostis    0 Oct 30 17:04 smmap\n",
      "-rwxrwxr-x+ 1 ankostis ankostis 6783 Oct 30 18:06 smmap_scratch.ipynb\n",
      "-rwxrw-r--+ 1 ankostis ankostis  461 Oct 29 17:36 tox.ini\n"
     ]
    }
   ],
   "source": [
    "!ls -l\n",
    "f='README.md'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def proc_buffer(m):\n",
    "    s = 0\n",
    "    for n in range(10000):\n",
    "        ofs1 = random.randint(10, 3000)\n",
    "        ofs2 = random.randint(10, 3000)\n",
    "        if ofs1 > ofs2:\n",
    "            ofs1, ofs2 = ofs2, ofs1\n",
    "        s += sum(m[ofs1:ofs2])\n",
    "    return s\n",
    "              "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PY3 (memview)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 319 ms\n"
     ]
    }
   ],
   "source": [
    "random.seed(0)\n",
    "with open(f, 'r+b') as fp:\n",
    "    with mmap.mmap(fp.fileno(), 0) as memmap:\n",
    "        with memoryview(memmap) as m:\n",
    "            %time proc_buffer(m[10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 196 ms\n"
     ]
    }
   ],
   "source": [
    "random.seed(0)\n",
    "with open(f, 'r+b') as fp:\n",
    "    with mmap.mmap(fp.fileno(), 0) as memmap:\n",
    "        m = memmap\n",
    "        %time proc_buffer(m[10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 203 ms\n"
     ]
    }
   ],
   "source": [
    "random.seed(0)\n",
    "with open(f, 'r+b') as fp:\n",
    "    with mmap.mmap(fp.fileno(), 0) as memmap:\n",
    "        %time proc_buffer(memmap[10:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### leaks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "BufferError",
     "evalue": "cannot close exported pointers exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mBufferError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-fa58329152a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mmemmap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmmap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfileno\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmemmap\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mmemmap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mBufferError\u001b[0m: cannot close exported pointers exist"
     ]
    }
   ],
   "source": [
    "with open(f, 'r+b') as fp:\n",
    "    memmap = mmap.mmap(fp.fileno(), 0)\n",
    "    m = memoryview(memmap)\n",
    "    memmap.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "m[5]\n",
    "del m\n",
    "memmap.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PY2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def proc_buffer(m):\n",
    "    s = 0\n",
    "    for n in range(10000):\n",
    "        ofs1 = random.randint(10, 3000)\n",
    "        ofs2 = random.randint(10, 3000)\n",
    "        if ofs1 > ofs2:\n",
    "            ofs1, ofs2 = ofs2, ofs1\n",
    "        s += sum(1 for b in m[ofs1:ofs2])\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 711 ms\n"
     ]
    }
   ],
   "source": [
    "random.seed(0)\n",
    "with open(f, 'r+b') as fp:\n",
    "    memmap = mmap.mmap(fp.fileno(), 0)\n",
    "    try:\n",
    "        m = memoryview(bytearray(memmap))\n",
    "        %time proc_buffer(m[10:])\n",
    "    finally:\n",
    "        memmap.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 544 ms\n"
     ]
    }
   ],
   "source": [
    "random.seed(0)\n",
    "with open(f, 'r+b') as fp:\n",
    "    memmap = mmap.mmap(fp.fileno(), 0)\n",
    "    try:\n",
    "        m = bytearray(memmap)\n",
    "        %time proc_buffer(m[10:])\n",
    "    finally:\n",
    "        memmap.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 534 ms\n"
     ]
    }
   ],
   "source": [
    "random.seed(0)\n",
    "with open(f, 'r+b') as fp:\n",
    "    memmap = mmap.mmap(fp.fileno(), 0)\n",
    "    m = memmap\n",
    "    try:\n",
    "        %time proc_buffer(m[10:])\n",
    "    finally:\n",
    "        memmap.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### leaks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'buffer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-e7c62fc157cb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r+b'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mmemmap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmmap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfileno\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmemmap\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mmemmap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'buffer' is not defined"
     ]
    }
   ],
   "source": [
    "with open(f, 'r+b') as fp:\n",
    "    memmap = mmap.mmap(fp.fileno(), 0)\n",
    "    m = memoryview(buffer(memmap))\n",
    "    memmap.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'm' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-b188f48de21f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mm\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;31m## WinPython CRASHES!!\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'm' is not defined"
     ]
    }
   ],
   "source": [
    "m[5] ## WinPython CRASHES!!\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}