dict split performance test in https://www.v2ex.com/t/133087

Performance of test key in dict.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:14c3574d5e06f32581abcf9914523733b8c50fd7185a3a21daed6a843db3d58d"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from timeit import timeit\n",
      "setup = \"\"\"\n",
      "from collections import defaultdict\n",
      "from random import random\n",
      "tabledata = {\n",
      "\"A%d_000000_%d\" % (i, j): \"%f\" % (random() * 100,) for i in xrange(100) for j in xrange(100000)\n",
      "}\n",
      "\"\"\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "use_setdefault = \"\"\"\n",
      "new_d = {}\n",
      "for k in tabledata:\n",
      "    key, mid, subkey = k.split('_')\n",
      "    new_d.setdefault(key, []).append({subkey:tabledata[k]})\n",
      "\"\"\"\n",
      "print 'use_setdefault:', timeit(use_setdefault, setup=setup, number=10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "use_setdefault: "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "220.957348824\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "use_ifelse = \"\"\"\n",
      "new_d = {}\n",
      "for k in tabledata:\n",
      "    key, mid, subkey = k.split('_')\n",
      "    if key not in new_d:\n",
      "        new_d[key] = []\n",
      "    new_d[key].append({subkey:tabledata[k]})\n",
      "\"\"\"\n",
      "print 'use_ifelse:', timeit(use_ifelse, setup=setup, number=10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "use_ifelse: "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "211.045136929\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "use_try = \"\"\"\n",
      "new_d = {}\n",
      "for k in tabledata:\n",
      "    key, mid, subkey = k.split('_')\n",
      "    try:\n",
      "        new_d[key]\n",
      "    except KeyError:\n",
      "        new_d[key] = []\n",
      "    new_d[key].append({subkey:tabledata[k]})\n",
      "\"\"\"\n",
      "print 'use_try:', timeit(use_try, setup=setup, number=10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "use_try: "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "217.287450075\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "use_defaultdict = \"\"\"\n",
      "new_d = defaultdict(list)\n",
      "for k in tabledata:\n",
      "    key, mid, subkey = k.split('_')\n",
      "    new_d[key].append({subkey:tabledata[k]})\n",
      "\"\"\"\n",
      "print 'use_defaultdict:', timeit(use_defaultdict, setup=setup, number=10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "use_defaultdict: "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "207.548582792\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    }
   ],
   "metadata": {}
  }
 ]
}