Quick evaluation of different idioms to concatenate lists in Python

List concatenation benchmarking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import chain\n",
    "from random import choices\n",
    "from timeit import timeit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "statements = ['a+b+c+d+e', 'list(chain(a,b,c,d,e))', '[*a,*b,*c,*d,*e]', '[*chain(a,b,c,d,e)]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Varying the size of the lists, 5 lists\n",
      "size  a + b + c + d + e  list(chain(a, b,   [*a, *b, *c, *d,   [*chain(a, b, c, \n",
      "      tot      per item  tot      per item  tot      per item  tot      per item\n",
      "--------------------------------------------------------------------------------\n",
      " 1e0  1.12e-05 2.25e-06  1.74e-05 3.47e-06  6.77e-06 1.35e-06  1.42e-05 2.83e-06  best: [*a, *b, *c, *d, *e]\n",
      " 1e1  1.57e-05 3.14e-07  3.04e-05 6.09e-07  1.27e-05 2.54e-07  2.46e-05 4.91e-07  best: [*a, *b, *c, *d, *e]\n",
      " 1e2  5.92e-05 1.18e-07  1.18e-04 2.36e-07  8.04e-05 1.61e-07  1.14e-04 2.28e-07  best: a + b + c + d + e\n",
      " 1e3  2.67e-04 5.35e-08  6.00e-04 1.20e-07  1.85e-04 3.69e-08  4.23e-04 8.45e-08  best: [*a, *b, *c, *d, *e]\n",
      " 1e4  2.56e-03 5.11e-08  3.24e-03 6.49e-08  7.20e-04 1.44e-08  3.31e-03 6.62e-08  best: [*a, *b, *c, *d, *e]\n",
      " 1e5  4.62e-02 9.24e-08  3.15e-02 6.30e-08  1.99e-02 3.97e-08  3.06e-02 6.11e-08  best: [*a, *b, *c, *d, *e]\n",
      " 1e6  7.56e-01 1.51e-07  6.10e-01 1.22e-07  3.61e-01 7.21e-08  5.43e-01 1.09e-07  best: [*a, *b, *c, *d, *e]\n"
     ]
    }
   ],
   "source": [
    "print(\"Varying the size of the lists, 5 lists\")\n",
    "\n",
    "# header\n",
    "print(f'{\"size\":4}', end='')\n",
    "for stmt in statements:\n",
    "    print(f'  {stmt:17.17}', end='')\n",
    "print()\n",
    "\n",
    "print(' ' * 4, end='')\n",
    "for _ in statements:\n",
    "    print(f'  {\"tot\":8} {\"per item\":8}', end='')\n",
    "print()\n",
    "    \n",
    "print('-' * (4 + 19 * len(statements)))\n",
    "\n",
    "for i in range(7):\n",
    "    size = 10**i\n",
    "    print(f' 1e{i}', end='')\n",
    "    \n",
    "    a, b, c, d, e = [choices(range(100), k=size) for _ in range(5)]\n",
    "    \n",
    "    best, tbest = None, float('inf')\n",
    "    for i, stmt in enumerate(statements):\n",
    "        t = timeit(stmt, number=10, globals=globals())\n",
    "        if t < tbest:\n",
    "            best, tbest = i, t\n",
    "        print(f'  {t:8.2e} {t / (5 * size):8.2e}', end='')\n",
    "    print(f'  best: {statements[best]}')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check, verify the 4 approaches do the same thing\n",
    "\n",
    "a + b + c + d + e == list(chain(a, b, c, d, e)) == [*a, *b, *c, *d, *e] == [*chain(a, b, c, d, e)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [f'l{i}' for i in range(1_000_000)]\n",
    "namespace = {name: choices(range(100), k=10) for name in names}\n",
    "namespace['chain'] = chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size  a + b + c + d + e  list(chain(a, b,   [*a, *b, *c, *d,   [*chain(a, b, c, \n",
      "      tot      per item  tot      per item  tot      per item  tot      per item\n",
      "--------------------------------------------------------------------------------\n",
      " 1e0  2.50e-06 4.99e-07  1.34e-05 2.69e-06  3.02e-06 6.05e-07  8.94e-06 1.79e-06  best: a + b + c + d + e\n",
      " 1e1  4.22e-05 8.44e-07  4.24e-05 8.48e-07  1.77e-05 3.55e-07  4.41e-05 8.83e-07  best: [*a, *b, *c, *d, *e]\n",
      " 1e2  1.90e-03 3.81e-06  1.17e-04 2.34e-07  6.94e-05 1.39e-07  1.12e-04 2.25e-07  best: [*a, *b, *c, *d, *e]\n",
      " 1e3  8.83e-02 1.77e-05  1.23e-03 2.46e-07  5.13e-04 1.03e-07  1.10e-03 2.19e-07  best: [*a, *b, *c, *d, *e]\n",
      " 1e4  RecursionError     1.75e-02 3.49e-07  1.61e-02 3.22e-07  2.39e-02 4.78e-07  best: [*a, *b, *c, *d, *e]\n",
      " 1e5  RecursionError     2.18e-01 4.36e-07  1.73e-01 3.47e-07  2.47e-01 4.94e-07  best: [*a, *b, *c, *d, *e]\n"
     ]
    }
   ],
   "source": [
    "# header\n",
    "print(f'{\"size\":4}', end='')\n",
    "for stmt in statements:\n",
    "    print(f'  {stmt:17.17}', end='')\n",
    "print()\n",
    "\n",
    "print(' ' * 4, end='')\n",
    "for _ in statements:\n",
    "    print(f'  {\"tot\":8} {\"per item\":8}', end='')\n",
    "print()\n",
    "    \n",
    "print('-' * (4 + 19 * len(statements)))\n",
    "for i in range(6):\n",
    "    size = 10**i\n",
    "    print(f' 1e{i}', end='')\n",
    "\n",
    "    \n",
    "    names_i = names[:size]\n",
    "    def join_with_comma(names):\n",
    "        return ', '.join(names)\n",
    "    cmds = [\n",
    "        ' + '.join(names_i),\n",
    "        f'list(chain({join_with_comma(names_i)}))',\n",
    "        f'[{join_with_comma(f\"*{name}\" for name in names_i)}]',\n",
    "        f'[*chain({join_with_comma(names_i)})]'\n",
    "    ]\n",
    "    \n",
    "    best, tbest = None, float('inf')\n",
    "    for j, cmd in enumerate(cmds):\n",
    "        try:\n",
    "            t = timeit(cmd, number=10, globals=namespace)\n",
    "        except Exception as e:\n",
    "            print(f'  {e.__class__.__name__:17.17}', end='')\n",
    "        else:\n",
    "            if t < tbest:\n",
    "                best, tbest = j, t\n",
    "            print(f'  {t:8.2e} {t / (5 * size):8.2e}', end='')\n",
    "    print(f'  best: {statements[best]}')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}