Python concurrency benchmarks

README

Using a MacBook Air: 1.7 GHz Intel Core i7. GNU parallel and Python's multiprocessing report 4 "CPU". It's 2 physical CPUs and 4 cores.

Parallel is from homebrew (parallel-20130922).

It's not exactly scientifc time measurement, but the numbers are consistent.

cf.py

import argparse
import concurrent.futures
import logging
import subprocess
import sys

logging.basicConfig(stream=sys.stderr, level=logging.INFO)
logger = logging.getLogger('pp')

parser = argparse.ArgumentParser(
    description="Emulates GNU parallel")
parser.add_argument(
    'program',
    metavar='PROGRAM',
    help="Program to run jobs")
known, unknown = parser.parse_known_args()

# find the parallel args
i = unknown.index(':::')
assert i >= 0
pargs = unknown[i+1:]

def runner(*args):
    return subprocess.check_output([known.program] + list(args))

def main():
    with concurrent.futures.ProcessPoolExecutor() as executor:
        for r in executor.map(runner, pargs):
            print(r.decode().rstrip())

if __name__ == '__main__':
    main()

GNU parallel vs concurrent.futures console

vas-y:parallelz seang$ time parallel wc ::: `find ~/writing/blog/2013 -name *.rst`
      18     128     791 /Users/seang/writing/blog/2013/10/07/downgrading-my-blog.rst
     106     567    4011 /Users/seang/writing/blog/2013/10/08/linking-geojson.rst
      34     249    1586 /Users/seang/writing/blog/2013/10/24/joining-mapbox.rst
      70     373    2910 /Users/seang/writing/blog/2013/11/24/introducing-rasterio.rst
      14      45     395 /Users/seang/writing/blog/2013/12/02/geojson-website-refreshed.rst
      25     158    1256 /Users/seang/writing/blog/2013/12/03/atom-extension-for-tinkerer.rst
      16      74     627 /Users/seang/writing/blog/2013/11/27/new-home-on-github-for-geojson.rst
     143     685    6211 /Users/seang/writing/blog/2013/12/04/json-diff-and-patch-for-geojson.rst
      26     202    1237 /Users/seang/writing/blog/2013/12/05/first-blog-post-at-mapbox.rst
      15      52     568 /Users/seang/writing/blog/2013/12/06/my-first-photo-in-wikimedia-commons.rst
     174     635    5351 /Users/seang/writing/blog/2013/12/06/some-consequences-of-geojson-features-as-arrays.rst
      24     162    1155 /Users/seang/writing/blog/2013/12/06/ylt-covers-nrbq-s-ridin-in-my-car.rst
      70     326    2315 /Users/seang/writing/blog/2013/12/13/is-rasterio-fast-enough.rst
      99     876    5611 /Users/seang/writing/blog/2013/12/17/teaching-python-gis-users-to-be-more-rational.rst
      69     283    2209 /Users/seang/writing/blog/2013/12/21/rasterio-windows-and-masks.rst
      14      47     345 /Users/seang/writing/blog/2013/12/25/gruss-vom-krampus.rst
      45     413    2559 /Users/seang/writing/blog/2013/09/11/leaving-isaw.rst
      30     205    1526 /Users/seang/writing/blog/2013/12/31/shapely-1-2-19-and-1-3-0.rst

real	0m0.241s
user	0m0.179s
sys	0m0.141s
vas-y:parallelz seang$ time python pp.py wc ::: `find ~/writing/blog/2013 -name *.rst`
      45     413    2559 /Users/seang/writing/blog/2013/09/11/leaving-isaw.rst
      18     128     791 /Users/seang/writing/blog/2013/10/07/downgrading-my-blog.rst
     106     567    4011 /Users/seang/writing/blog/2013/10/08/linking-geojson.rst
      34     249    1586 /Users/seang/writing/blog/2013/10/24/joining-mapbox.rst
      70     373    2910 /Users/seang/writing/blog/2013/11/24/introducing-rasterio.rst
      16      74     627 /Users/seang/writing/blog/2013/11/27/new-home-on-github-for-geojson.rst
      14      45     395 /Users/seang/writing/blog/2013/12/02/geojson-website-refreshed.rst
      25     158    1256 /Users/seang/writing/blog/2013/12/03/atom-extension-for-tinkerer.rst
     143     685    6211 /Users/seang/writing/blog/2013/12/04/json-diff-and-patch-for-geojson.rst
      26     202    1237 /Users/seang/writing/blog/2013/12/05/first-blog-post-at-mapbox.rst
      15      52     568 /Users/seang/writing/blog/2013/12/06/my-first-photo-in-wikimedia-commons.rst
     174     635    5351 /Users/seang/writing/blog/2013/12/06/some-consequences-of-geojson-features-as-arrays.rst
      24     162    1155 /Users/seang/writing/blog/2013/12/06/ylt-covers-nrbq-s-ridin-in-my-car.rst
      70     326    2315 /Users/seang/writing/blog/2013/12/13/is-rasterio-fast-enough.rst
      99     876    5611 /Users/seang/writing/blog/2013/12/17/teaching-python-gis-users-to-be-more-rational.rst
      69     283    2209 /Users/seang/writing/blog/2013/12/21/rasterio-windows-and-masks.rst
      14      47     345 /Users/seang/writing/blog/2013/12/25/gruss-vom-krampus.rst
      30     205    1526 /Users/seang/writing/blog/2013/12/31/shapely-1-2-19-and-1-3-0.rst

real	0m0.172s
user	0m0.076s
sys	0m0.076s
vas-y:parallelz seang$ time ~/code/buildout.python/python-3.4/bin/python cf.py wc ::: `find ~/writing/blog/2013 -name *.rst`
      45     413    2559 /Users/seang/writing/blog/2013/09/11/leaving-isaw.rst
      18     128     791 /Users/seang/writing/blog/2013/10/07/downgrading-my-blog.rst
     106     567    4011 /Users/seang/writing/blog/2013/10/08/linking-geojson.rst
      34     249    1586 /Users/seang/writing/blog/2013/10/24/joining-mapbox.rst
      70     373    2910 /Users/seang/writing/blog/2013/11/24/introducing-rasterio.rst
      16      74     627 /Users/seang/writing/blog/2013/11/27/new-home-on-github-for-geojson.rst
      14      45     395 /Users/seang/writing/blog/2013/12/02/geojson-website-refreshed.rst
      25     158    1256 /Users/seang/writing/blog/2013/12/03/atom-extension-for-tinkerer.rst
     143     685    6211 /Users/seang/writing/blog/2013/12/04/json-diff-and-patch-for-geojson.rst
      26     202    1237 /Users/seang/writing/blog/2013/12/05/first-blog-post-at-mapbox.rst
      15      52     568 /Users/seang/writing/blog/2013/12/06/my-first-photo-in-wikimedia-commons.rst
     174     635    5351 /Users/seang/writing/blog/2013/12/06/some-consequences-of-geojson-features-as-arrays.rst
      24     162    1155 /Users/seang/writing/blog/2013/12/06/ylt-covers-nrbq-s-ridin-in-my-car.rst
      70     326    2315 /Users/seang/writing/blog/2013/12/13/is-rasterio-fast-enough.rst
      99     876    5611 /Users/seang/writing/blog/2013/12/17/teaching-python-gis-users-to-be-more-rational.rst
      69     283    2209 /Users/seang/writing/blog/2013/12/21/rasterio-windows-and-masks.rst
      14      47     345 /Users/seang/writing/blog/2013/12/25/gruss-vom-krampus.rst
      30     205    1526 /Users/seang/writing/blog/2013/12/31/shapely-1-2-19-and-1-3-0.rst

real	0m0.107s
user	0m0.108s
sys	0m0.069s

pp.py

import argparse
from multiprocessing import Pool
import logging
import subprocess
import sys

logging.basicConfig(stream=sys.stderr, level=logging.INFO)
logger = logging.getLogger('pp')

parser = argparse.ArgumentParser(
    description="Emulates GNU parallel")
parser.add_argument(
    'program',
    metavar='PROGRAM',
    help="Program to run jobs")
known, unknown = parser.parse_known_args()

# find the parallel args
i = unknown.index(':::')
assert i >= 0
pargs = unknown[i+1:]

def runner(*args):
    return subprocess.check_output([known.program] + list(args))

if __name__ == '__main__':
    pool = Pool()
    results = pool.map(runner, pargs)
    for r in results:
        print(r.rstrip())

I love it a lot, why not turning it into a Python library ? (We can beat perl any day of the week)