Compares the performance of AstroPy between different versions of Python (2.7 vs 3.x).

plot-astropy-benchmarks-by-python-version.py

"""Compares the performance of AstroPy between different versions of Python (2.7 vs 3.x).

This script reads AstroPy's benchmarking output (cf. github.com/astropy/astropy-benchmarks.git)
as generated by the airspeed velocity benchmarking package (cf. asv.readthedocs.io).
It then compares & plots the median execution time for different Python versions.

This script was hacked together as part of the Hack Day at the 229th American Astronomical 
Society (AAS) meeting held in Grapevine, Texas. (Twitter: #hackaas #aas229.)

The output of this script is shown in the following tweet:
https://twitter.com/GeertHub/status/817857735484198912

Author: Geert Barentsen
Date: Jan 7th, 2017.
"""
import json
import numpy as np

import seaborn as sns
import matplotlib.pyplot as pl
from matplotlib.ticker import MultipleLocator
from matplotlib import style

style.use('gray.mplstyle')


def flatdict(d):
    result = {}
    for key in d.keys():
        if isinstance(d[key], float) or isinstance(d[key], int):
            result[key] = d[key]
    return result


# Read in the Python 2 benchmarking result as the comparison dataset
js2 = json.load(open('m3large/a48bcbe5-conda-py2.7-Cython-jinja2-nomkl-numpy-scipy.json'))
js2_results = flatdict(js2['results'])

# Read in different Python 3.x benchmarking results to compare against 2.7
for version in ['3.4', '3.5', '3.6']:
    js3 = json.load(open('m3large/a48bcbe5-conda-py{}-Cython-jinja2-nomkl-numpy-scipy.json'.format(version)))
    js3_results = flatdict(js3['results'])

    delta = []
    delta_mem = []
    for key in js2_results.keys():
        if 'io' not in key and 'mem' not in key:
            d = js2_results[key] - js3_results[key]
            norm = js2_results[key] #+ js2_results[key]) / 2.
            delta.append(100 * (d / norm))
        if 'mem_' in key:
            d = js3_results[key] - js2_results[key]
            norm = (js3_results[key] + js2_results[key]) / 2.
            delta_mem.append(100 * (d / norm))

    print('{}'.format(version))
    print('====================')
    print('Using {} time benchmarks'.format(len(delta)))
    print('Median speed change: {:.0f}%'.format(np.median(delta)))
    print('Mean speed change: {:.0f}%'.format(np.mean(delta)))
    print('')

    #print('Using {} memory benchmarks'.format(len(delta_mem)))
    #print('Median mem use change: {:.2f}%'.format(np.median(delta_mem)))
    #print('Mean mem use change: {:.2f}%'.format(np.mean(delta_mem)))
    #print('')

    pl.hist(delta, bins=10)
    pl.title('AstroPy benchmarks on Python 2.7 vs 3.6')
    pl.xlabel('Time change [%]')
    pl.tight_layout()
    pl.savefig('performance-hist-{}.png'.format(version))
    pl.close()


# Make a pretty plot
pl.figure()
pl.tick_params(labelsize=20)
pl.barh(1, -18, align='center', facecolor='#27ae60')
pl.barh(2, -16, align='center', facecolor='#27ae60')
pl.barh(3, 1, align='center', facecolor='#c0392b')
pl.yticks([1, 2, 3], ('3.6 vs 2.7', '3.5 vs 2.7','3.4 vs 2.7'), fontsize=24)
pl.xticks([-15, -10, -5, 0], ['-15%', '-10%', '-5%', '0%'], fontsize=24)
pl.xlabel('Median Execution Time', fontsize=24)
pl.title('AstroPy Benchmarks*\nby Python version', fontsize=30)
pl.xlim([-20, 3])
pl.annotate('(*) https://github.com/astropy/astropy-benchmarks', (0,0), (50, -70),
            xycoords='axes fraction', textcoords='offset points', va='top')
pl.tight_layout(rect=(0, 0.05, 1, 1))
pl.savefig('performance-summary.png')
pl.close