bench_shells.py

#!/usr/bin/env python3

"""%prog [options] program_text

Executes a program in available shells for a number of runs, then calculates
basic statistics.
"""

import collections
import logging
import math
import optparse
import os
import sys
from subprocess import Popen, PIPE


name_to_shellpath = {
    'v8': os.getenv('V8_SHELL'),
    'tm': os.getenv('TM_SHELL'),
}
for name, shellpath in name_to_shellpath.items():
    if not (shellpath and os.path.exists(shellpath)):
        print('Shell does not exist:', name, '@ path:', shellpath, file=sys.stderr)
        sys.exit(-1)
name_to_options = {
    'tm': ['-j'],
}

avg = lambda s: float(sum(s)) / len(s)
stddev = lambda s, avg_val: math.sqrt(sum((item - avg_val) ** 2 for item in s) / float(len(s)))

def create_parser():
    parser = optparse.OptionParser(usage=__doc__)
    parser.add_option('--run-count', type=int, default=50, metavar='COUNT',
            help='number of times to run the benchmark')
    parser.add_option('--no-warmup', action='store_false', dest='warmup', default=True,
            help="don't perform a warmup execution of the program before timing it.")
    parser.add_option('--global-scope', action='store_true', default=False,
            help="execute the program in a global scope, not function scope")
    parser.add_option('--logging', action='store_true', dest='enable_logging', default=False)
    return parser


def _wrap_program_global_scope(program, warmup):
    wrapped_program_lines = []
    if warmup:
        wrapped_program_lines.append('{program}; /* Warmup. */'.format(program=program))
    wrapped_program_lines.append("""
    var start = Date.now();
    {program}
    print(Date.now() - start);
    """.format(program=program))
    wrapped_program = os.linesep.join(wrapped_program_lines)
    return wrapped_program


def _wrap_program_fun_scope(program, warmup):
    wrapped_program_lines = ['function __bench__() {{ {program} }};'.format(program=program)]
    if warmup:
        wrapped_program_lines.append("__bench__(); /* Warmup. */")
    wrapped_program_lines.append("""
    var start = Date.now();
    __bench__(); /* Tested. */
    print(Date.now() - start);
    """)
    wrapped_program = os.linesep.join(wrapped_program_lines)
    return wrapped_program


def wrap_program(program, global_scope, warmup):
    if global_scope:
        return _wrap_program_global_scope(program, warmup)
    else:
        return _wrap_program_fun_scope(program, warmup)


def gather_data(wrapped_program, options):
    name_to_cmd = {name: [shellpath] + name_to_options.get(name, []) + ['-e', wrapped_program]
            for name, shellpath in name_to_shellpath.items()}
    name_to_data = collections.defaultdict(list)
    for name, cmd in name_to_cmd.items():
        logging.info('Running command: %s', cmd)
        for i in range(options.run_count):
            stdout, _ = Popen(cmd, stdout=PIPE).communicate()
            name_to_data[name].append(int(stdout))
    return name_to_data


def print_data(name_to_data):
    print('Run count: ', len(next(iter(name_to_data.values()))))
    max_name_len = max(len(name) for name in name_to_data)
    for name, data in name_to_data.items():
        avg_val = avg(data)
        stddev_val = stddev(data, avg_val)
        print('{0:>{max_name_len}}: {1:<10} (+/- {2:>2.2f})'.format(name, avg_val, stddev_val, max_name_len=max_name_len))


def main():
    parser = create_parser()
    options, args = parser.parse_args()
    try:
        program = args[0]
    except IndexError:
        parser.error('program_text required')
    if options.enable_logging:
        logging.basicConfig(level=logging.DEBUG)
    wrapped_program = wrap_program(program, options.global_scope, options.warmup)
    name_to_data = gather_data(wrapped_program, options)
    print_data(name_to_data)


if __name__ == '__main__':
    main()