bench.py

# with pyramid tweak (and 80-column wrapping)

import time
import sys
import StringIO

class Bench(object):
    def __init__(self, flags=None):
       ''' Flags:
           * cgi: Build a new app every time.
           * dny: Return the :name part of a /hello/:name url.
       '''
       self.flags = flags or []
       self.url = '/'
       self.body = 'Hello World!'
       if 'urlargs' in self.flags:
           self.url = '/hello/bench'
           self.body = 'bench'

    def start_response(self, status, header):
        if status != '200 OK':
            raise AssertionError('%r %s %r' % (self, status, header))
        pass

    def make_environ(self, run_once=False):
       env = {
         'REQUEST_METHOD': 'GET',
         'SCRIPT_NAME': '',
         'PATH_INFO': self.url,
         'QUERY_STRING': '',
         'SERVER_NAME': 'localhost',
         'SERVER_PORT': '80',
         'SERVER_PROTOCOL': 'HTTP/1.0',
         'wsgi.version': (1,0),
         'wsgi.url_scheme': 'http',
         'wsgi.input': StringIO.StringIO(),
         'wsgi.errors': sys.stdout,
         'wsgi.multithread': False,
         'wsgi.multiprocess': False,
         'wsgi.run_once': run_once,
       }
       return env

    def test_cold(self, mode, n=1000):
        start = time.time()
        for i in xrange(n):
            app = self.make_app()
            body = ''.join(app(self.make_environ(run_once=True),
                               self.start_response))
        if body != self.body:
            raise AssertionError('%r: %r != %r' % (self, body, self.body))
        return n / (time.time() - start)

    def test_warm(self, mode, n=1000):
        app = self.make_app()
        ''.join(app(self.make_environ(), self.start_response))
        start = time.time()
        for i in xrange(n):
            body = ''.join(app(self.make_environ(), self.start_response))
        if body != self.body:
            raise AssertionError('%r: %r != %r' % (self, body, self.body))
        return n / (time.time() - start)
       

class WSGIBench(Bench):
    name = 'Plain WSGI'
    def make_app(self):
        def app(environ, start_response):
            start_response('200 OK',  [('content-type','text/html')])
            path = environ['PATH_INFO']
            if path.startswith('/hello/'):
                return [path[7:]]
            return ['Hello World!']
        return app

class BottleBench(Bench):
    name = 'Bottle'
    def make_app(self):
        import bottle
        app = bottle.Bottle()
        @app.route('/')
        @app.route('/hello/:name')
        def hello(name='Hello World!'):
            return name
        return app

class PyramidBench(Bench):
    name = 'Pyramid'
    def make_app(self):
        from pyramid.config import Configurator
        from pyramid.response import Response
        response = Response('Hello World!')
        def hello_world(request):
            return response
        def hello_name(request):
            return Response(request.matchdict['name'])
        config = Configurator()
        config.add_view(hello_world)
        config.add_route('foo', '/hello/{name}', view=hello_name)
        return config.make_wsgi_app()

class FlaskBench(Bench):
    name = 'Flask'
    def make_app(self):
        import flask
        app = flask.Flask(__name__)
        @app.route('/')
        @app.route('/hello/<name>')
        def hello(name='Hello World!'):
            return name
        return app

suites = [obj for obj in globals().values() if isinstance(obj, type) and
          issubclass(obj, Bench)]
suites.remove(Bench)
suites.sort(key=lambda c: c.__name__)

if __name__ == '__main__':

    flags = sys.argv[1:]
    n, r = 100, 10

    print
    print 'Python Framework Benchmark'
    print
    print 'All results are shown as requests per second (high is good).'
    print 'In "Normal" mode, setup and first request are not considered '
    print '(warm start).  In "CGI" mode, a new app is created for each '
    print 'request (cold start).'
    print
    print '% 20s  % 10s % 10s' % ('', 'Normal', 'CGI')

    for suite in suites:
        print '% 20s ' % suite.name,
        sys.stdout.flush()
    
        rps = min(suite(flags).test_warm(n) for i in xrange(r))
        print '% 10d' % rps,
        sys.stdout.flush()
    
        rps = min(suite(flags).test_cold(n) for i in xrange(r))
        print '% 10d' % rps
        sys.stdout.flush()