Command line http proxy (for benchmarking)

cmd-proxy.py

#!/usr/bin/env python
import os
import tornado.httpserver
import tornado.ioloop
import tornado.web
import subprocess
import shlex
import fcntl
import errno
from cStringIO import StringIO

BUFFSIZE = 8192
ID = 0
DEBUG = 0
class MainHandler(tornado.web.RequestHandler):
    auto_etag = False
    
    @tornado.web.asynchronous
    def get(self, args):
        global ID
        ID += 1
        self.id = ID
        cmd = shlex.split(str(args))
        
        if DEBUG: 
            print "#%s ------------------------------------- " % self.id
            print "#%s --- RUNNING: %r" % (self.id, cmd)
            print "#%s ------------------------------------- " % self.id
        else:
            print "#%s --- RUNNING: %r" % (self.id, cmd)


        self.ioloop = tornado.ioloop.IOLoop.instance()
        self.proc = proc = subprocess.Popen(
            cmd,
            stdout = subprocess.PIPE,
            stderr = subprocess.PIPE,
            bufsize = BUFFSIZE,
        )
        self.buff = StringIO()
        self.stderr_fd = proc.stderr.fileno()
        self.stdout_fd = proc.stdout.fileno()
        self.set_nonblocking(self.stderr_fd)
        self.set_nonblocking(self.stdout_fd)
        self.ioloop.add_handler(self.stdout_fd, self.on_output, self.ioloop.READ)
        self.ioloop.add_handler(self.stderr_fd, self.on_output, self.ioloop.READ)

    def set_nonblocking(self, fd):
        flags = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)

    def read_from(self, fd):
        self.set_nonblocking(fd)
        try:
            while 1:
                data = os.read(fd, BUFFSIZE)
		if not data:
                    break
                if DEBUG:
                    for line in data.splitlines():
                        print "#%s: %s" % (self.id, line)
                self.buff.write(data)
        except OSError, e:
            if e.errno not in (errno.EAGAIN, errno.EWOULDBLOCK,
                            errno.EINPROGRESS):
                raise
    
    @tornado.web.asynchronous
    def on_output(self, fd, events):
        self.read_from(self.stdout_fd)
        self.read_from(self.stderr_fd)
        
        ret = self.proc.poll()
        if ret is not None:
            self.ioloop.remove_handler(self.stdout_fd)
            self.ioloop.remove_handler(self.stderr_fd)
            self.set_status(200 if ret is 0 else 500)
            self.set_header('Content-Type', 'text/plain')
            self.write(self.buff.getvalue())
            self.finish()
            if DEBUG:
                print "#%s ================= " % self.id
                print "#%s === COMPLETED === " % self.id
                print "#%s ================= " % self.id
            else:
                print "#%s === COMPLETED === " % self.id

            if ret:
                import time
                file("error-%s-retcode-%s" % (time.time(), ret), 'wb').write(self.buff.getvalue())	


class JunkHandler(tornado.web.RequestHandler):
    def get(self):
        self.finish()

application = tornado.web.Application([
    (r"/favicon.ico", JunkHandler),
    (r"/(.*)", MainHandler),
])

if __name__ == "__main__":
    http_server = tornado.httpserver.HTTPServer(application)
    http_server.listen(19999, '127.0.0.1')
    print "Using:", tornado.ioloop._poll.__name__
    tornado.ioloop.IOLoop.instance().start()

httperf-results-to-csv.py

import re

REGEX = re.compile(
    ".*?"
    "connections (\d+).+"
    "duration ([\d.]+).+"
    "Connection rate: ([\d.]+).+"
    "Connection time \[ms\]: "
        "min ([\d.]+) "
        "avg ([\d.]+) "
        "max ([\d.]+) "
        "median ([\d.]+) "
        "stddev ([\d.]+).+"
    "Request rate: ([\d.]+) "
        "req\/s \(([\d.]+).+"
    "Reply rate.+"
        "min ([\d.]+) "
        "avg ([\d.]+) "
        "max ([\d.]+) "
        "stddev ([\d.]+).+"
    "Reply time.+"
        "response ([\d.]+).*"
    "Reply status: "
        "1xx=([\d.]+) "
        "2xx=([\d.]+) "
        "3xx=([\d.]+) "
        "4xx=([\d.]+) "
        "5xx=([\d.]+).*"
    ,
    re.IGNORECASE|re.MULTILINE|re.DOTALL
)
NAMES = (
    "total_connections",
    "test_duration",
    "connections_per_sec",
    "min_ms_per_connection",
    "avg_ms_per_connection",
    "max_ms_per_connection",
    "median_ms_per_connection",
    "stddev_ms_per_connection",
    "request_rate_per_sec",
    "ms_per_request",
    "min_ms_per_request",
    "avg_ms_per_request",
    "max_ms_per_request",
    "stddev_ms_per_request",
    "reply_time_response",
    "status_1xx",
    "status_2xx",
    "status_3xx",
    "status_4xx",
    "status_5xx",
)

def parse(data):
    m = REGEX.match(data)
    if not m:
        raise Exception("Failed to parse")
    return zip(NAMES, m.groups())


import sys
import csv
from cStringIO import StringIO
output = StringIO()
csv_data = csv.writer(output)
csv_data.writerow(('concurrency',) + NAMES)


import os
for name in os.listdir(sys.argv[1]):
    path = os.path.join(sys.argv[1], name)
    if os.path.isfile(path):
        m = re.match('^c(.*)\.httperf$', name)
        if m:
            try:
                data = parse(file(path).read())
                csv_data.writerow([m.group(1)] + [val for key, val in data]) 
            except Exception, e:
                print e
print output.getvalue()