replay_traffic.py

#!/usr/bin/fades --python=python2
import eventlet # fades==0.14.0
from eventlet.green import urllib2 # fades==0.14.0
import argparse
import time
import logging
from logging.handlers import RotatingFileHandler
eventlet.monkey_patch()
#logging config
logfile = "replay.log"
handler = RotatingFileHandler(logfile, maxBytes=10*1024*1024, backupCount=10)
formatter = logging.Formatter("%(asctime)s  %(name)-22s  "
                              "%(levelname)-8s %(message)s")
handler.setFormatter(formatter)
logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(logging.INFO)
log = logging.getLogger('log_replay')


def replay_get(uri, userAgent):
    url = domain + uri
    request = urllib2.Request(url)
    request.add_unredirected_header('User-Agent', userAgent)
    log.debug("url: %s , user agent= %s", url, userAgent )
    try:
        response = urllib2.urlopen(request)
        log.info('GET a %s [OK] http Status = %s', url, response.code)
    except (urllib2.HTTPError, urllib2.URLError), exc:
        log.error("GET a %s [ERROR]: %s", url, exc)


def apache_log_parser(access_log):
    pool = eventlet.GreenPool(size=concurrency)
    encolados = 0
    for row in access_log:
        access_log_line = row.split()
        uri = access_log_line[7]
        userAgent = access_log_line[13].strip('"')
        pool.spawn_n(replay_get, uri, userAgent)
        encolados += 1
        if encolados % 10 == 0:
            log.info('running: %s', pool.running())
            log.info('free: %s', pool.free())
    while pool.running():
        time.sleep(1)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Replay a Apache access.log (NCSA) file to a specific domain')
    parser.add_argument(
        '-l',
        '--log',
        type= file,
        help='Access log file to replay')
    parser.add_argument('-d', '--domain', help='Domain to replay http://example.com/')
    parser.add_argument(
        '-c',
        '--concurrency',
        help='Number of multiple requests to make. Default = 5',
        type=int,
        default=5)
    parser.add_argument(
        '-v',
        '--verbose',
        help='Print debug info',
        action='store_true',
        default= False)
    args = parser.parse_args()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    access_log = args.log
    domain = args.domain
    concurrency = args.concurrency
    apache_log_parser(access_log)

• depends on fades
• works with this log:

xxx.xxx.xxx.xxx - - [17/Sep/2013:00:25:55 -0300] "GET /foo/bar HTTP/1.1" 200 17516 "-" "UserAgent" f:/path/to/some/index.php h:xxx.xxx.xxx.xxx