Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Data processing script to calculate the relationship between request time and DOM load time.
Data processing script to calculate the relationship between request time and
DOM load time.
import apachelog #
import re
import urlparse
def get_stats(log_data, format=None, max_time=6000):
Parses an apache log file for two parameters ("dom" and "request").
Returns a list of tuples of related request and dom times.
if format is None:
format = r'%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"'
p = apachelog.parser(format)
log = [] # Stats collected from the log file.
for line in log_data:
parsed = p.parse(line)
url = re.match(r'(.*) (.*) (.*)', parsed['%r']) # Break out the URL components.
parsed['method'] =
parsed['url'] =
parsed['protocol'] =
parsed['urlparsed'] = urlparse.urlparse(
parsed['params'] = urlparse.parse_qs(parsed['urlparsed'].query)
if 'dom' in parsed['params'] and 'request' in parsed['params']:
# We have the two params we need.
result = [] # Output data.
for entry in log:
dom = int(entry['params']['dom'][0])
request = int(entry['params']['request'][0])
if max_time > request > 0 and max_time > dom > 0:
# We have a pair of valid values so add to result.
result.append((request, dom))
except ValueError:
return result
if __name__ == '__main__':
import sys
import numpy
import matplotlib.pyplot as plt
data = sys.stdin.readlines() # Take log data from stdin - quick and dirty.
stats = get_stats(data)
# Working out y = nx + c where y is the Request time, x is the DOM time and n is the fudge factor.
x_list = []
y_list = []
for point in stats:
average_request = int(sum(y_list)) / len(y_list)
# This is the mechanics of solving y = nx + c using numpy.
x = numpy.array(x_list)
y = numpy.array(y_list)
A = numpy.vstack([x, numpy.ones(len(x))]).T
# Linear regression using least squares:
n, c = numpy.linalg.lstsq(A, y)[0]
# Now draw the result as a graph.
plt.axhline(y=average_request, label='Avg request (%d)' % (average_request), color='g')
plt.plot(x, y, 'o', label='Original data points (%d)' % (len(x)), markersize=2.5)
plt.plot(x, n*x + c, 'r', label='Fitted line: y = %sx + %s' % (round(n,3), int(c)))
plt.legend(loc='upper left')
plt.xlabel('DOM timer (ms)')
plt.ylabel('Request timer (ms)')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment