get_lag.py

#!/usr/bin/env python
import requests
import time, datetime

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

import models
import sys, os

sqlite_fname = sys.argv[1]
website_url = sys.argv[2]

sqlite_fullpath = os.path.abspath( sqlite_fname )
bind_url = 'sqlite:///' + sqlite_fullpath

engine = create_engine(bind_url)

models.Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)

session = Session()

while 1:
    start_time = datetime.datetime.now()
    t1 = time.time()
    content = None
    try:
        r = requests.get(website_url)
        status = r.status_code

        content = r.content
        del r
    except requests.exceptions.ConnectionError:
        status = -1
    t2 = time.time()
    dur_msec = (t2-t1)*1000.0

    if content is not None:
        clen = len(content)
    else:
        clen = None

    lag_row = models.Lag(website_url, start_time, dur_msec, status, clen)
    session.add(lag_row)
    session.commit()

    time.sleep(600.0)

models.py

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, DateTime, Float

Base = declarative_base()
class Lag(Base):
    __tablename__ = 'lags'

    id = Column(Integer, primary_key=True)
    url = Column(String)
    start_time = Column(DateTime)
    duration_msec = Column(Float)
    status = Column(Integer)
    content_length = Column(Integer)

    def __init__(self, url, start_time, duration_msec, status, content_length=None):
        self.url = url
        self.start_time = start_time
        self.duration_msec = duration_msec
        self.status = status
        self.content_length = content_length

    def __repr__(self):
        return "<Lag(%r,%r,%r,%r,%r)>" % (self.url, self.start_time, self.duration_msec, self.status, self.content_length)

plot_status.py

#!/usr/bin/env python
import requests
import time, datetime
import numpy as np

from sqlalchemy import create_engine, desc
from sqlalchemy.orm import sessionmaker

from models import Lag
import sys, os

sqlite_fname = sys.argv[1]
url = sys.argv[2]

sqlite_fullpath = os.path.abspath( sqlite_fname )
bind_url = 'sqlite:///' + sqlite_fullpath

engine = create_engine(bind_url)

Session = sessionmaker(bind=engine)

session = Session()

results = session.query(Lag).filter_by(url=url).order_by( desc(Lag.start_time)).all()
times = []
status = []
duration_msec = []
for r in results:
    times.append( r.start_time )
    status.append( r.status )
    duration_msec.append( r.duration_msec )

times = np.array(times)
status=np.array(status)
duration_msec = np.array(duration_msec)

if not len(times):
    sys.stderr.write('ERROR: no data for %r\n'%url)
    sys.exit(1)

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.colors

import datetime
two_weeks = datetime.timedelta(days=14)
now = datetime.datetime.now()
then = now-two_weeks

f = plt.figure()
ax = f.add_subplot(111)

for color,status_codes,mesg in [('g',[200],'OK'),
                                ('r',[404,503],'404 or 503 error'),
                                ('r',[-1],'timeout')]:
    conds = np.zeros( times.shape, dtype=np.bool )
    for status_code in status_codes:
        conds |= status==status_code
    clipdur = np.clip(duration_msec,0,5000)
    x = times[conds]
    y = clipdur[conds]
    if 1:
        cond = x>=then
        x = x[cond]
        y = y[cond]
    N = len(x)
    if N>0:
        ax.plot(x,y,'.',c=color,label=mesg+' (n=%d)'%N)
ax.legend(loc='upper left')
ax.set_title(url)
ax.set_ylim(0,5100)
ax.set_ylabel('duration to response (msec)')
ax.set_yticks([0,2500,5000])
ax.set_yticklabels(['0','2500','5000+'])
f.autofmt_xdate()
f.savefig('status.png')

To log, call:

python get_lag.py data.sqlite http://some.website/here

To plot, call:

python plot_status.py data.sqlite http://some.website/here