Fetch current traffic conditions from http://traffic.calit2.net/bayarea/ and save them to a timestamped file.

traffic_scraper.py

"""
Fetch current traffic conditions from http://traffic.calit2.net/bayarea/ and save them to a timestamped file.
"""

import re
import json
import codecs
import datetime
import requests
from lxml import html
import pandas as pd

rows = []
for hwyid in range(48):
    headers = {'User-agent':'Mozilla/5.0 (X11; Linux x86_64; rv:2.0.1) Gecko/20110506 Firefox/4.0.1'}
    R = requests.get('http://traffic.calit2.net/bayarea/traffic.jsp?hwyid='+str(hwyid), headers=headers)
    T = R.content

    U = codecs.decode(T, 'ISO-8859-1')
    H = html.fromstring(U)

    L = H.xpath('//table[@id="AutoNumber3"]/tr/td[1]')
    location_list = [l.text_content().strip() for l in L]

    L = H.xpath('//table[@id="AutoNumber3"]/tr/td[3]')
    speed_list = [l.text_content().strip() for l in L]

    rows = rows + [{'hwy_id':hwyid,'location':l,'speed':s} for l,s in zip(location_list, speed_list)]

D = pd.DataFrame(rows)

filename = '../data/traffic/'+datetime.datetime.now().strftime("%Y%m%d-%H-%M")+".csv"
D.to_csv(filename, index=False)

"""
--- Done ---
* Loop over highways
* Add cron job

--- ToDo ---
"""