warc-work.py

import warc

from StringIO import StringIO
from httplib import HTTPResponse

class FakeSocket():
    def __init__(self, response_str):
        self._file = StringIO(response_str)
    def makefile(self, *args, **kwargs):
        return self._file

for record in warc.open("eada.warc.gz"):
    if record.type == "response":
        resp = HTTPResponse(FakeSocket(record.payload.read()))
        resp.begin()
        if resp.getheader("content-type") == "text/html":
            print record['WARC-Target-URI']