warc2json.py

import sys
import warc
import json

if len(sys.argv) != 2:
    print("usage: %s FILENAME" % sys.argv[0])
    exit()

filename = sys.argv[1]

f = warc.open(filename)
for record in f:
    if record.type != 'response':
        continue
    trec_id = record.header['warc-trec-id']
    body = unicode(record.payload.read(), errors='ignore')
    body = body[body.find("\r\n\r\n")+4:]
    print(json.dumps({'id': trec_id, 'content': body}))