crawler.py

#!/usr/bin/env python

import os, sys, json
from urllib import urlopen, urlencode


GRAPH = 'https://graph.facebook.com'
VERBOSE = True

def main(argv):
    if len(argv) not in (4, 5):
        print >> sys.stderr, "usage: %s <access_token> <opengraph_path> <output_file_prefix> [request_limit]" % argv[0]
        return 1
    token, target, prefix = argv[1:4]
    limit = None
    if len(argv) == 5:
        limit = int(argv[4])
    for i, chunk in enumerate(request("%s/%s" % (GRAPH, target), token, limit), 1):
        output = '%s.%04d.json' % (prefix, i)
        if os.path.exists(output):
            print >> sys.stderr, "File exists, aborting: %s" % output
            return 1
        with open(output, 'wb') as stream:
            json.dump(chunk, stream, indent=2)
    return 0

def request(target, token, limit=None):
    params = dict(access_token=token, date_format='r')
    if limit is not None:
        params['limit'] = limit
    reply = fetch(target, params)
    while True:
        if not reply['data']:
            break
        # The next URL will maintain all necessary query parameters.
        target = reply['paging']['next']
        yield reply
        reply = fetch(target)

def fetch(url, params=None):
    if params:
        url += '?%s' % urlencode(params)
    if VERBOSE:
        print url
    reply = json.load(urlopen(url))
    if 'error' in reply:
        raise Exception(json.dumps(reply))
    return reply

if __name__ == '__main__':
    sys.exit(main(sys.argv))