Scraping xkcd's click & drag bit by bit.

scrape_xkcd.py

import json
import os
import time
import requests

ROOT_DIR = "/Users/melinath/Pictures/xkcd"
MAX_QUADRANT_SIZE = 100
NOT_FOUND_PATH = os.path.join(ROOT_DIR, 'not_found.json')

if os.path.exists(NOT_FOUND_PATH):
    with open(NOT_FOUND_PATH, 'r') as f:
        not_found = set(json.load(f))
else:
    not_found = set(())

for quadrant_size in xrange(MAX_QUADRANT_SIZE):
    print "Quadrant size: ", quadrant_size + 1
    for vdir in ('n', 's'):
        for hdir in ('w', 'e'):
            for vdist in xrange(quadrant_size):
                for hdist in xrange(quadrant_size):
                    name = "".join((str(vdist + 1), vdir, str(hdist + 1), hdir, '.png'))
                    path = os.path.join(ROOT_DIR, name)
                    if not os.path.exists(path) and name not in not_found:
                        response = requests.get("http://imgs.xkcd.com/clickdrag/{name}".format(name=name))
                        if response.status_code == 200:
                            fp = open(path, 'w')
                            fp.write(response.content)
                        elif response.status_code == 404:
                            not_found.add(name)
                        else:
                            print response.status_code, name
                        time.sleep(1)
    with open(NOT_FOUND_PATH, 'w') as f:
        json.dump(list(not_found), f)