xkcd 1416 - 'Pixels' image scraper

xkcd.py

import requests
import os

JSON_BASE = 'http://c.xkcd.com/turtle/'
IMAGE_BASE = 'http://imgs.xkcd.com/turtledown/'
IMAGE_SUFFIX = '-tiled.png'
START = 'turtles'

IMAGE_DIR = 'imgs/'

if not os.path.exists(IMAGE_DIR):
    os.mkdir(IMAGE_DIR)

seen = set()

def fetch_json(name):
    return requests.get(JSON_BASE + name).json()


def fetch_image(name, directory):
    if not os.path.exists(os.path.join(IMAGE_DIR, directory)):
        os.mkdir(os.path.join(IMAGE_DIR, directory))
    f = open(os.path.join(IMAGE_DIR, directory, name + IMAGE_SUFFIX), 'wb')
    f.write(requests.get(os.path.join(IMAGE_BASE, name + IMAGE_SUFFIX)).content)
    f.close()

def extract(seen, current):
    for key in current:
        for item in current[key]:
            if item not in seen:
                seen.add(item)
                fetch_image(item, key)
                extract(seen, fetch_json(item))

def main():
    seen = set()
    extract(seen, {'white': [START]})

if __name__ == "__main__":
    main()