Gevent Crawler Demo

crawl.py

from gevent import monkey, spawn, joinall
monkey.patch_all()  # Magic!
import re, requests, random
URLS = set(['http://umd.edu', 'http://nytimes.com'])
def crawl(urls):
    try:
        while True:
            url = random.choice(list(urls))
            URLS.add(url)
            print(url)
            d = requests.get(url).content
            urls = re.findall('href="(http[^"]+)"', d)
            urls = set(urls) - URLS
    except:
        spawn(crawl, URLS)
joinall([spawn(crawl, URLS) for _ in range(100)])