crawlable

#!/usr/bin/env python

import sys
import urlparse
import robotparser # yes it's part of the python core!

try:
    ua, url = sys.argv[0:2]
except:
    print "usage: crawlable googlebot http://example.com/awesome.html"
    sys.exit(1)

robots_url = 'http://%s/robots.txt' % urlparse.urlparse(url).netloc

p = robotparser.RobotFileParser()
p.set_url(robots_url)

if p.can_fetch(ua, url):
    print "yup"
else:
    print "nope"