Sitemap-driven crawler

crawlsitemap.py

import xml.dom.minidom, urllib

def getText(nodelist):
    rc = []
    for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
            rc.append(node.data)
    return ''.join(rc)


dom = xml.dom.minidom.parseString(urllib.urlopen('http://www.lesters.com/sitemap/sitemap.xml').read())

locs = dom.getElementsByTagName('loc')

for loc in locs:
	url = getText(loc.childNodes)
	urllib.urlretrieve(url)
	print url