Simple script that uses BeautifulSoup, requests, and urlparse to spider a sitemap.xml file (CNN used as example)

sitemap_spider.py

import os
import requests
from BeautifulSoup import BeautifulSoup
from urlparse import urlparse

sitemap_xml = "http://www.cnn.com/sitemaps/sitemap-specials-2013-11.xml"

sitemap_response = requests.get(sitemap_xml)
soup = BeautifulSoup(sitemap_response.content)

elements = soup.findAll("url")
urls = [elem.find("loc").string for elem in elements]

for url in urls:
    parsed = urlparse(url)
    # group all files from single domain in same folder
    folder = parsed.netloc
    # replace "/" with "__" so that files can work on-disk
    file = parsed.path.replace("/", "__")

    print "Downloading {url} to {folder}/{file}".format(
        url=url, folder=folder, file=file)

    try:
        os.mkdir(folder)
    except:
        pass

    resp = requests.get(url)
    with open(folder + "/" + file, "wb") as output:
        output.write(resp.content)