Entire Site Archiver By sitemap.xml

archiver.py

#!/usr/bin/python

import sys, requests
from xml.dom.minidom import parseString
from user_agent import generate_user_agent

if len(sys.argv) <= 1:
  print("Usage: ./archiver.py <sitemap_url>")
  sys.exit(0)

def save_page(page_url):
  headers = {"User-Agent": generate_user_agent(os=("mac", "linux")), "Referer": "http://web.archive.org"}
  try:
    r = requests.get("http://web.archive.org/save/" + page_url, headers=headers)
    print("Saving: " + page_url)
    return True
  except requests.ConnectionError as e:
    print("Retry: " + page_url)
    return False

sitemap_url = str(sys.argv[1]).strip()
sitemap_src = requests.get(sitemap_url)
dom = parseString(sitemap_src.text)
locs = dom.getElementsByTagName("loc")
for loc in locs:
  page_url = loc.firstChild.nodeValue.strip()
  while True:
    status = save_page(page_url)
    if status == True:
      break