Sitemap Archiver

archiver.ts

import * as got from "got";
import { parseString as parseXML } from "xml2js";
import * as pify from "pify";
import * as puppeteer from "puppeteer";

const sitemapLocation = "";

async function run() {
  console.log("Requesting Sitemap");
  const sitemapResponse = await got(sitemapLocation);

  const sitemap: {
    urlset: {
      url: Array<{ loc: [string] }>;
    };
  } = await pify(parseXML)(sitemapResponse.body);

  const browser = await puppeteer.launch({
    headless: false
  });

  for (const url of sitemap.urlset.url) {
    const [location] = url.loc;

    console.log(`Archiving ${location}`);

    const page = await browser.newPage();
    await page.goto(`https://web.archive.org/save/${location}`, {
      waitUntil: "networkidle2"
    });
    await page.close();

    console.log(`Archived ${location}`);
  }

  await browser.close();
}

try {
  run();
} catch (e) {
  console.error("A Bad Happened!");
  console.error(e);
}