Reads a text file of URLs and writes out a CSV report of whether the URL is in swap.stanford.edu

swap_check.py

#!/usr/bin/env python3

import csv
import sys
import json
import time
import requests

def get_snapshots(url):
    url = f"https://swap.stanford.edu/was/cdx?url={url}&output=json"
    resp = requests.get(url)
    if resp.status_code == 200 and resp.text:
        return sorted(
            [json.loads(line) for line in resp.text.strip().split("\n")],
            key=lambda d: d['timestamp'],
            reverse=True
        )
    else:
        return

def main():
    seed_filename = sys.argv[1]
    writer = csv.writer(sys.stdout)
    writer.writerow(["url", "latest_timestamp", "swap_url"])
    for url in open(seed_filename):
        url = url.strip()
        snapshots = get_snapshots(url)
        if snapshots is not None and len(snapshots) > 0:
            ts = snapshots[0]["timestamp"]
            writer.writerow([url, ts, f"https://swap.stanford.edu/was/{ts}/{url}"])
        else:
            writer.writerow([url, None, None])
        time.sleep(1)

if __name__ == "__main__":
    main()