Script to test the status for a bunch of urls, with aggregate counts in the end. Usage: python test_urls.py file_with_404_links.txt

test_urls.py

import sys
from collections import defaultdict

import requests

file_name = sys.argv[1]

results = defaultdict(list)

with open(file_name) as f:
    for url in f:
        r = requests.get(url, allow_redirects=False)
        results[r.status_code].append(url)
        print "%d - %s" % (r.status_code, url)

print "========================"
for status, urls in results.items():
    print "%d - %d" % (status, len(urls))