Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
check a list of urls to see if they exist
#!/usr/bin/env python
import logging
from optparse import OptionParser
import os
import requests
import sys
def check_urls_from_file(f):
statuses = {}
for line in f:
url = line.strip().rstrip()
if len(url) == 0:
continue
logging.debug("checking " + url)
r = requests.head(url)
status = r.status_code
if not status in statuses:
statuses[status] = 0
statuses[status] += 1
logging.info("%s %i" % (url, status))
total = 0
for status, count in statuses.iteritems():
print "%s: %i" % (status, count)
total += count
print "total: %i" % total
def _main():
usage = "usage: %prog"
parser = OptionParser(usage=usage,
description="")
parser.add_option("-d", "--debug", action="store_true", dest="debug",
help="Turn on debug logging")
parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
help="turn off all logging")
(options, args) = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if options.debug else
(logging.ERROR if options.quiet else logging.INFO))
if not options.debug:
logging.getLogger("requests").setLevel(logging.WARNING)
for arg in args:
with open(arg, "rb") as f:
check_urls_from_file(f)
if len(args) == 0:
check_urls_from_file(sys.stdin)
if __name__ == "__main__":
_main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment