Skip to content

Instantly share code, notes, and snippets.

@theassyrian
Forked from keg/python_url_checker.py
Created August 30, 2020 10:19
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save theassyrian/ae61ddffe7b22af18efedbae308d2769 to your computer and use it in GitHub Desktop.
Save theassyrian/ae61ddffe7b22af18efedbae308d2769 to your computer and use it in GitHub Desktop.
Python redirect checker
#!/usr/bin/python
import csv
import requests
csv_data_to_write = [['original url', 'destination url',
'final http status', 'redirect path']]
def redirect_history(history):
'''repack urls to a neat list'''
hops = []
for response in history:
hops.append(response.url)
history = ' -> '.join(hops)
return history
def parse_file(filename='redirects_to_check.txt'):
'''parse a csv file of redirects and generate a results csv'''
with open(filename, 'rb') as redirect_file:
reader = csv.reader(redirect_file, delimiter=',')
for line in reader:
source_url = line[0]
destination_url = line[1]
reponse_text=''
response_status_code=''
response_history=''
csv_row_to_write = []
csv_row_to_write.append(source_url)
csv_row_to_write.append(destination_url)
try:
response = requests.head(source_url, allow_redirects=True,
timeout=20)
except Exception as e:
response = None
response_text = e
if response != None:
response_status_code=response.status_code
response_history = redirect_history(response.history)
for hop in response.history:
if hop.url == destination_url:
response_text = 'successful'
csv_row_to_write.append(response_text)
csv_row_to_write.append(response_status_code)
csv_row_to_write.append(response_history)
csv_data_to_write.append(csv_row_to_write)
def write_results():
'''helper function for writing out csv'''
with open('redirects_results.csv', 'w') as fp:
a = csv.writer(fp, delimiter=',')
a.writerows(csv_data_to_write)
#kick the process off
parse_file()
#write the results
write_results()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment