Skip to content

Instantly share code, notes, and snippets.

@numanturle
Last active September 12, 2019 22:59
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save numanturle/d7a89f13c70d0ffa9e9b30bb90ccc85f to your computer and use it in GitHub Desktop.
Save numanturle/d7a89f13c70d0ffa9e9b30bb90ccc85f to your computer and use it in GitHub Desktop.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# encoding=utf8
import urllib.request
import urllib.error
import time
from multiprocessing import Pool
start = time.time()
with open("url10.txt",'r',encoding='utf-8',errors='ignore') as inputdata:
urls=inputdata.readlines()
def checkurl(url):
try:
conn = urllib.request.urlopen(url)
except urllib.error.HTTPError as e:
# Return code error (e.g. 404, 501, ...)
# ...
print('HTTPError: {}'.format(e.code) + ', ' + url)
except urllib.error.URLError as e:
# Not an HTTP-specific error (e.g. connection refused)
# ...
print('URLError: {}'.format(e.reason) + ', ' + url)
else:
# 200
# ...
print('good' + ', ' + url)
if __name__ == "__main__":
p = Pool(processes=20)
result = p.map(checkurl, urls)
print("done in : ", time.time()-start)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment