Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
import requests
from bs4 import BeautifulSoup
from random import choice
def proxy_generator():
response = requests.get("https://sslproxies.org/")
soup = BeautifulSoup(response.content, 'html5lib')
proxy = {'https': choice(list(map(lambda x:x[0]+':'+x[1], list(zip(map(lambda x:x.text, soup.findAll('td')[::8]), map(lambda x:x.text, soup.findAll('td')[1::8]))))))}
return proxy
def data_scraper(request_method, url, **kwargs):
while True:
try:
proxy = proxy_generator()
print("Proxy currently being used: {}".format(proxy))
response = requests.request(request_method, url, proxies=proxy, timeout=7, **kwargs)
break
# if the request is successful, no exception is raised
except:
print("Connection error, looking for another proxy")
pass
return response
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment