Skip to content

Instantly share code, notes, and snippets.

@ChiChou ChiChou/hidemyass.py Secret
Last active Nov 8, 2016

Embed
What would you like to do?
#!/usr/bin/env python
#coding:utf-8
'''
Author: @CodeColorist
Requirements: pip install lxml beautifulsoup4 tinycss
Usage:
from hidemyass import proxies
...
for proxy in proxies():
# do something
'''
import urllib2
import tinycss
from bs4 import BeautifulSoup
def proxies():
# check whether decl is a css property that hides the element
isinvisible = lambda decl: (decl.name == 'display' and decl.value.as_css() == 'none') \
or (decl.name == 'visibility' and decl.value.as_css() == 'hidden')
css_parser = tinycss.make_parser()
html = urllib2.urlopen('http://proxylist.hidemyass.com').read()
soup = BeautifulSoup(html, 'lxml')
table = soup.find('table', id='listable')
rows = table.find_all('tr')[1:] # skip first row
# remove invisible elements
for style in table.find_all('style'):
invisible_classes = [rule.selector.as_css() for rule in css_parser.parse_stylesheet(style.text).rules if \
any(map(isinvisible, rule.declarations))]
[e.extract() for e in table.select(','.join(invisible_classes))]
style.extract()
[e.extract() for e in table.select('[style]') if e.name != 'tr' and any(filter(
lambda rule: any(map(isinvisible, rule)), css_parser.parse_style_attr(e.get('style'))))]
# parse data
order = ('lastupdate', 'ip', 'port', 'country', 'speed', 'conectiontime', 'type', 'anon')
for tr in rows:
td = tr.find_all('td')
yield {key: td[i].div.get('value') \
if key in ('speed', 'conectiontime') else td[i].text.strip() \
for i, key in enumerate(order)}
if __name__ == '__main__':
print list(proxies())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.