get proxies

proxy_leech.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# quick & dirty nntime proxy leech
# 
# @author:     starenka
# @email:      'moc]tod[liamg].T.E[0aknerats'[::-1]
# @version:    1.0
# @since       Dec 22, 2010

import urlgrabber,re,sys
from BeautifulSoup import BeautifulSoup
from optparse import OptionParser
from random import choice

def get_page(number,anonymous=True):
    proxies = []
    data = urlgrabber.urlread('http://nntime.com/proxy-list-%s.htm'%
                              str(number).zfill(2))
    soup = BeautifulSoup(data)

    obfuscated = {}
    ob = re.compile(r'\+(\S)')

    ob_src = soup.findAll('script',text=re.compile(r'([a-z]=\d;)'))
    for pair in [i.split('=') for i in ob_src[0][0:-1].strip().split(';')]:
        obfuscated[pair[0]] = pair[1]

    for tr in soup.findAll('tr',{'class':'odd','class':'even'}):
        tds = tr.findAll('td')[1:]
        if anonymous and tds[1].string.strip()[0] not in ['a','e','h']:
            break
        proxies.append({'type':tds[1].string.strip()[0].upper(),
                        'ip':tds[0].contents[0],
                        'port':''.join([obfuscated[o] for o in ob.findall(str(tds[0].contents[1].contents))]),
                        'country':tds[3].string,
                        'isp':tds[4].string
                        }
        )
    return proxies

usage = '%s target_url'%(sys.argv[0])
oparser = OptionParser(usage)
oparser.add_option('-l','--list',action='store_true',dest='list',default = False, help='print plain list w/ ip:port')
oparser.add_option('-a','--anonymous',action='store_true',dest='anonymous',default = False, help='no transparent proxies')

(options,args) = oparser.parse_args()

proxies = []
while not proxies:
    proxies = get_page(choice(range(1,50)),options.anonymous)

for proxy in proxies:
    if options.list:
        print '%s:%s'%(proxy['ip'],proxy['port'])
    else:
        print '[%s] %s:%s\n    %s, %s'%(proxy['type'],
                                       proxy['ip'],
                                       proxy['port'],
                                       proxy['country'],
                                       proxy['isp'])