自定义user agent

define_ua.py

UA_LIST = [a.split("\n")[0].strip() for a in open("user_agent", "r").readlines()]

random_value = lambda lst: random.randint(0, len(lst))

def down_page(url):
    global UA_LIST
    headers = { "User-Agent" : random_value(UA_LIST)}
    req = urllib2.Request(url, headers=headers)
    response = urllib2.urlopen(req)
    the_page = response.read()
    return the_page