urllib2 get the web content .

urllib.py

import urllib2, zlib

def GetContent(url, gzip=False, charset=None, headers=None):
    "用于解析url内容"
    try:
        msg = None
        if headers:
            request = urllib2.Request(url=url, headers=headers)
        else:
            request = urllib2.Request(url)
        content = urllib2.urlopen(request)
        if content.msg == 'OK' and content.getcode() == 200:
            encoding = content.headers.get('content-encoding', None)
            if encoding in ['gzip', 'deflate']:
                if encoding == 'gzip':
                    msg = GzipStream(content)
                elif encoding == 'deflate':
                    msg = zlib.decompress(content).read()
            else:
                msg = content.read()
            if charset:
                return msg.decode(charset).encode('utf8')
            return msg
    except Exception  as e:
        print e
    else:
        content.close()