gistfile1.py

b.c# @author Rolando Espinoza La fuente <contacto@rolandoespinoza.info>
import httplib
import urllib

try:
    import cStringIO as StringIO
except ImportError:
    import StringIO

class HTTPError(Exception):
    """ raised when there is an HTTP error"""

class HTTPResponse(object):
    content = None
    content_was_truncated = False
    status_code = None
    headers = None
    final_url = None

class UrlFetch(object):
    _curl = None

    def curl(self):
        if not UrlFetch._curl:
            #print 'setting up curl'
            import pycurl
            ## Load curl on demand
            curl = pycurl.Curl()
            opts = (
                (pycurl.CONNECTTIMEOUT, 10),
                (pycurl.TIMEOUT, 30),
                (pycurl.MAXCONNECTS, 10),
                (pycurl.USERAGENT, 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)'),
                (pycurl.FOLLOWLOCATION, 1),
                (pycurl.NOSIGNAL, 1),
            )
            for (opt, val) in opts:
                curl.setopt(opt, val)
            UrlFetch._curl = curl
        return UrlFetch._curl

    def get(self, url, **kwargs):
        curl = self.curl()
        curl.setopt(curl.HTTPGET, 1)
        return self.urlfetch(url, **kwargs)

    def post(self, url, data_dict, **kwargs):
        data = urllib.urlencode(data_dict)
        curl = self.curl()
        curl.setopt(curl.POSTFIELDS, data)
        curl.setopt(curl.POST, 1)
        return urlfetch(url, **kwargs)

    def urlfetch(self, url, **kwargs):
        import pycurl
        data = StringIO.StringIO()
        header = StringIO.StringIO()
        curl = self.curl()
        curl.setopt(curl.WRITEFUNCTION, data.write)
        curl.setopt(curl.HEADERFUNCTION, header.write)
        curl.setopt(curl.URL, url)

        if kwargs.has_key('referer'):
            curl.setopt(curl.REFERER, kwargs.get('referer'))
        if kwargs.has_key('debug'):
            curl.setopt(curl.VERBOSE, kwargs.get('debug'))

        if kwargs.has_key('curlopts'):
            curlopts = kwargs.get('curlopts')
            assert isinstance(curlopts, tuple), 'curlopts not tuple'
            for opt, val in curlopts:
                curl.setopt(getattr(pycurl, opt.upper()), val)

        try:
            curl.perform()
        except pycurl.error, e:
            raise HTTPError("curl error %s" % e)

        response_headers = self._parseHeaders(header)
        code = curl.getinfo(curl.RESPONSE_CODE)

        response = HTTPResponse()
        response.content = data.getvalue()
        response.status_code = code
        response.headers = response_headers
        try:
            response.final_url = response_headers['location']
        except KeyError:
            response.final_url = url

        return response

    def _parseHeaders(self, status_and_headers):
        status_and_headers.seek(0)
        # Ignore status line
        status_and_headers.readline()
        msg = httplib.HTTPMessage(status_and_headers)
        return dict(msg.items())

    def __del__(self):
        if self._curl:
            self._curl.close()


## direct access
urlfetch = UrlFetch().urlfetch