python-requests-gevent-geventhttpclient

# coding: utf8
""" Attempt at gevent-compatible requests without any monkeypatching """

import geventhttpclient
from geventhttpclient import httplib as gehttplib
from geventhttpclient.useragent import CompatResponse

import httplib as _httplib

import requests
from requests import adapters


class GHTTPAdapter(adapters.HTTPAdapter):
    """ HTTP Adapter with overrides to use geventhttpclient """

    def init_poolmanager(self, connections, maxsize, block=adapters.DEFAULT_POOLBLOCK):
        super(GHTTPAdapter, self).init_poolmanager(connections, maxsize, block=block)
        self.poolmanager = PoolManager(
            num_pools=connections, maxsize=maxsize, block=block)


from requests.packages.urllib3.poolmanager import PoolManager as _PoolManager
from requests.packages.urllib3.poolmanager import SSL_KEYWORDS
#from requests.packages.urllib3.response import HTTPResponse
#from requests.packages.urllib3.exceptions import ...


class PoolManager(_PoolManager):

    ## XXX: copypasted (for the different globals)
    ## TODO?: just hack the globals... somehow.
    def _new_pool(self, scheme, host, port):
        pool_cls = pool_classes_by_scheme[scheme]
        kwargs = self.connection_pool_kw
        if scheme == 'http':
            kwargs = self.connection_pool_kw.copy()
            for kw in SSL_KEYWORDS:
                kwargs.pop(kw, None)

        return pool_cls(host, port, **kwargs)


from requests.packages.urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from requests.packages.urllib3.connection import HTTPConnection as _HTTPConnection
from requests.packages.urllib3.connection import HTTPSConnection as _HTTPSConnection


class HTTPResponseProxy(object):
    def __init__(self, resp):
        self.__resp = resp
        self.__respc = CompatResponse(resp)  ## Making this class a second layer of compatibility-
        self.msg = HTTPMessageAlike(resp.headers)
        #self.__content_consumed = False

    # @property
    # def fp(self):
    #     """ A hack for requests.packages.urllib3.util.response.is_fp_closed which is used in requests.packages.urllib3.response.HTTPResponse.stream. """
    #     #if self.__content_consumed:
    #     #    return None
    #     if self._body_buffer:
    #         ## Not everything from the geventhttpclient's buffer was read
    #         return self
    #     return None  ## 'virtually closed'
    # def read(self, *ar, **kwa):
    #     res = self.__respc.read(*ar, **kwa)
    #     if not res:
    #         self.__content_consumed = True
    #     return res

    # @property
    # def closed(self):
    #     return self.__resp.isclosed()
    @property
    def closed(self):
        return not self._body_buffer

    def __getattr__(self, name):
        try:
            return getattr(self.__respc, name)
        except AttributeError:
            try:
                return getattr(self.__resp, name)
            except AttributeError:
                if name == 'length':
                    return None
                raise


class HTTPMessageAlike(_httplib.HTTPMessage, object):
    def __init__(self, data):
        self.dict = dict(data)
        self.headers = ['%s: %s\r\n' % (k, v) for k, v in self.dict.items()]


fix_gehttplib_httpresponse = HTTPResponseProxy


class HTTPConnection(gehttplib.HTTPConnection, object):
    """ ... """
    def getresponse(self, *ar, **kwa):
        #res = super(HTTPConnection, self).getresponse(*ar, **kwa)
        res = gehttplib.HTTPConnection.getresponse(self, *ar, **kwa)
        res = fix_gehttplib_httpresponse(res)
        return res
#     def connect(self):
#         res = super(HTTPConnection, self).connect()
#         self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY,
#                         self.tcp_nodelay)
#         return res

class HTTPSConnection(gehttplib.HTTPSConnection, object):
    """ ... """
    def getresponse(self, *ar, **kwa):
        #res = super(HTTPConnection, self).getresponse(*ar, **kwa)
        res = gehttplib.HTTPSConnection.getresponse(self, *ar, **kwa)
        res = fix_gehttplib_httpresponse(res)
        return res
#     def connect(self):


pool_classes_by_scheme = {
    'http': type('GHTTPConnectionPool', (HTTPConnectionPool,), dict(ConnectionCls=HTTPConnection)),
    'https': type('GHTTPSConnectionPool', (HTTPSConnectionPool,), dict(ConnectionCls=HTTPSConnection)),
}




def make_session():
    session = requests.Session()
    session.mount('http://', GHTTPAdapter())
    session.mount('https://', GHTTPAdapter())
    return session