httplib error with gevent, and xmlrpclib (update: no longer requires sleekxmpp to repro)

xmlrpc_broken.py

# from geventhttpclient import httplib
# httplib.patch()
from gevent import monkey; monkey.patch_all()
import gevent
from time import sleep
import logging
import xmlrpclib

server = 'dev.vine.im'
logging.basicConfig(level=logging.DEBUG, format='%(asctime)-15s %(levelname)-7s - %(message)s')
xmlrpc_server = xmlrpclib.ServerProxy('http://%s:4560' % server)
        
def add_rosteritem(user, other_user):
   gevent.spawn(_xmlrpc_command, 'add_rosteritem', {
        'localuser': user,
        'localserver': server,
        'user': other_user,
        'server': server,
        'group': 'argh',
        'nick': other_user,
        'subs': 'both'
    })

def _xmlrpc_command(command, data):
    logging.debug('XMLRPC ejabberdctl: %s %s' % (command, str(data)))
    fn = getattr(xmlrpc_server, command)
    return fn({
        'user': '_leaves', 
        'server': server,
        'password': 'somepassword'
    }, data)

if __name__ == '__main__':
    for i in range(1,20):
        add_rosteritem('jabberwocky', 'user%d' % i)
    sleep(2)

xmlrpc_fixed.py

from time import sleep
import logging
import xmlrpclib
import httplib
import io

server = 'dev.vine.im'
logging.basicConfig(level=logging.DEBUG, format='%(asctime)-15s %(levelname)-7s - %(message)s')

class FireAndForget(xmlrpclib.Transport):
    mock_xml_response = u'<?xml version="1.0"?><methodResponse><params><param><value><struct><member><name>res</name><value><int>0</int></value></member></struct></value></param></params></methodResponse>'
    
    def single_request(self, host, handler, request_body, verbose=0):
        logging.warn(self._connection)
        h = self.make_connection(host)
        if verbose:
            h.set_debuglevel(1)
        try:
            self.send_request(h, handler, request_body)
            self.send_host(h, host)
            self.send_user_agent(h)
            self.send_content(h, request_body)
            self.verbose = verbose
            h.close()  # h is closed by the standard transport anyway, so it's fine to ignore the response by doing this.
            response = io.StringIO(self.mock_xml_response)
            return self.parse_response(response)
        except xmlrpclib.Fault:
            raise
        except Exception:
            # All unexpected errors leave connection in a strange state, so we clear it.
            self.close()
            raise

xmlrpc_server = xmlrpclib.ServerProxy('http://%s:4560' % server, transport=FireAndForget())
        
def add_rosteritem(user, other_user):
    _xmlrpc_command('add_rosteritem', {
        'localuser': user,
        'localserver': server,
        'user': other_user,
        'server': server,
        'group': 'argh',
        'nick': other_user,
        'subs': 'both'
    })

def _xmlrpc_command(command, data):
    logging.debug('XMLRPC ejabberdctl: %s %s' % (command, str(data)))
    fn = getattr(xmlrpc_server, command)
    return fn({
        'user': '_leaves', 
        'server': server,
        'password': 'somepassword'
    }, data)

if __name__ == '__main__':
    for i in range(3):
        add_rosteritem('jabberwocky', 'user%d' % i)
    sleep(2)

UPDATE on Jan 8th at 5am EST (old comments pushed down)

I no longer need SleekXMPP to repro this issue. The problem was that the script was exiting before the XMLRPC requests were made. The xmpp.connect() call was delaying the script long enough for them to be made, and to fail, but a sleep(2) accomplishes the same thing. I've removed the SleekXMPP code for simplicity.

On one of my VM's (with Python 2.6.5) this prints:

(xmpp-env)vagrant@lucid32:/vagrant/xmpp-env/xmpp$ python test.py 
gevent 1.0rc2
sleekxmpp 1.1.11
xmlrpclib 1.0.1
instantiation successful

But on another, newer VM (with Python 2.7.1) this fails with:

(xmpp-env)vagrant@vagrant:/vagrant/source_dir/xmpp-env/xmpp$ python test.py 
gevent 1.0rc2
sleekxmpp 1.1.11
xmlrpclib 1.0.1
instantiation successful
Traceback (most recent call last):
  File "build/bdist.linux-x86_64/egg/gevent/greenlet.py", line 328, in run
    result = self._run(*self.args, **self.kwargs)
  File "test.py", line 36, in _xmlrpc_command
    }, data)
  File "/usr/lib/python2.7/xmlrpclib.py", line 1224, in __call__
    return self.__send(self.__name, args)
  File "/usr/lib/python2.7/xmlrpclib.py", line 1578, in __request
    verbose=self.__verbose
  File "/usr/lib/python2.7/xmlrpclib.py", line 1264, in request
    return self.single_request(host, handler, request_body, verbose)
  File "/usr/lib/python2.7/xmlrpclib.py", line 1289, in single_request
    self.send_request(h, handler, request_body)
  File "/usr/lib/python2.7/xmlrpclib.py", line 1391, in send_request
    connection.putrequest("POST", handler, skip_accept_encoding=True)
  File "/usr/lib/python2.7/httplib.py", line 856, in putrequest
    raise CannotSendRequest()
CannotSendRequest
<Greenlet at 0x168bb90: <bound method TestComponent._xmlrpc_command of <__main__.TestComponent object at 0x168e890>>('add_rosteritem', {'localuser': 'user2', 'localserver': 'dev.vine.im)> failed with CannotSendRequest

It works, however, on both VMs if I comment out line the last line of my script and don't connect the XMPP component. It also works if I only make a single add_rosteritem call, but in my application I need to be able to make several at once.

When running code from my actual application on the new VM, I see errors like this as well:

Traceback (most recent call last):
  File "build/bdist.linux-x86_64/egg/gevent/greenlet.py", line 328, in run
    result = self._run(*self.args, **self.kwargs)
  File "/vagrant/source_dir/xmpp-env/xmpp/ejabberdctl.py", line 87, in _xmlrpc_command
    }, data)
  File "/usr/lib/python2.7/xmlrpclib.py", line 1224, in __call__
    return self.__send(self.__name, args)
  File "/usr/lib/python2.7/xmlrpclib.py", line 1578, in __request
    verbose=self.__verbose
  File "/usr/lib/python2.7/xmlrpclib.py", line 1264, in request
    return self.single_request(host, handler, request_body, verbose)
  File "/usr/lib/python2.7/xmlrpclib.py", line 1294, in single_request
    response = h.getresponse(buffering=True)
  File "/usr/lib/python2.7/httplib.py", line 1018, in getresponse
    raise ResponseNotReady()
ResponseNotReady
<Greenlet at 0x1d9f370: <bound method EjabberdCTL._xmlrpc_command of <ejabberdctl.EjabberdCTL object at 0x1c9efd0>>('add_rosteritem', {'localuser': 'dormouse', 'localserver': 'dev.vine)> failed with ResponseNotReady

I think this is the relevant bit of httplib.py that's failing (full source here):

    ...
    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
        """Send a request to the server.

        `method' specifies an HTTP request method, e.g. 'GET'.
        `url' specifies the object being requested, e.g. '/index.html'.
        `skip_host' if True does not add automatically a 'Host:' header
        `skip_accept_encoding' if True does not add automatically an
           'Accept-Encoding:' header
        """

        # if a prior response has been completed, then forget about it.
        if self.__response and self.__response.isclosed():
            self.__response = None


        # in certain cases, we cannot issue another request on this connection.
        # this occurs when:
        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
        #   2) a response to a previous request has signalled that it is going
        #      to close the connection upon completion.
        #   3) the headers for the previous response have not been read, thus
        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
        #
        # if there is no prior response, then we can request at will.
        #
        # if point (2) is true, then we will have passed the socket to the
        # response (effectively meaning, "there is no prior response"), and
        # will open a new one when a new request is made.
        #
        # Note: if a prior response exists, then we *can* start a new request.
        #       We are not allowed to begin fetching the response to this new
        #       request, however, until that prior response is complete.
        #
        if self.__state == _CS_IDLE:
            self.__state = _CS_REQ_STARTED
        else:
            raise CannotSendRequest()
        ...

Also, I think this:

    ...
    def getresponse(self, buffering=False):
        "Get the response from the server."

        # if a prior response has been completed, then forget about it.
        if self.__response and self.__response.isclosed():
            self.__response = None

        #
        # if a prior response exists, then it must be completed (otherwise, we
        # cannot read this response's header to determine the connection-close
        # behavior)
        #
        # note: if a prior response existed, but was connection-close, then the
        # socket and response were made independent of this HTTPConnection
        # object since a new request requires that we open a whole new
        # connection
        #
        # this means the prior response had one of two states:
        #   1) will_close: this connection was reset and the prior socket and
        #                  response operate independently
        #   2) persistent: the response was retained and we await its
        #                  isclosed() status to become true.
        #
        if self.__state != _CS_REQ_SENT or self.__response:
            raise ResponseNotReady()
        ...