additions to twisted.internet.defer.inlineCallbacks with lots of debugging info

noisyInlineTracebacks.py

# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
# See LICENSE for details.

import traceback
import warnings
from sys import exc_info

# Twisted imports
from twisted.python import failure, lockfile
from twisted.python.util import mergeFunctionMetadata
from twisted.internet.defer import Deferred, _DefGen_Return, returnValue

def _inlineCallbacks(result, g, deferred):
    """
    See L{inlineCallbacks}.
    """
    # This function is complicated by the need to prevent unbounded recursion
    # arising from repeatedly yielding immediately ready deferreds.  This while
    # loop and the waiting variable solve that by manually unfolding the
    # recursion.

    waiting = [True, # waiting for result?
               None] # result

    print "starting _inlineCallbacks(result=%r, g=%r (line %d), deferred=%r)" \
          % (result, g, g.gi_frame.f_lineno, deferred)
    while 1:
        try:
            # Send the last result back as the result of the yield expression.
            isFailure = isinstance(result, failure.Failure)
            if isFailure:
                print "throwing exception in"
                result = result.throwExceptionIntoGenerator(g)
            else:
                print "sending result"
                result = g.send(result)
        except StopIteration:
            print "g=%r complete. callbacking deferred=%r with None." % (g, deferred)
            # fell off the end, or "return" statement
            deferred.callback(None)
            print "exiting _inlineCallbacks (pos 1)"
            return deferred
        except _DefGen_Return, e:
            print "g=%r complete: returned %r. callbacking deferred=%r." % (g, e.value, deferred)
            # returnValue() was called; time to give a result to the original
            # Deferred.  First though, let's try to identify the potentially
            # confusing situation which results when returnValue() is
            # accidentally invoked from a different function, one that wasn't
            # decorated with @inlineCallbacks.

            # The traceback starts in this frame (the one for
            # _inlineCallbacks); the next one down should be the application
            # code.
            appCodeTrace = exc_info()[2].tb_next
            if isFailure:
                # If we invoked this generator frame by throwing an exception
                # into it, then throwExceptionIntoGenerator will consume an
                # additional stack frame itself, so we need to skip that too.
                appCodeTrace = appCodeTrace.tb_next
            # Now that we've identified the frame being exited by the
            # exception, let's figure out if returnValue was called from it
            # directly.  returnValue itself consumes a stack frame, so the
            # application code will have a tb_next, but it will *not* have a
            # second tb_next.
            if appCodeTrace.tb_next.tb_next:
                # If returnValue was invoked non-local to the frame which it is
                # exiting, identify the frame that ultimately invoked
                # returnValue so that we can warn the user, as this behavior is
                # confusing.
                ultimateTrace = appCodeTrace
                while ultimateTrace.tb_next.tb_next:
                    ultimateTrace = ultimateTrace.tb_next
                filename = ultimateTrace.tb_frame.f_code.co_filename
                lineno = ultimateTrace.tb_lineno
                warnings.warn_explicit(
                    "returnValue() in %r causing %r to exit: "
                    "returnValue should only be invoked by functions decorated "
                    "with inlineCallbacks" % (
                        ultimateTrace.tb_frame.f_code.co_name,
                        appCodeTrace.tb_frame.f_code.co_name),
                    DeprecationWarning, filename, lineno)
            deferred.callback(e.value)
            print "exiting _inlineCallbacks (pos 2)"
            return deferred
        except:
            t, v, tb = exc_info()
            print "g=%r threw an error: %r" % (g, v)
            print "errbacking deferred=%r." % deferred
            deferred.errback()
            print "current traceback for %r:" % deferred
            print deferred.result.getTraceback()
            print "exiting _inlineCallbacks (pos 3)"
            return deferred

        print "g=%r (line %d) yielded a value: %r" % (g, g.gi_frame.f_lineno, result)
        if isinstance(result, Deferred):
            print "value is a Deferred. checking if result is available immediately."
            if waiting[0] != True:
                print "SO WEIRD: waiting is %r. was expecting [True, None]." % (waiting,)
            # a deferred was yielded, get the result.
            def gotResult(r):
                if waiting[0]:
                    print "[gotResult] g=%r deferred gave us a value: %r. _inlineCallbacks is still waiting for us, so yay, that loop will continue" % (g, r)
                    waiting[0] = False
                    waiting[1] = r
                else:
                    print "[gotResult] g=%r deferred gave us a value: %r. _inlineCallbacks is no longer waiting, so we'll invoke a new loop." % (g, r)
                    _inlineCallbacks(r, g, deferred)

            result.addBoth(gotResult)
            if waiting[0]:
                print "g=%r no result was immediately available. exiting _inlineCallbacks (pos 4) with deferred=%r" % (g, deferred)
                # Haven't called back yet, set flag so that we get reinvoked
                # and return from the loop
                waiting[0] = False
                return deferred

            result = waiting[1]
            print "g=%r now has a result: %r. will send back in next iteration." % (g, result)
            # Reset waiting to initial values for next loop.  gotResult uses
            # waiting, but this isn't a problem because gotResult is only
            # executed once, and if it hasn't been executed yet, the return
            # branch above would have been taken.


            waiting[0] = True
            waiting[1] = None
        else:
            print "g=%r value %r was not a Deferred. hand it back." % (g, result)


    print "exiting _inlineCallbacks (pos IMPOSSIBLE)"
    return deferred


def inlineCallbacks(f):
    def unwindGenerator(*args, **kwargs):
        return _inlineCallbacks(None, f(*args, **kwargs), Deferred())
    return mergeFunctionMetadata(f, unwindGenerator)