Depth first non blocking traversal - use task.deferLater to allow other callbacks to execute

Depth first non blocking traversal

Errors for running the recursive version

['37']
['36']
^CUnhandled error in Deferred:


Traceback (most recent call last):
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 1184, in gotResult
    _inlineCallbacks(r, g, deferred)
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 1131, in _inlineCallbacks
    deferred.callback(getattr(e, "value", None))
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 393, in callback
    self._startRunCallbacks(result)
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 501, in _startRunCallbacks
    self._runCallbacks()
--- <exception caught here> ---
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 588, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 1184, in gotResult
    _inlineCallbacks(r, g, deferred)
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 1131, in _inlineCallbacks
    deferred.callback(getattr(e, "value", None))
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 393, in callback
    self._startRunCallbacks(result)
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 501, in _startRunCallbacks
    self._runCallbacks()
  File "/home/adi/chevah/twisted/twisted/internet/defer.py", line 601, in _runCallbacks
    current.result = failure.Failure(captureVars=self.debug)
exceptions.RuntimeError: maximum recursion depth exceeded while calling a Python object

inlineCallbacks_non_recursive.py

import os
import random
import shutil
import time

from twisted.internet import defer, reactor
from twisted.internet.task import Cooperator, deferLater, LoopingCall

# Input variables.
DEEP = 30
WIDE = 14
LOOP_INTERVAL = 2
# You should not use verbose for big structures.
# Verbose is just to check that traversal order is OK.
VERBOSE = True


def start_loop():
    pwd = os.getcwd()
    # Clean existing structure and recreate root.
    print "Cleaning folder structure"
    shutil.rmtree('deep', ignore_errors=True)
    os.mkdir('deep')
    os.chdir('deep')

    print "Creating folder structure"
    # Deep structure.
    for i in xrange(DEEP):

        # Wide structure.
        for j in xrange(WIDE):
            if j == i:
                continue
            child = str(j)
            if random.randint(0, 1):
                os.mkdir(child)
            else:
                with open(child, 'w'):
                    pass

        child = str(i)
        os.mkdir(child)
        os.chdir(child)

    # Monitor the dir initial dir.
    print "Start looping"
    os.chdir(pwd)
    loop = LoopingCall(non_recursive_list, 'deep', verbose=VERBOSE)
    deferred = loop.start(interval=LOOP_INTERVAL, now=True)

    tick = LoopingCall(tick_loop)
    tick.start(interval=0.5, now=True)

    def eb_loop_failed(failure):
        print "Loop has failed"
        print failure.getBriefTraceback()

    deferred.addErrback(eb_loop_failed)


def tick_loop():
    print time.time()


def list_path(path, verbose):
    if verbose:
        print "Listing %s" % (path,)

    deferred = deferLater(reactor, 0, os.listdir, path)

    def cb_expand_path(result):
        return [os.path.join(path, child) for child in result]

    deferred.addCallback(cb_expand_path)
    return deferred


@defer.inlineCallbacks
def non_recursive_list(path, verbose=True):
    print "Start listing"
    from collections import deque

    if verbose:
        print "Listing %s" % (path,)

    listing = yield list_path(path, verbose)

    members_queue = deque(listing)
    result = deque(listing)

    while members_queue:
        member = members_queue.popleft()

        if os.path.isdir(member):
            listing = yield list_path(member, verbose)
            members_queue.extendleft(reversed(listing))
            result.extendleft(reversed(listing))

    if verbose:
        for member in result:
            print member
    print "End listing"


reactor.callWhenRunning(start_loop)
reactor.run()

inlineCallbacks_recursive.py

import os
import shutil

from twisted.internet import defer, reactor
from twisted.internet.task import Cooperator, deferLater, LoopingCall

# Input variables.
DEEP = 200
WIDE = 300
LOOP_INTERVAL = 2


def start_loop():
    pwd = os.getcwd()
    # Clean existing structure and recreate root.
    shutil.rmtree('deep', ignore_errors=True)
    os.mkdir('deep')
    os.chdir('deep')

    # Deep structure.
    for i in xrange(DEEP):
        child = str(i)
        os.mkdir(child)
        os.chdir(child)

    # Wide structure.
    for i in xrange(WIDE):
        child = str(i)
        os.mkdir(child)

    # Monitor the dir initial dir.
    os.chdir(pwd)
    loop = LoopingCall(recursive_list, 'deep')
    deferred = loop.start(interval=LOOP_INTERVAL, now=True)

    def eb_loop_failed(failure):
        print "Loop has failed"
        failure.printBriefTraceback()

    deferred.addErrback(eb_loop_failed)


@defer.inlineCallbacks
def recursive_list(path=None):
    if path is None:
        path = 'deep'
    print "Listing %s" % (path,)
    members = os.listdir(path)

    for member in members:
        member_path = os.path.join(path, member)
        if os.path.isdir(member_path):
            yield deferLater(reactor, 0, recursive_list, member_path)

    print members


reactor.callWhenRunning(start_loop)
reactor.run()