an idea for paul

insert_iter.py

import random
import unittest

from eventlet import sleep
from swift.common.utils import GreenAsyncPile, GreenthreadSafeIterator


class NodeIter(object):

    def __init__(self, nodes):
        self._node_iter = iter(nodes)
        self.extra_nodes = []

    def __iter__(self):
        return self

    def __next__(self):
        return next(self)

    def next(self):
        try:
            next_node = self.extra_nodes.pop(0)
        except IndexError:
            next_node = next(self._node_iter)
        return next_node

    def add_node(self, node):
        self.extra_nodes.append(node)


def get_good_response(node_iter):
    """
    Will "connect" to nodes until it finds one with a non-empty frags key.

    :returns: a tuple, in the form of (frag, node), where frag is an
              integer and node might have extra frags.
    """
    for node in node_iter:
        sleep(random.random())  # connect
        print 'connected to %r' % node
        try:
            frag = node.get('frags', []).pop(0)
        except IndexError:
            continue
        else:
            return frag, node
    # node iter exhausted
    return None, {}


def get_responses(nodes):
    print 'getting frags from %r' % nodes
    node_iter = NodeIter(nodes)
    safe_iter = GreenthreadSafeIterator(iter(node_iter))
    resp_pile = GreenAsyncPile(3)
    for i in range(3):
        resp_pile.spawn(get_good_response, safe_iter)
    frags = set()
    for frag, node in resp_pile:
        if frag is None:
            continue
        frags.add(frag)
        if len(frags) > 3:
            break
        if node.get('frags'):
            # add nodes with extra frags back into the *node* iter
            node_iter.add_node(node)
    return frags


class TestCase(unittest.TestCase):

    def test_all_expected(self):
        nodes = [
            {'frags': [0]},
            {'frags': [1]},
            {'frags': [2]},
        ]
        resp = get_responses(nodes)
        self.assertEqual(resp, set([0, 1, 2]))

    def test_node_with_no_frags(self):
        nodes = [
            {},
            {'frags': [0]},
            {'frags': [1]},
            {'frags': [2]},
        ]
        resp = get_responses(nodes)
        self.assertEqual(resp, set([0, 1, 2]))

    def test_node_with_multiple_frags(self):
        nodes = [
            {'frags': [0, 1]},
            {'frags': [2]},
            {},
        ]
        resp = get_responses(nodes)
        self.assertEqual(resp, set([0, 1, 2]))

    def test_node_with_multiple_frags(self):
        nodes = [
            {'frags': [0, 1]},
            {'frags': [2]},
            {},
        ]
        resp = get_responses(nodes)
        self.assertEqual(resp, set([0, 1, 2]))

if __name__ == "__main__":
    unittest.main()