worker.py

#!/usr/bin/env python

import os
import threading
import time


class Job(object):
    def __init__(self, name, dependants=None):
        self.name = name
        self.dependants = dependants or []

    def print_(self, depth=0):
        print "  " * depth + self.name
        for dependant in self.dependants:
            dependant.print_(depth + 1)

    def weight(self):
        return sum(dep.weight() for dep in self.dependants) + 1


class Worker(threading.Thread):
    def __init__(self, jobs, job):
        super(Worker, self).__init__()
        self.jobs = jobs
        self.job = job

    def run(self):
        print "Worker: %s" % self.job.name
        time.sleep(0.5)
        # Queue dependants
        for dep in self.job.dependants:
            if dep.dependants:  # Is a dir
                self.jobs.append(dep)
            else:
                self.jobs.insert(0, dep)


def schedule(files):
    # Transform jobs into a tree
    tree = {}
    for fn in sorted(files):
        parts = fn.split(os.sep)
        subtree = tree
        for i, part in enumerate(parts):
            subtree = subtree.setdefault(part, {})

    return schedule_tree(tree)


def schedule_tree(tree, parent=None):
    jobs = []
    for name, dir_ in tree.iteritems():
        full_name = name
        if parent:
            full_name = os.path.join(parent, name)
        jobs.append(Job(full_name, schedule_tree(dir_, full_name)))

    jobs.sort(key=lambda x: x.weight())
    return jobs


def runner(jobs):
    threads = []
    workers = 5
    while threads or jobs:
        if jobs and len(threads) < workers:
            thread = Worker(jobs, jobs.pop())
            threads.append(thread)
            thread.start()
        elif len(threads) < workers:
            print "Starved"
            time.sleep(0.1)
        else:
            time.sleep(0.1)
        threads = [thread for thread in threads if thread.is_alive()]


def main():
    tree = (
        'foo/foo/foo',
        'foo/foo/bar',
        'foo/foo/baz',
        'foo/bar/foo',
        'foo/bar/baz',
        'bar',
        'baz',
        'quux',
    )
    jobs = schedule(tree)

    print "Tree:"
    for job in jobs:
        job.print_()

    runner(jobs)

if __name__ == '__main__':
    main()