a simple job-runner

dependency.py

def check_dependency(target, dependency):
    if dependency is None:
        return True

    if isinstance(target, list):
        for t in target:
            if check_dependency(t, dependency) is True:
                return True

    if not os.path.exists(target):
        return True

    def needs_rebuild(targ_t, dep_f):
        if targ_t < os.stat(dep_f).st_mtime:
            return True
        else:
            return False

    target_time = os.stat(target).st_mtime
    if isinstance(dependency, list):
        ret = False
        for dep in dependency:
            if needs_rebuild(target_time, dep):
                ret = True
                break
        return ret
    else:
        return needs_rebuild(target_time, dependency)

runner.py

from dependency import check_dependency

def runner(jobs, pool=None, parallel=True, retval='count', force=False):
    if pool == 1 and parallel is True:
        parallel = False

    if parallel is True:
        if pool is not None:
            p = Pool(pool)
        else:
            p = Pool()

    count = 0
    results = []
    for job in jobs:
        if force is True or check_dependency(job['target'], job['dependency']):
            if parallel is True:
                if isinstance(job['args'], dict):
                    results.append(p.apply_async(job['job'], kwds=job['args']))
                else:
                    results.append(p.apply_async(job['job'], args=job['args']))
            else:
                if isinstance(job['args'], dict):
                    results.append(job['job'](**job['args']))
                else:
                    results.append(job['job'](*job['args']))

            count +=1

    if parallel is True:
        p.close()
        p.join()

    # return values differ based on retval argument
    if retval == 'count':
        return count
    elif retval == 'results':
        return ( o.get() for o in results )
    elif retval is None:
        return None
    else:
        return dict(count=count,
                    results=( o.get() for o in results ) )