Use a directory as a task queue. Process files in they were added and keep track of progress.

gistfile1.txt

import os, glob
import cPickle as pickle

class QueueDir:
    def __init__(self,dir,pattern='*',statefilename='.queuedir.pc'):
        self.statefilename = statefilename
        self.pattern = pattern
        self.dir = dir
        self.file_in_progress_mtime = None
        self.infinite = False
        # let's see if we have saved how old files are already done
        try:
            with open(self.statefilename) as statefile:
                self.last_file_done_mtime = pickle.load(statefile)
        except IOError:
            self.last_file_done_mtime = -1
        except EOFError:
            self.last_file_done_mtime = -1
            
    def done(self,file_done_mtime = None):
        if file_done_mtime == None:
            file_done_mtime = self.file_in_progress_mtime
        # we have finished what was in progress. save that.
        if file_done_mtime:
            self.last_file_done_mtime = file_done_mtime
            with open(self.statefilename,'wb') as statefile:
                pickle.dump(self.last_file_done_mtime,statefile)

    def next(self):
        self.done() # the next one is needed, so the previous one is cnsidered done
        # find earliest file that is older than the one we finished
        earliest_file_not_done_name = None
        earliest_file_not_done_mtime = 1e400
        for filename in glob.glob(os.path.join(self.dir,self.pattern)):
            mtime = os.stat(filename)[8]
            if self.last_file_done_mtime < mtime < earliest_file_not_done_mtime:
                earliest_file_not_done_name = filename
                earliest_file_not_done_mtime = mtime
        # if we found a file, this is now in progress
        self.file_in_progress_mtime = earliest_file_not_done_mtime if earliest_file_not_done_mtime != 1e400 else None
        return earliest_file_not_done_name # if no file was found, return None

    def __iter__(self):
        while True:
            n = self.next()
            if n:
                yield n
            else:
                if not self.infinite:
                    break