Paul's BatchSaver

batchsaver.py

class BatchSaver(object):
    def __init__(self, max_queue=2000, save_callback=None):
        self.queue = []
        self.max_queue = max_queue
        self.callback = save_callback
        import atexit
        atexit.register(self.save_now)

    def push(self, row):
        self.queue.append(row)
        if len(self.queue) > self.max_queue:
            self.save_now()

    def save_now(self):
        print("Saving")
        self.callback(self.queue)
        self.queue = []