offset batch

batch.py

def batched_result_generator(cur):
    offset = 0
    batchsize = 100
    exhausted = False
    while not exhausted:
        results = cur.execute('select ... from offset = ? limit = ?', (offset, batchsize)).fetchall()
        if len(results) < batchsize:
            exhausted = True
        for row in results:
            yield row