chunk and logs

chunk_util.py

import time
import alog


def chunks(l, n, logging_step=10):
    count = 0
    total = len(l)
    start_time = time.perf_counter()
    for i in range(0, len(l), n):
        chunk = l[i:i + n]
        yield chunk
        count += len(chunk)
        ratio = count * 100 // total
        if (count * logging_step // total) - ((count - len(chunk)) * logging_step // total) >= 1:
            elapsed = time.perf_counter() - start_time
            alog.info(f'processing ... {count}/{total} ({ratio}%) (took: {elapsed:0.6f} usec)')