values_list_queryset_chunker

values_list_queryset_chunker.py

def values_list_queryset_chunker(queryset, chunk_size=10000):
    """
    Yields batched querysets from a values_list queryset.
    First element of each record must always be pk
    """
    queryset = queryset.order_by('pk')
    pk = queryset.order_by('pk')[0][0] - 1
    batch = queryset.filter(pk__gt=pk)[:chunk_size]
    while batch.count() > 0:
        chunk_size = min([chunk_size, batch.count()])
        pk = batch[chunk_size - 1][0]
        yield batch[:chunk_size]
        batch = queryset.filter(pk__gt=pk)