django_query_example.py

django_query_example.py

################################################################

def get_products_list(last_pk=0, batch=1000):
    product = Product.objects\
        .filter(
            id__gte=last_pk,
            ...
        )[:batch]

    return product


def some_generator():
    step = 0
    batch = 1000

    while True:
        products = get_products_list(step * batch, batch)
    
    
        if len(products):
            do_some(products) # write to file
            step += 1
        else:
            break


################################################################

def queryset_iterator(queryset, chunksize=1000):
    """
    Iterate over a Django Queryset ordered by the primary key

    This method loads a maximum of chunksize (default: 1000) rows in it's
    memory at the same time while django normally would load all rows in it's
    memory. Using the iterator() method only causes it to not preload all the
    classes.

    Note that the implementation of the iterator does not support ordered query
    sets.
    """
    pk = 0
    last_pk = queryset.order_by('-pk')[0].pk
    queryset = queryset.order_by('pk')

    while pk < last_pk:
        for row in queryset.filter(pk__gt=pk)[:chunksize]:
            pk = row.pk
            yield row

def get_products_list(last_pk=0):
    product = Product.objects\
        .filter(
            id__gte=last_pk,
            ...
        )

    return product


def some_generator():
    batch = 1000
    products_list = []
    
    for product in queryset_iterator(get_products_list()):
        products_list.append(product)
        if len(products) % batch:
            do_some(products) # write to file
            products_list = []