Server cursor query with Django

server_cursor_query.py

# Based on http://thebuild.com/blog/2010/12/13/very-large-result-sets-in-django-using-postgresql/
# Allows a queryset.
# Does not support defer(), annotate(), select_related() or prefetch_related()

import django
from django.db import connection, transaction

assert django.VERSION[0:2] == (1, 8), "only tested on Django 1.8, make sure to test on your version"


def server_cursor_query(queryset):
    compiler = queryset.query.get_compiler(using=queryset.db)
    sql, params = compiler.as_sql()

    model = compiler.klass_info['model']
    select_fields = compiler.klass_info['select_fields']
    fields = [field[0].target.attname
              for field in compiler.select[select_fields[0]:select_fields[-1] + 1]]

    cursor = connection.connection.cursor(name='gigantic_cursor')
    with transaction.atomic(savepoint=False):
        cursor.execute(sql, params)

        while True:
            rows = cursor.fetchmany(1000)
            if not rows:
                break
            for row in rows:
                obj = model.from_db(queryset.db, fields, row[select_fields[0]:select_fields[-1] + 1)
                yield obj

It does not check that a connection to the database already exists. If this could be your first query, you should ensure the connection exists. Calling connection.cursor() will force django to connect to the database.