Skip to content

Instantly share code, notes, and snippets.

@balazs-endresz
Created March 21, 2018 20:44
Show Gist options
  • Save balazs-endresz/ca9132af9f9c4142e3009dbbdba43ac0 to your computer and use it in GitHub Desktop.
Save balazs-endresz/ca9132af9f9c4142e3009dbbdba43ac0 to your computer and use it in GitHub Desktop.
Constant memory streaming CSV writer (works with prefetch_related too)
def generator_from_queryset(queryset, chunksize=2000, enumerate=False):
assert queryset.query.order_by is not None
offset = 0
i = 0
finished = False
while not finished:
finished = True
for row in queryset[offset:offset + chunksize]:
finished = False
if enumerate:
yield i, row
i = i + 1
else:
yield row
offset = offset + chunksize
class Echo:
"""An object that implements just the write method of the file-like interface."""
def write(self, value):
"""Write the value by returning it, instead of storing in a buffer."""
return value
def write_csv_response(self, queryset):
pseudo_buffer = Echo()
writer = csv.writer(pseudo_buffer)
data = generator_from_queryset(queryset, chunksize=500)
generator = itertools.chain(
(writer.writerow(self.get_headers()) for _ in range(1)),
(writer.writerow(self.to_csv_row(item)) for item in data)
)
response = StreamingHttpResponse(generator, content_type="text/csv")
response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(self.filename)
return response
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment