Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
sodapy.Socrata extenion to page through results using a single call to get
from sodapy import Socrata
class SocrataPager(Socrata):
def get(self, dataset, *args, limit=None, offset=0, where=None, order=None, page_size=None, **kwargs):
""" get dataset by pages or within limit
uses Socrata.get to page through results until limit is reached. This adds the page_size parameter
which is passed as the limit kwarg to Socrata.get. limit is the total number of records to
return, across all pages. Note that the method is a generator.
See https://dev.socrata.com/docs/paging.html
"""
done = False
while not done:
results = super().get(dataset, *args, limit=page_size, offset=offset, where=where, order=order, **kwargs)
for item in results[slice(0, limit)]:
yield item
offset += len(results)
done = (len(results) == 0) or (limit and offset >= limit)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.