Skip to content

Instantly share code, notes, and snippets.

@Sjlver
Created Apr 27, 2021
Embed
What would you like to do?
#!/usr/bin/env python3
"""Downloads data from Kobo"""
import argparse
import urllib.request
# Number of records to download per batch
DEFAULT_BATCH_SIZE = 10000
BASE_URL = 'https://kobo.humanitarianresponse.info/api/v2/assets/CHANGE_ME/data.json'
AUTHORIZATION = 'Token CHANGE_ME_TOO_SEE_API_GUIDE'
def download_data(start_index, batch_size):
request = urllib.request.Request("{}?limit={}&start={}".format(BASE_URL, batch_size, start_index))
request.add_header('Authorization', AUTHORIZATION)
print("Downloading: ", request.full_url)
with urllib.request.urlopen(request) as f:
data = f.read().decode('utf-8')
output_file = 'kobo-{:07d}-to-{:07d}.json'.format(start_index, start_index + batch_size)
print("Writing to: ", output_file)
with open(output_file, 'w') as f:
f.write(data)
def main():
parser = argparse.ArgumentParser(description="Download registration/distribution data from Kobo for Togo.")
parser.add_argument('--start_index', type=int, help="Index of the first record to download.", required=True)
parser.add_argument('--end_index', type=int, help="Index one past the last record to download.", required=True)
parser.add_argument('--batch_size', type=int, help="How many records to download per batch.", default=DEFAULT_BATCH_SIZE)
args = parser.parse_args()
for start_index in range(args.start_index, args.end_index, args.batch_size):
download_data(start_index, args.batch_size)
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment