download_kobo_data.py

#!/usr/bin/env python3

"""Downloads data from Kobo"""

import argparse
import urllib.request

# Number of records to download per batch
DEFAULT_BATCH_SIZE = 10000

BASE_URL = 'https://kobo.humanitarianresponse.info/api/v2/assets/CHANGE_ME/data.json'
AUTHORIZATION = 'Token CHANGE_ME_TOO_SEE_API_GUIDE'


def download_data(start_index, batch_size):
    request = urllib.request.Request("{}?limit={}&start={}".format(BASE_URL, batch_size, start_index))
    request.add_header('Authorization', AUTHORIZATION)
    print("Downloading: ", request.full_url)
    with urllib.request.urlopen(request) as f:
        data = f.read().decode('utf-8')
    
    output_file = 'kobo-{:07d}-to-{:07d}.json'.format(start_index, start_index + batch_size)
    print("Writing to: ", output_file)
    with open(output_file, 'w') as f:
        f.write(data)


def main():
    parser = argparse.ArgumentParser(description="Download registration/distribution data from Kobo for Togo.")
    parser.add_argument('--start_index', type=int, help="Index of the first record to download.", required=True)
    parser.add_argument('--end_index', type=int, help="Index one past the last record to download.", required=True)
    parser.add_argument('--batch_size', type=int, help="How many records to download per batch.", default=DEFAULT_BATCH_SIZE)
    args = parser.parse_args()

    for start_index in range(args.start_index, args.end_index, args.batch_size):
        download_data(start_index, args.batch_size)


if __name__ == '__main__':
    main()