Get a set number of pages of GO Appeal data

Download All GO Appeals to CSV.py

#!/usr/bin/env python3

import csv
import requests

def flatten_dict(d, parent_key='', sep='_'):
	"""
	recursively flattens a nested dictionary to handle GO API data structure.

	function takes a dictionary that may contain nested dictionaries 
	and flattens it so that all keys from nested dictionaries are combined 
	with their parent keys, separated by a specified separator.

	Parameters:
	d (dict): dictionary to flatten.
	parent_key (str): base key to use for the current level of recursion. Default is an empty string.
	sep (str): separator to use when combining keys. Default is an underscore ('_').

	Returns:
	dict: new dictionary with all nested dictionaries flattened into a single level, with keys indicating the original structure.

	Example:
	>>> nested_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
	>>> flatten_nested_data(nested_dict)
	{'a': 1, 'b_c': 2, 'b_d_e': 3}
	"""
	
	items = []
	for k, v in d.items():
		new_key = f"{parent_key}{sep}{k}" if parent_key else k
		if isinstance(v, dict):
			items.extend(flatten_dict(v, new_key, sep=sep).items())
		else:
			items.append((new_key, v))
	return dict(items)

url = "https://goadmin.ifrc.org/api/v2/appeal/"
data = []
page_limit = 100 # change this number to limit results
pages_retrieved = 0

response = requests.get(url)
if response.status_code == 200:
	json_data = response.json()
	for item in json_data['results']:
		data.append(flatten_dict(item))
	pages_retrieved += 1
	
	while json_data['next'] and pages_retrieved < page_limit:
		next_page_url = json_data['next']
		response = requests.get(next_page_url)
		if response.status_code == 200:
			json_data = response.json()
			for item in json_data['results']:
				data.append(flatten_dict(item))
			pages_retrieved += 1
			
	csv_filename = "appeal_data.csv"
	with open(csv_filename, "w", newline="") as file:
		if data:
			writer = csv.DictWriter(file, fieldnames=data[0].keys())
			writer.writeheader()
			writer.writerows(data)
			
	print(f"Appeal data saved from {pages_retrieved} pages and saved as {csv_filename}.")
else:
	print("Couldn't access the data!")