Skip to content

Instantly share code, notes, and snippets.

@gchristian
Created June 10, 2023 03:54
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save gchristian/7462f9758bd9331958d610a7bb9a5116 to your computer and use it in GitHub Desktop.
Save gchristian/7462f9758bd9331958d610a7bb9a5116 to your computer and use it in GitHub Desktop.
def make_request(self, id="", type="deck", username="TMind"):
endpoint = f"{self.base_url}/{type}/{id}"
self.params.update({"username" : username})
print(f"Requesting Data from Website: {','.join([username,type,id])}")
response = requests.get(endpoint, params=self.params)
data = json.loads(response.content)
#insert pagination code
if "LastEvaluatedKey" in data:
paginatationRequired = True
else:
paginatationRequired = False
while paginatationRequired == True:
self.params["exclusiveStartKey"] = json.dumps(data['LastEvaluatedKey'])
page_response = requests.get(endpoint, params=self.params)
page_data = json.loads(page_response.content)
if 'error' in page_data:
print(f"Error in response: {page_data['error']}")
return []
data["Items"].extend(page_data["Items"])
if "LastEvaluatedKey" in page_data:
data["LastEvaluatedKey"] = page_data["LastEvaluatedKey"]
else:
paginatationRequired = False
#end pagination code
if 'error' in data:
print(f"Error in response: {data['error']}")
return []
with open('data/online_request.json', 'w') as f:
json.dump(data, f)
print("Loading Data...")
decks_data = self.ucl.load_data('data/online_request.json')
return decks_data
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment