Created
September 1, 2018 14:30
-
-
Save aking1012/f29f825563b11ed5d7971833707d19c1 to your computer and use it in GitHub Desktop.
Fetch all the things. https://stackoverflow.com/a/52129017/486399
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#The stackoverflow link is because the existing answer is no longer accurate. | |
import gzip | |
import io | |
import urllib3 | |
class EDDBMultiDataFetcher(): | |
def __init__(self): | |
self.files_dict = { | |
'Populated Systems':'https://eddb.io/archive/v5/systems_populated.jsonl', | |
'Stations':'https://eddb.io/archive/v5/stations.jsonl', | |
'Minor factions':'https://eddb.io/archive/v5/factions.jsonl', | |
'Commodities':'https://eddb.io/archive/v5/commodities.json' | |
} | |
self.http = urllib3.PoolManager() | |
def fetch_all(self): | |
for item, url in self.files_dict.items(): | |
self.fetch(item, url) | |
def fetch(self, item, url, save_file = None): | |
print("Fetching: " + item) | |
request = self.http.request( | |
'GET', | |
url, | |
headers={ | |
'Accept-encoding': 'gzip, deflate, sdch' | |
}) | |
data = request.data.decode('utf-8') | |
print("Fetch complete") | |
print(data) | |
print(request.headers) | |
#quit is in here so you don't fetch them all. don't be a dick rules apply. | |
#Also, save the data, don't just print it. | |
quit() | |
if __name__ == '__main__': | |
print("Fetching files from eddb.io") | |
fetcher = EDDBMultiDataFetcher() | |
fetcher.fetch_all() | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment