Skip to content

Instantly share code, notes, and snippets.

@aking1012
Created September 1, 2018 14:30
Show Gist options
  • Save aking1012/f29f825563b11ed5d7971833707d19c1 to your computer and use it in GitHub Desktop.
Save aking1012/f29f825563b11ed5d7971833707d19c1 to your computer and use it in GitHub Desktop.
#The stackoverflow link is because the existing answer is no longer accurate.
import gzip
import io
import urllib3
class EDDBMultiDataFetcher():
def __init__(self):
self.files_dict = {
'Populated Systems':'https://eddb.io/archive/v5/systems_populated.jsonl',
'Stations':'https://eddb.io/archive/v5/stations.jsonl',
'Minor factions':'https://eddb.io/archive/v5/factions.jsonl',
'Commodities':'https://eddb.io/archive/v5/commodities.json'
}
self.http = urllib3.PoolManager()
def fetch_all(self):
for item, url in self.files_dict.items():
self.fetch(item, url)
def fetch(self, item, url, save_file = None):
print("Fetching: " + item)
request = self.http.request(
'GET',
url,
headers={
'Accept-encoding': 'gzip, deflate, sdch'
})
data = request.data.decode('utf-8')
print("Fetch complete")
print(data)
print(request.headers)
#quit is in here so you don't fetch them all. don't be a dick rules apply.
#Also, save the data, don't just print it.
quit()
if __name__ == '__main__':
print("Fetching files from eddb.io")
fetcher = EDDBMultiDataFetcher()
fetcher.fetch_all()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment