Skip to content

Instantly share code, notes, and snippets.

@adamhunter
Created July 11, 2018 02:16
Show Gist options
  • Save adamhunter/4089fbfeb471c555eadaed3f7bf31f49 to your computer and use it in GitHub Desktop.
Save adamhunter/4089fbfeb471c555eadaed3f7bf31f49 to your computer and use it in GitHub Desktop.
batch download darksky timemachine api
import os
import time
from datetime import datetime, timedelta
from dotenv import load_dotenv, find_dotenv
from urllib.request import urlopen
from concurrent.futures import ThreadPoolExecutor, wait
def main():
print("running!")
load_dotenv(find_dotenv())
date = datetime(2015, 1, 1)
days = range(0, 500)
dates = (date + timedelta(days=i) for i in days)
with ThreadPoolExecutor(5) as executor:
futures = [executor.submit(maybe_download, d) for d in dates]
wait(futures)
def maybe_download(date: datetime) -> int:
stamp = str(date).replace(' ', '_')
cache = 'data/%s.json' % stamp
if not os.path.exists(cache):
print('Downloading file...')
return download(date, cache)
else:
size = os.stat(cache).st_size
print('found {file} with size {size}'.format(file=cache, size=size))
return size
def download(date: datetime, cache: str) -> int:
api = os.environ['DARKSKY_API_KEY']
location = '37.8267,-122.4233'
epoch = int(time.mktime(date.timetuple()))
host = 'https://api.darksky.net'
path = '/forecast/{key}/{loc},{time}?exclude=currently,flags'
url = host + path.format(key=api, loc=location, time=epoch)
print('downloading %s' % url)
try:
with urlopen(url) as u, open(cache, 'wb') as f:
return f.write(u.read())
except:
print('Exception downloading or saving file %s %s' % url, cache)
return 0
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment