Skip to content

Instantly share code, notes, and snippets.

@Tokariew
Last active February 19, 2020 18:51
Show Gist options
  • Save Tokariew/394c20270166c737dac3327f49b36534 to your computer and use it in GitHub Desktop.
Save Tokariew/394c20270166c737dac3327f49b36534 to your computer and use it in GitHub Desktop.
Download weather data from wunderground site using new API
import json
import os
from concurrent.futures import ThreadPoolExecutor
from datetime import date, timedelta
from time import sleep
import requests
API_KEY = '' # fill your API key
STATION_ID = '' # pws of intertest
UNITS = 'm'
FORMAT = 'json'
URL = 'https://api.weather.com/v2/pws/history/all?stationId={}&format={}&units={}&date={}&apiKey={}'
START_DATE = date(2018, 6, 1)
END_DATE = date.today()
def flat_json(x):
def flat_dict(x):
out = dict()
for key, val in x.items():
if type(val) == dict:
for key2, val2 in val.items():
out[key2] = val2
else:
out[key] = val
return out
if type(x) == dict:
return flat_dict(x)
elif type(x) == list:
ret = []
for item in x:
ret.append(flat_dict(item))
return ret
def download(day):
url = URL.format(STATION_ID, FORMAT, UNITS, day.strftime("%Y%m%d"),
API_KEY)
file_name = f'{day.strftime("%Y%m%d")}.json'
r = requests.get(url)
if r.status_code != 200:
print(f'Skipping {file_name}, status: {r.status_code}')
return
response = r.content
js = json.loads(response)['observations']
js = flat_json(js)
with open(file_name, 'w') as file:
json.dump(js, file, indent=2)
delay = r.elapsed.total_seconds()
sleep(delay)
def daterange(start_date, end_date):
for n in range(int((end_date - start_date).days)):
yield start_date + timedelta(n)
with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
for day in executor.map(download, daterange(START_DATE, END_DATE)):
pass
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment