Skip to content

Instantly share code, notes, and snippets.

@cynici

cynici/http-get-wunderground.py

Created December 5, 2012 05:44
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save cynici/4212722 to your computer and use it in GitHub Desktop.
Save cynici/4212722 to your computer and use it in GitHub Desktop.
Download ZIP
Get weather data from Wunderground stations listed in input JSON and output to a CSV file
Raw
http-get-wunderground.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os, sys
from optparse import OptionParser
import string
import urllib2
import simplejson
import logging
from datetime import datetime
import time
help_text = """%prog [options] JSON_file ...
Get weather data from Wunderground stations listed in input JSON file(s)
and output in a CSV file. Details: http://www.wunderground.com/weather/api/
JSON format:
[{
"c": "AO",
"l": "/q/zmw:00000.1.66116",
"name": "AO Noqui Angola",
"type": "city",
"tz": "Africa/Luanda",
"tzs": "WAT",
"zmw": "00000.1.66116"
}]
CSV format:
Station_id,Longitude,Latitude,Name,yyyy-mm-dd HH:MM:SS,temp_c,rh_pct,w_deg,w_ms
W_66160,13.224200,-8.797928,Angola Luanda,2012-07-31 06:06:56,19.8,-999,0,1.03"""
geolookup_url = """http://api.wunderground.com/api/b80d370b52a348da/geolookup/conditions%(l)s.json"""
def format_line(w):
'''Format query response data from dictionary to a output line'''
d = {}
l = w["location"]
o = w["current_observation"]
if "station_id" in o and o["station_id"]:
wmo = o["station_id"]
if "wmo" and l["wmo"]:
wmo = l["wmo"]
if "country" in l and l["country"]:
cc = l['country']
if "country_iso3166" in l and l["country_iso3166"]:
cc = l["country_iso3166"]
if "city" in l and l["city"]:
city = l["city"]
if "lat" in l and l["lat"]:
d["lat"] = l["lat"]
if "lon" in l and l["lon"]:
d["lon"] = l["lon"]
for pkey in ["display_location", "observation_location"]:
if pkey not in o:
continue
pdict = o[pkey]
if "latitude" in pdict and pdict["latitude"]:
d["lat"] = pdict["latitude"]
if "longitude" in pdict and pdict["longitude"]:
d["lon"] = pdict["longitude"]
if "country" in pdict and pdict["country"]:
cc = pdict["country"]
if "country_iso3166" in pdict and pdict["country_iso3166"]:
cc = pdict["country_iso3166"]
if "city" in pdict and pdict["city"]:
city = pdict["city"]
if "lat" not in d or "lon" not in d or not cc or not city:
raise ValueError("Missing key data: lat, lon, country or city.")
d["id"] = "W_"+wmo
d["name"] = ("%s %s"%(cc, city)).replace(',','')
if "observation_epoch" in o and o["observation_epoch"]:
key = "observation_epoch"
else:
if "local_epoch" in o and o["local_epoch"]:
key = "local_epoch"
else:
raise ValueError("No timestamp")
dt = datetime.utcfromtimestamp(int(o[key]))
d["datestr"] = dt.strftime("%Y-%m-%d %H:00:00")
if "temp_c" in o:
d["temp_c"] = str(o["temp_c"])
else:
d["temp_c"] = "-999"
if "relative_humidity" in o and o["relative_humidity"].upper() != "N/A":
d["rh_pct"] = o["relative_humidity"].replace('%','')
else:
d["rh_pct"] = "-999"
if "wind_degrees" in o:
d["w_deg"] = str(o["wind_degrees"])
else:
d["w_deg"] = "-999"
if "wind_kph" in o:
d["w_ms"] = str(o["wind_kph"]/3.6)
else:
d["w_ms"] = "-999"
return "%(id)s,%(lon)s,%(lat)s,%(name)s,%(datestr)s,%(temp_c)s,%(rh_pct)s,%(w_deg)s,%(w_ms)s" % d
def main(argv=None):
if argv is None:
argv = sys.argv
debuglevelD = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL,
}
scriptdir = os.path.dirname(__file__)
scriptname = os.path.basename(__file__)
scriptnoext, scriptext = os.path.splitext(__file__)
defvals = {
'output': datetime.utcnow().strftime('$HOME/afisdata/non-ftp/w_wunderground/wg-%%Y%%m%%d-%%H%%M-%s.txt'%os.getpid()),
'interval': 20, # seconds, free API limit 10 calls per minute
}
parser = OptionParser(help_text)
parser.add_option("-o", "--output", dest="output", type="string", help="Output specification (%s)"%defvals['output'])
parser.add_option("--interval", dest="interval", type="int", help="Interval between API calls (%s seconds)"%defvals['interval'], metavar='SECS')
parser.add_option("-l", "--loglevel", dest="loglevel", type="string", help="Verbosity %s"%debuglevelD.keys(), metavar='LOGLEVEL')
parser.set_defaults(**defvals)
(options, args) = parser.parse_args()
if options.loglevel:
if options.loglevel not in debuglevelD: raise AssertionError("Verbosity level must be one of: %s"%debuglevelD.keys())
dbglvl = debuglevelD[options.loglevel]
else:
dbglvl = logging.WARNING
logger = logging.getLogger()
logger.setLevel(dbglvl)
ch = logging.StreamHandler()
ch.setFormatter( logging.Formatter('%(asctime)s %(lineno)d %(name)s %(funcName)s - %(levelname)s - %(message)s') )
ch.setLevel(dbglvl)
logger.addHandler(ch)
jsonfiles = []
for f in args:
if os.path.exists(f):
jsonfiles.append(f)
else:
if os.path.exists(os.path.join(scriptdir, f)):
jsonfiles.append(os.path.join(scriptdir, f))
else:
parser.error("Input JSON file not found: %s" % f)
if not jsonfiles:
parser.error("Requires at least one input JSON file.")
ofh = None
linecount = 0
for f in jsonfiles:
station_list = simplejson.loads(open(f).read().decode("ISO-8859-1"))
logger.debug("Found %s stations in %s" % (len(station_list), f))
for d in station_list:
try:
if '/q/' not in d['l']:
# Some locations are like "/global/ZB.html" which are invalid
logger.debug("Skipped %s %s" % (f, d['l']))
continue
if linecount and options.interval > 0:
logger.debug("Going to sleep ...")
time.sleep(options.interval)
url = geolookup_url % dict(l=d['l'])
logger.debug("GET %s ..."%(url))
weather = simplejson.loads(urllib2.urlopen(url).read().decode("ISO-8859-1"))
line = format_line(weather)
if "-999" in line or ",," in line:
logger.warning("Skipped %s %s : %s" % (f, url, line))
continue
except Exception, err:
logger.warning("Failed %s %s : %s"%(f, url, err))
continue
if ofh is None:
ofh = open(os.path.expandvars(options.output),"w+")
ofh.write(line+'\n')
ofh.flush()
linecount += 1
if ofh: ofh.close()
logger.info("Wrote %s lines to %s" % (linecount, options.output))
return 0
if __name__ == "__main__":
sys.exit(main())
"""
http://autocomplete.wunderground.com/aq?query=A&c=CL
{ "RESULTS": [ { "name": "Antofagasta, Chile", "type": "city", "c": "CL", "zmw": "00000.1.85442", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.1.85442" }, { "name": "Arica, Chile", "type": "city", "c": "CL", "zmw": "00000.1.85406", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.1.85406" }, { "name": "Angol, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85703", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85703" }, { "name": "Ancud, Chile", "type": "city", "c": "CL", "zmw": "00000.10.85799", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.10.85799" }, { "name": "Arauco, Chile", "type": "city", "c": "CL", "zmw": "00000.6.85682", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.6.85682" }, { "name": "Almirante Schroeders, Chile", "type": "city", "c": "CL", "zmw": "00000.7.85934", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.7.85934" }, { "name": "Alto Palena, Chile", "type": "city", "c": "CL", "zmw": "00000.1.85836", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.1.85836" }, { "name": "ANF, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85442", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85442" }, { "name": "ARI, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85406", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85406" }, { "name": "ARR, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85864", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85864" }, { "name": "Arturo Merino Benitez International, Chile", "type": "city", "c": "CL", "zmw": "00000.7.85574", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.7.85574" } ] }
http://api.wunderground.com/api/b80d370b52a348da/geolookup/conditions/q/zmw:00000.1.68335.json
{
"response": {
"version": "0.1"
,"termsofService": "http://www.wunderground.com/weather/api/d/terms.html"
,"features": {
"geolookup": 1
,
"conditions": 1
}
}
, "location": {
"type":"INTLCITY",
"country":"ZA",
"country_iso3166":"ZA",
"country_name":"South Africa",
"state":"",
"city":"Taung",
"tz_short":"SAST",
"tz_long":"Africa/Johannesburg",
"lat":"-27.54999924",
"lon":"24.77000046",
"zip":"00000",
"magic":"1",
"wmo":"68335",
"l":"/q/zmw:00000.1.68335",
"requesturl":"global/stations/68335.html",
"wuiurl":"http://www.wunderground.com/global/stations/68335.html",
"nearby_weather_stations": {
"airport": {
"station": [
{ "city":"Taung", "state":"", "country":"ZA", "icao":"", "lat":"-27.54999924", "lon":"24.77000046" }
]
}
,
"pws": {
"station": [
]
}
}
}
, "current_observation": {
"image": {
"url":"http://icons-ak.wxug.com/graphics/wu2/logo_130x80.png",
"title":"Weather Underground",
"link":"http://www.wunderground.com"
},
"display_location": {
"full":"Taung, South Africa",
"city":"Taung",
"state":"",
"state_name":"South Africa",
"country":"ZA",
"country_iso3166":"ZA",
"zip":"00000",
"latitude":"-27.54999924",
"longitude":"24.77000046",
"elevation":"1100.00000000"
},
"observation_location": {
"full":"Taung, ",
"city":"Taung",
"state":"",
"country":"ZA",
"country_iso3166":"ZA",
"latitude":"-27.54999924",
"longitude":"24.77000046",
"elevation":" ft"
},
"estimated": {
},
"station_id":"68335",
"observation_time":"Last Updated on December 4, 8:00 AM SAST",
"observation_time_rfc822":"Tue, 04 Dec 2012 08:00:00 +0200",
"observation_epoch":"1354600800",
"local_time_rfc822":"Tue, 04 Dec 2012 13:13:34 +0200",
"local_epoch":"1354619614",
"local_tz_short":"SAST",
"local_tz_long":"Africa/Johannesburg",
"local_tz_offset":"+0200",
"weather":"",
"temperature_string":"69 F (21 C)",
"temp_f":69,
"temp_c":21,
"relative_humidity":"51%",
"wind_string":"From the West at 2 MPH",
"wind_dir":"West",
"wind_degrees":260,
"wind_mph":2,
"wind_gust_mph":0,
"wind_kph":4,
"wind_gust_kph":0,
"pressure_mb":"",
"pressure_in":"",
"pressure_trend":"+",
"dewpoint_string":"55 F (13 C)",
"dewpoint_f":55,
"dewpoint_c":13,
"heat_index_string":"NA",
"heat_index_f":"NA",
"heat_index_c":"NA",
"windchill_string":"NA",
"windchill_f":"NA",
"windchill_c":"NA",
"feelslike_string":"69 F (21 C)",
"feelslike_f":"69",
"feelslike_c":"21",
"visibility_mi":"",
"visibility_km":"",
"solarradiation":"",
"UV":"-1",
"precip_1hr_string":" in ( mm)",
"precip_1hr_in":"",
"precip_1hr_metric":"",
"precip_today_string":" in ( mm)",
"precip_today_in":"",
"precip_today_metric":"",
"icon":"clear",
"icon_url":"http://icons-ak.wxug.com/i/c/k/clear.gif",
"forecast_url":"http://www.wunderground.com/global/stations/68335.html",
"history_url":"http://www.wunderground.com/history/airport/68335/2012/12/4/DailyHistory.html",
"ob_url":"http://www.wunderground.com/cgi-bin/findweather/getForecast?query=-27.54999924,24.77000046"
}
}
"""
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment