Skip to content

Instantly share code, notes, and snippets.

@martijnvg
Created July 4, 2017 08:35
Show Gist options
  • Save martijnvg/72a3711cb26fd84f196e9a1c4a41d038 to your computer and use it in GitHub Desktop.
Save martijnvg/72a3711cb26fd84f196e9a1c4a41d038 to your computer and use it in GitHub Desktop.
import shapefile
import os
import csv
import json
from datetime import datetime
weatherDataFile = '2016-sorted.csv'
def processWeatherDoc(currentStationDoc):
if 'TMAX' in currentStationDoc:
currentStationDoc['TMAX'] = float(currentStationDoc['TMAX']) / 10.0
if 'TMIN' in currentStationDoc:
currentStationDoc['TMIN'] = float(currentStationDoc['TMIN']) / 10.0
if 'PRCP' in currentStationDoc:
currentStationDoc['PRCP'] = float(currentStationDoc['PRCP']) / 10.0
if 'AWND' in currentStationDoc:
currentStationDoc['AWND'] = float(currentStationDoc['AWND']) / 10.0
if 'EVAP' in currentStationDoc:
currentStationDoc['EVAP'] = float(currentStationDoc['EVAP']) / 10.0
if 'MDEV' in currentStationDoc:
currentStationDoc['MDEV'] = float(currentStationDoc['MDEV']) / 10.0
if 'MDPR' in currentStationDoc:
currentStationDoc['MDPR'] = float(currentStationDoc['MDPR']) / 10.0
if 'MDTN' in currentStationDoc:
currentStationDoc['MDTN'] = float(currentStationDoc['MDTN']) / 10.0
if 'MDTX' in currentStationDoc:
currentStationDoc['MDTX'] = float(currentStationDoc['MDTX']) / 10.0
if 'MNPN' in currentStationDoc:
currentStationDoc['MNPN'] = float(currentStationDoc['MNPN']) / 10.0
if 'MXPN' in currentStationDoc:
currentStationDoc['MXPN'] = float(currentStationDoc['MXPN']) / 10.0
if 'TAVG' in currentStationDoc:
currentStationDoc['TAVG'] = float(currentStationDoc['TAVG']) / 10.0
if 'THIC' in currentStationDoc:
currentStationDoc['THIC'] = float(currentStationDoc['THIC']) / 10.0
if 'TOBS' in currentStationDoc:
currentStationDoc['TOBS'] = float(currentStationDoc['TOBS']) / 10.0
if 'WESD' in currentStationDoc:
currentStationDoc['WESD'] = float(currentStationDoc['WESD']) / 10.0
if 'WESF' in currentStationDoc:
currentStationDoc['WESF'] = float(currentStationDoc['WESF']) / 10.0
if 'WSF1' in currentStationDoc:
currentStationDoc['WSF1'] = float(currentStationDoc['WSF1']) / 10.0
if 'WSF2' in currentStationDoc:
currentStationDoc['WSF2'] = float(currentStationDoc['WSF2']) / 10.0
if 'WSF5' in currentStationDoc:
currentStationDoc['WSF5'] = float(currentStationDoc['WSF5']) / 10.0
if 'WSFG' in currentStationDoc:
currentStationDoc['WSFG'] = float(currentStationDoc['WSFG']) / 10.0
if 'WSFI' in currentStationDoc:
currentStationDoc['WSFI'] = float(currentStationDoc['WSFI']) / 10.0
if 'WSFM' in currentStationDoc:
currentStationDoc['WSFM'] = float(currentStationDoc['WSFM']) / 10.0
if 'TMIN' in currentStationDoc and 'TMAX' in currentStationDoc:
if currentStationDoc['TMIN'] > currentStationDoc['TMAX']:
tmp = currentStationDoc['TMIN']
currentStationDoc['TMIN'] = currentStationDoc['TMAX']
currentStationDoc['TMAX'] = tmp
currentStationDoc['TRANGE'] = {
"gte" : currentStationDoc['TMIN'],
"lte" : currentStationDoc['TMAX']
}
if 'MDTN' in currentStationDoc and 'MDTX' in currentStationDoc:
if currentStationDoc['MDTN'] > currentStationDoc['MDTX']:
tmp = currentStationDoc['MDTN']
currentStationDoc['MDTN'] = currentStationDoc['MDTX']
currentStationDoc['MDTX'] = tmp
currentStationDoc['MDTRANGE'] = {
"gte" : currentStationDoc['MDTN'],
"lte" : currentStationDoc['MDTX']
}
return currentStationDoc
def processWeatherFile(weatherDataFile):
with open(weatherDataFile, 'r') as file:
csvreader = csv.reader(file, delimiter=',', quotechar='"')
currentStationDoc = None
stationDocsProcessed = 0
for row in csvreader:
station = row[0]
date = datetime.strptime(row[1], '%Y%m%d')
elementType = row[2]
elementValue = row[3]
if currentStationDoc == None:
currentStationDoc = {
'station': station,
'date': date.isoformat(),
elementType: elementValue
}
elif currentStationDoc['station'] != station or currentStationDoc['date'] != date:
doc = processWeatherDoc(currentStationDoc)
print(json.dumps(doc, ensure_ascii=False))
stationDocsProcessed = stationDocsProcessed + 1
currentStationDoc = {
'station': station,
'date': date.isoformat(),
elementType: elementValue
}
else:
currentStationDoc[elementType] = elementValue
if __name__ == "__main__":
processWeatherFile(weatherDataFile)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment