convert-weather-data-to-json.py

import shapefile
import os
import csv
import json
from datetime import datetime

weatherDataFile = '2016-sorted.csv'

def processWeatherDoc(currentStationDoc):
    if 'TMAX' in currentStationDoc:
        currentStationDoc['TMAX'] = float(currentStationDoc['TMAX']) / 10.0
    if 'TMIN' in currentStationDoc:
        currentStationDoc['TMIN'] = float(currentStationDoc['TMIN']) / 10.0
    if 'PRCP' in currentStationDoc:
        currentStationDoc['PRCP'] = float(currentStationDoc['PRCP']) / 10.0
    if 'AWND' in currentStationDoc:
        currentStationDoc['AWND'] = float(currentStationDoc['AWND']) / 10.0
    if 'EVAP' in currentStationDoc:
        currentStationDoc['EVAP'] = float(currentStationDoc['EVAP']) / 10.0
    if 'MDEV' in currentStationDoc:
        currentStationDoc['MDEV'] = float(currentStationDoc['MDEV']) / 10.0
    if 'MDPR' in currentStationDoc:
        currentStationDoc['MDPR'] = float(currentStationDoc['MDPR']) / 10.0
    if 'MDTN' in currentStationDoc:
        currentStationDoc['MDTN'] = float(currentStationDoc['MDTN']) / 10.0
    if 'MDTX' in currentStationDoc:
        currentStationDoc['MDTX'] = float(currentStationDoc['MDTX']) / 10.0
    if 'MNPN' in currentStationDoc:
        currentStationDoc['MNPN'] = float(currentStationDoc['MNPN']) / 10.0
    if 'MXPN' in currentStationDoc:
        currentStationDoc['MXPN'] = float(currentStationDoc['MXPN']) / 10.0
    if 'TAVG' in currentStationDoc:
        currentStationDoc['TAVG'] = float(currentStationDoc['TAVG']) / 10.0
    if 'THIC' in currentStationDoc:
        currentStationDoc['THIC'] = float(currentStationDoc['THIC']) / 10.0
    if 'TOBS' in currentStationDoc:
        currentStationDoc['TOBS'] = float(currentStationDoc['TOBS']) / 10.0
    if 'WESD' in currentStationDoc:
        currentStationDoc['WESD'] = float(currentStationDoc['WESD']) / 10.0
    if 'WESF' in currentStationDoc:
        currentStationDoc['WESF'] = float(currentStationDoc['WESF']) / 10.0
    if 'WSF1' in currentStationDoc:
        currentStationDoc['WSF1'] = float(currentStationDoc['WSF1']) / 10.0
    if 'WSF2' in currentStationDoc:
        currentStationDoc['WSF2'] = float(currentStationDoc['WSF2']) / 10.0
    if 'WSF5' in currentStationDoc:
        currentStationDoc['WSF5'] = float(currentStationDoc['WSF5']) / 10.0
    if 'WSFG' in currentStationDoc:
        currentStationDoc['WSFG'] = float(currentStationDoc['WSFG']) / 10.0
    if 'WSFI' in currentStationDoc:
        currentStationDoc['WSFI'] = float(currentStationDoc['WSFI']) / 10.0
    if 'WSFM' in currentStationDoc:
        currentStationDoc['WSFM'] = float(currentStationDoc['WSFM']) / 10.0
        
    if 'TMIN' in currentStationDoc and 'TMAX' in currentStationDoc:
        if currentStationDoc['TMIN'] > currentStationDoc['TMAX']:
            tmp = currentStationDoc['TMIN']
            currentStationDoc['TMIN'] = currentStationDoc['TMAX']
            currentStationDoc['TMAX'] = tmp
        currentStationDoc['TRANGE'] = { 
            "gte" : currentStationDoc['TMIN'],
            "lte" : currentStationDoc['TMAX']
          }
    if 'MDTN' in currentStationDoc and 'MDTX' in currentStationDoc:
        if currentStationDoc['MDTN'] > currentStationDoc['MDTX']:
            tmp = currentStationDoc['MDTN']
            currentStationDoc['MDTN'] = currentStationDoc['MDTX']
            currentStationDoc['MDTX'] = tmp
        currentStationDoc['MDTRANGE'] = { 
            "gte" : currentStationDoc['MDTN'],
            "lte" : currentStationDoc['MDTX']
          }
    
    return currentStationDoc

def processWeatherFile(weatherDataFile):
    with open(weatherDataFile, 'r') as file:
        csvreader = csv.reader(file, delimiter=',', quotechar='"')
        currentStationDoc = None
        stationDocsProcessed = 0
        for row in csvreader:
            station = row[0]
            date = datetime.strptime(row[1], '%Y%m%d')
            elementType = row[2]
            elementValue = row[3]
            if currentStationDoc == None:
                currentStationDoc = {
                    'station': station,
                    'date': date.isoformat(),
                    elementType: elementValue
                }
            elif currentStationDoc['station'] != station or currentStationDoc['date'] != date:
                doc = processWeatherDoc(currentStationDoc)
                print(json.dumps(doc, ensure_ascii=False))
                stationDocsProcessed = stationDocsProcessed + 1
                currentStationDoc = {
                    'station': station,
                    'date': date.isoformat(),
                    elementType: elementValue
                }
            else:
                currentStationDoc[elementType] = elementValue


if __name__ == "__main__":
    processWeatherFile(weatherDataFile)