noaa.py

# Name:         storm data conversion, extraction, and selection
# Description:  Final Project
# Created by:   socio
# Date:         7/10/2016

print '---RESET---'

# Import system modules
import arcpy, csv, datetime, gzip, os, shutil, urllib, urllib2
from bs4 import BeautifulSoup

# Functions
def message(message):
    'This prints a passed string as both print and arcpy.AddMessage().'
    print message
    arcpy.AddMessage(message)
    return

def fgdb(workSpace):
    'Verifies the existence of a FGDB or creates a new one.'
    stormFGDB = 'NOAA_StormDatabase.gdb'
    pathFGDB = os.path.join(workSpace, stormFGDB)
    if os.path.exists(pathFGDB):
        message('The FGDB already exists and will be used to store new storm data.')
    else:
        message('The FGDB does not exist. A new one will be created.')
        arcpy.CreateFileGDB_management(workSpace, stormFGDB)
        return pathFGDB

def download(htmlName, year):
    'Downloads NOAA CSV data.'
    urlNOAA = 'https://www1.ncdc.noaa.gov/pub/data/swdi/stormevents/csvfiles/'
    site = urllib2.urlopen(urlNOAA)
    html = site.read()
    soup = BeautifulSoup(html, 'html.parser')

    for link in soup.find_all('a'):
        links = link.get('href')
        if 'd' + year in links:
            if htmlName in links:
                downloadURL = urlNOAA + links
                download = urllib.urlretrieve(downloadURL, links)
                message('NOAA ' + htmlName + '.csv.gz has been downloaded.')
                return links

def decompress(csvName, workSpace, links, year):
    'This decompresses and saves CSV files for use.'
    newCSV = year + '_' + csvName + '.csv'
    oldFilePath = os.path.join(workSpace, links)
    newFilePath = os.path.join(workSpace, newCSV)

    with gzip.open(oldFilePath, 'rb') as file_in, open(newFilePath, 'wb') as file_out:
        shutil.copyfileobj(file_in, file_out)
    message('NOAA ' + csvName + '.csv.gz has been decompressed.')
    arcpy.Delete_management(links)
    message('NOAA ' + csvName + '.csv has been written to disk, compressed file has been deleted.')
    return newCSV

def xyEvent(newCSV):
    'Creates an XY Event feature layer.'
    xField = 'LONGITUDE'
    yField = 'LATITUDE'
    xyLayer = 'NOAA_' + year + '_xyFeature'
    spatialRef = 4326
    arcpy.MakeXYEventLayer_management(newCSV, xField, yField, xyLayer, spatialRef, '')
    print 'XY feature event created. Creating temporary feature class.'
    return xyLayer

def copyFeature(makeFeature, pathFGDB, newCSV):
    'Creates new feature and deletes the feature layer.'
    xyFeature = os.path.join(pathFGDB, makeFeature)
    arcpy.CopyFeatures_management(makeFeature, xyFeature)
    arcpy.Delete_management(makeFeature)
    arcpy.Delete_management(newCSV)
    return xyFeature

def replacechars(replaceCSV):
    'Replaces characters for other characters in a CSV.'
    message('Replacing illegal pipe characters in details.csv with hyphens.')
    with open(replaceCSV, 'rb') as infile, open(replaceCSV.strip('.csv') + '_temp.csv', 'wb') as outfile:
        reader = csv.reader(infile)
        writer = csv.writer(outfile)
        for row in reader:
            row = [x.replace('|', '-') for x in row]
            writer.writerow(row)
    os.remove(replaceCSV)
    os.rename(replaceCSV.strip('.csv') + '_temp.csv', replaceCSV)
    return replaceCSV

def joinFeature(xyFeature, joinTable, pathFGDB, year, stateName, eventType):
    'Joins XY feature to CSV and creates new feature.'
    layerNameJoin = 'stormJoin_' + year
    joinField = 'EVENT_ID'
    outNameJoin = stateName + '_' + eventType.replace(' ', '_') + '_' + year + '_join'
    outFeatureJoin = os.path.join(pathFGDB, outNameJoin)

    message('Preparing for join procedure.')
    arcpy.MakeFeatureLayer_management(xyFeature, layerNameJoin)

    message('Join procedure complete.')
    arcpy.AddJoin_management(layerNameJoin, joinField, joinTable, joinField)
    
    message('Creating new joined feature class.')
    joinFeature = arcpy.CopyFeatures_management(layerNameJoin, outFeatureJoin)

    message('Cleaning up unneeded files')
    arcpy.Delete_management(layerNameJoin)
    arcpy.Delete_management(xyFeature)
    arcpy.Delete_management(joinTable)

    return joinFeature

def selection(joinFeature, pathFGDB, year, stateName, eventType):
    'Selects user input features from the joined feature.'
    selectName = 'storm_' + year
    outName = stateName + '_' + eventType.replace(' ', '_') + '_' + year
    outFeature = os.path.join(pathFGDB, outName)

    eventField = 'EVENT_TYPE'
    stateField = 'STATE'
    stateQuery = '"' + stateField + '" = ' + "'" + stateName + "'"
    stormQuery = '"' + eventField + '" = ' + "'" + eventType + "'"
    attributeQuery = stateQuery + ' AND ' + stormQuery

    arcpy.MakeFeatureLayer_management(joinFeature, selectName)
                            
    message('Selecting from user input varaibles: ' + stateName + ', ' + eventType)
    arcpy.SelectLayerByAttribute_management(selectName, 'NEW_SELECTION', attributeQuery)
    attributeCount = int(str(arcpy.GetCount_management(selectName)))

    if not attributeCount > 0:
        arcpy.Delete_management(selectName)
        message('No features selected.')
    else:
        message('Features selected.')
                                   
        message('Creating new feature class.')
        arcpy.CopyFeatures_management(selectName, outFeature)

        message('Cleaning up unneeded files')
        arcpy.Delete_management(joinFeature)
        arcpy.Delete_management(selectName)
        return
    

# Set user input variables - these variables will be set by the user
workSpace = r'D:\PennState\Geog485\FinalProject\ScriptTest'
years = ['1950', '2015', '2016', '2020']
eventType = 'Heavy Rain'
stateName = 'TEXAS'
message('Input variables successfully set.')

# Set environments
os.chdir(workSpace)
arcpy.env.workspace = workSpace
arcpy.env.overwriteOutput = True
arcpy.env.qualifiedFieldNames = False
now = datetime.datetime.now()
message('Environments successfully set.')

# Check FGDB availability, use or create
pathFGDB = fgdb(workSpace)

# Loop multiple years for data acquisition and generation
for year in years:

    # Validate year
    if not 1950 <= int(year) <= now.year:
        message(year + ' is out of range.')
        continue
    message(year + ' is in range.')

    # Download the NOAA storm location file
    locLinks = download('location', year)

    # Decompress and save CSV file
    locCSV = decompress('location', workSpace, locLinks, year)

    try:
        # Create xyEvent layer
        xyLayer = xyEvent(locCSV)

        # Make xyEvent feature
        xyFeature = copyFeature(xyLayer, pathFGDB, locCSV)
        
        # Download the NOAA storm details file
        detLinks = download('details', year)

        # Decompress and save CSV file
        detCSV = decompress('details', workSpace, detLinks, year)

        # Replace pipes ( | ) with hyphens ( - )
        replaceCSV = replacechars(detCSV)

        # Create join feature, delete temporary feature
        joinFC = joinFeature(xyFeature, replaceCSV, pathFGDB, year, stateName, eventType)

        # Select and export user-specified location and event type
        selection(joinFC, pathFGDB, year, stateName, eventType)
    except(arcpy.ExecuteError):
        message('The NOAA location csv was not generated. No features were joined.')
        arcpy.Delete_management(locCSV)