noaa.py

# Name:         storm data conversion, extraction, and selection
# Description:  Final Project
# Created by:   socio
# Date:         7/10/2016

# Import system modules
import arcpy, csv, datetime, gzip, os, shutil, urllib, urllib2
from bs4 import BeautifulSoup

# Functions
def message(message):
    'This prints a passed string as both print and arcpy.AddMessage().'
    print message
    arcpy.AddMessage(message)
    return

def download(htmlName, year, links, urlNOAA):
    'Downloads the CSV file for decompression'
    if 'd' + year in links:
        if htmlName in links:
            downloadURL = urlNOAA + links
            download = urllib.urlretrieve(downloadURL, links)
            message('NOAA ' + htmlName + '.csv.gz has been downloaded.')
            return

def decompress(csvName, workSpace, oldCSV, newCSV):
    'This decompresses and saves CSV files for use.'
    oldFilePath = os.path.join(workSpace, oldCSV)
    newFilePath = os.path.join(workSpace, newCSV)

    with gzip.open(oldFilePath, 'rb') as file_in, open(newFilePath, 'wb') as file_out:
        shutil.copyfileobj(file_in, file_out)
    message('NOAA ' + csvName + '.csv.gz has been decompressed.')
    arcpy.Delete_management(oldCSV)
    message('NOAA ' + csvName +'.csv has been written to disk, compressed file has been deleted.')
    return

# Set user input variables - these variables will be set by the user
workSpace = r'C:\Users\nrose\Desktop\TestScript'
years = ['2016']
eventType = 'Heavy Rain'
stateName = 'TEXAS'
message('Input variables successfully set.')

# Set environments
os.chdir(workSpace)
arcpy.env.workspace = workSpace
arcpy.env.overwriteOutput = True
arcpy.env.qualifiedFieldNames = False
urlNOAA = 'https://www1.ncdc.noaa.gov/pub/data/swdi/stormevents/csvfiles/'
now = datetime.datetime.now()
message('Environments successfully set.')

# Check FGDB availability, use or create
stormFGDB = 'NOAA_StormDatabase.gdb'
pathFGDB = os.path.join(workSpace, stormFGDB)
if os.path.exists(pathFGDB):
    message('The FGDB already exists and will be used to store new storm data.')
else:
    # Create the FGDB
    message('The FGDB does not exist.  A new one will be created.')
    arcpy.CreateFileGDB_management(workSpace, stormFGDB)

# Loop multiple years for data acquisition and generation
for year in years:

    # Validate year
    if not 1950 <= int(year) <= now.year:
        message(year + ' is out of range.')
        continue
    message(year + ' is in range.')
    
    # Download the NOAA storm location file
    message('Beginning input year validation.')
    site = urllib2.urlopen(urlNOAA)
    html = site.read()
    soup = BeautifulSoup(html, 'html.parser')

    for link in soup.find_all('a'):
        links = link.get('href')
        download('location', year, links, urlNOAA)
        
        # Decompress and store NOAA location CSV file
        oldCSV = links
        newCSV = year + '_location.csv'
        decompress('location', workSpace, oldCSV, newCSV)
        
        try:
            # Make XY event layer
            stormTable = newCSV
            xField = 'LONGITUDE'
            yField = 'LATITUDE'
            xyLayer = 'NOAA_' + year + '_xyFeature'
            tempFC = xyLayer
            spatialRef = 4326
            arcpy.MakeXYEventLayer_management(stormTable, xField, yField, xyLayer, spatialRef, '')
            message('XY feature event created. Creating temporary feature class.')
            
            # Make temporary shapefile
            arcpy.CopyFeatures_management(xyLayer, os.path.join(pathFGDB, tempFC))
            arcpy.Delete_management(xyLayer)
            message('Temporary feature class created.')
        except(arcpy.ExecuteError):
            message('The NOAA location csv was empty or incomplete. No feature created.')

        # Clean up
        arcpy.Delete_management(newCSV)

        # Download the NOAA storm detail file
        site = urllib2.urlopen(urlNOAA)
        html = site.read()
        soup = BeautifulSoup(html, 'html.parser')

        for link in soup.find_all('a'):
            links = link.get('href')
            if 'd' + year in links:
                if 'details' in links:
                    downloadURL = urlNOAA + links
                    download = urllib.urlretrieve(downloadURL, links)
                    message('NOAA details.csv.gz has been downloaded.')
                    
                    # Decompress the NOAA storm location file and write it to the working space
                    oldCSV = links
                    newCSV = year + '_details' + '.csv'
                    decompress('details', workSpace, oldCSV, newCSV)

                    # Replace pipes ( | ) with dashes ( - )
                    message('Replacing illegal pipe characters in details.csv with hyphens.')
                    with open(newCSV, 'rb') as infile, open(newCSV.strip('.csv') + '_temp.csv', 'wb') as outfile:
                        reader = csv.reader(infile)
                        writer = csv.writer(outfile)
                        for row in reader:
                            row = [x.replace('|', '-') for x in row]
                            writer.writerow(row)

                    os.remove(newCSV)
                    os.rename(newCSV.strip('.csv') + '_temp.csv', newCSV)

                    # Set targets for join
                    message('Preparing for join procedure...')
                    inFeatures = os.path.join(pathFGDB, tempFC)
                    layerNameJoin = 'stormJoin_' + year
                    layerName = 'storm_' + year
                    joinTable = newCSV
                    joinField = 'EVENT_ID'
                    outNameJoin = stateName + '_' + eventType.replace(' ', '_') + '_' + year + '_join'
                    outFeatureJoin = os.path.join(pathFGDB, outNameJoin)
                    outName = stateName + '_' + eventType.replace(' ', '_') + '_' + year
                    outFeature = os.path.join(pathFGDB, outName)

                    try:
                        # Make feature layer for first attribute selection
                        arcpy.MakeFeatureLayer_management(inFeatures, layerNameJoin)

                        # Join the feature layer to the CSV
                        arcpy.AddJoin_management(layerNameJoin, joinField, joinTable, joinField)
                        message('...join procedure complete.')

                        # Copy the layer to a new permanent feature class
                        message('Creating new joined feature class.')
                        arcpy.CopyFeatures_management(layerNameJoin, outFeatureJoin)

                        # Make feature layer for first attribute selection
                        arcpy.MakeFeatureLayer_management(outFeatureJoin, layerName)

                        # Select by attributes
                        # State and Storm query
                        eventField = 'EVENT_TYPE'
                        stateField = 'STATE'
                        stateQuery = '"' + stateField + '" = ' + "'" + stateName + "'"
                        stormQuery = '"' + eventField + '" = ' + "'" + eventType + "'"
                        attributeQuery = stateQuery + ' AND ' + stormQuery
                                                        
                        message('Selecting from user input varaibles: ' + stateName + ', ' + eventType)
                        arcpy.SelectLayerByAttribute_management(layerName, 'NEW_SELECTION', attributeQuery)
                        attributeCount = int(str(arcpy.GetCount_management(layerName)))
                        print attributeCount
                        if not attributeCount > 0:
                            arcpy.Delete_management(layerNameJoin)
                            arcpy.Delete_management(layerName)
                            arcpy.Delete_management(joinTable)
                            message('No features selected.')
                            continue
                        message('Features selected.')
                                                       
                        # Copy the layer to a new permanent feature class
                        message('Creating new feature class.')
                        arcpy.CopyFeatures_management(layerName, outFeature)

                        # Clean up
                        message('Cleaning up extraneous files.')
                        arcpy.Delete_management(layerNameJoin)
                        arcpy.Delete_management(layerName)
                        arcpy.Delete_management(outFeatureJoin)
                        arcpy.Delete_management(inFeatures)
                        arcpy.Delete_management(joinTable)
                    except(arcpy.ExecuteError):
                        message('The NOAA location csv was not generated. No features were joined.')
                        arcpy.Delete_management(joinTable)