Created
July 24, 2017 18:48
-
-
Save anonymous/3dc70f46475c16557fb6c3c1f234008b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Functions | |
def message(message): | |
'This prints a passed string as both print and arcpy.AddMessage().' | |
print message | |
arcpy.AddMessage(message) | |
return | |
def download(htmlName): | |
'Downloads the CSV file for decompression' | |
if 'd' + year in links: | |
if csvName in links: | |
downloadURL = urlNOAA + links | |
download = urllib.urlretrieve(downloadURL, links) | |
download | |
message('NOAA ' + csvName + '.csv.gz has been downloaded.') | |
def decompress(csvName): | |
'This decompresses and saves CSV files for use.' | |
oldFilePath = os.path.join(workSpace, oldCSV) | |
newFilePath = os.path.join(workSpace, newCSV) | |
with gzip.open(oldFilePath, 'rb') as file_in, open(newFilePath, 'wb') as file_out: | |
shutil.copyfileobj(file_in, file_out) | |
message('NOAA ' + csvName + '.csv.gz has been decompressed.') | |
arcpy.Delete_management(oldCSV) | |
message('NOAA ' + csvName +'.csv has been written to disk, compressed file has been deleted.') | |
return | |
# Import system modules | |
import arcpy, csv, datetime, gzip, os, shutil, urllib, urllib2 | |
from bs4 import BeautifulSoup | |
message('System modules successfully imported.') | |
# Set user input variables - these variables will be set by the user | |
workSpace = r'C:\Users\nrose\Desktop\TestScript' | |
years = ['2016'] | |
eventType = 'Heavy Rain' | |
stateName = 'TEXAS' | |
message('Input variables successfully set.') | |
# Set environments | |
os.chdir(workSpace) | |
arcpy.env.workspace = workSpace | |
arcpy.env.overwriteOutput = True | |
arcpy.env.qualifiedFieldNames = False | |
urlNOAA = 'https://www1.ncdc.noaa.gov/pub/data/swdi/stormevents/csvfiles/' | |
now = datetime.datetime.now() | |
message('Environments successfully set.') | |
# Check FGDB availability, use or create | |
stormFGDB = 'NOAA_StormDatabase.gdb' | |
pathFGDB = os.path.join(workSpace, stormFGDB) | |
if os.path.exists(pathFGDB): | |
message('The FGDB already exists and will be used to store new storm data.') | |
else: | |
# Create the FGDB | |
message('The FGDB does not exist. A new one will be created.') | |
arcpy.CreateFileGDB_management(workSpace, stormFGDB) | |
# Loop multiple years for data acquisition and generation | |
for year in years: | |
# Validate year | |
if not 1950 <= int(year) <= now.year: | |
message(year + ' is out of range.') | |
continue | |
message(year + ' is in range.') | |
# Download the NOAA storm location file | |
message('Beginning input year validation.') | |
site = urllib2.urlopen(urlNOAA) | |
html = site.read() | |
soup = BeautifulSoup(html, 'html.parser') | |
for link in soup.find_all('a'): | |
links = link.get('href') | |
download('location') | |
# Decompress and store NOAA location CSV file | |
oldCSV = links | |
newCSV = year + '_location.csv' | |
decompress('location') | |
try: | |
# Make XY event layer | |
stormTable = newCSV | |
xField = 'LONGITUDE' | |
yField = 'LATITUDE' | |
xyLayer = 'NOAA_' + year + '_xyFeature' | |
tempFC = xyLayer | |
spatialRef = 4326 | |
arcpy.MakeXYEventLayer_management(stormTable, xField, yField, xyLayer, spatialRef, '') | |
message('XY feature event created. Creating temporary feature class.') | |
# Make temporary shapefile | |
arcpy.CopyFeatures_management(xyLayer, os.path.join(pathFGDB, tempFC)) | |
arcpy.Delete_management(xyLayer) | |
message('Temporary feature class created.') | |
except(arcpy.ExecuteError): | |
message('The NOAA location csv was empty or incomplete. No feature created.') | |
# Clean up | |
arcpy.Delete_management(newCSV) | |
# Download the NOAA storm detail file | |
site = urllib2.urlopen(urlNOAA) | |
html = site.read() | |
soup = BeautifulSoup(html, 'html.parser') | |
for link in soup.find_all('a'): | |
links = link.get('href') | |
if 'd' + year in links: | |
if 'details' in links: | |
downloadURL = urlNOAA + links | |
download = urllib.urlretrieve(downloadURL, links) | |
download | |
message('NOAA details.csv.gz has been downloaded.') | |
# Decompress the NOAA storm location file and write it to the working space | |
oldCSV = links | |
newCSV = year + '_details' + '.csv' | |
decompress('details') | |
# Replace pipes ( | ) with dashes ( - ) | |
message('Replacing illegal pipe characters in details.csv with hyphens.') | |
with open(newCSV, 'rb') as infile, open(newCSV.strip('.csv') + '_temp.csv', 'wb') as outfile: | |
reader = csv.reader(infile) | |
writer = csv.writer(outfile) | |
for row in reader: | |
row = [x.replace('|', '-') for x in row] | |
writer.writerow(row) | |
os.remove(newCSV) | |
os.rename(newCSV.strip('.csv') + '_temp.csv', newCSV) | |
# Set targets for join | |
message('Preparing for join procedure...') | |
inFeatures = os.path.join(pathFGDB, tempFC) | |
layerNameJoin = 'stormJoin_' + year | |
layerName = 'storm_' + year | |
joinTable = newCSV | |
joinField = 'EVENT_ID' | |
outNameJoin = stateName + '_' + eventType.replace(' ', '_') + '_' + year + '_join' | |
outFeatureJoin = os.path.join(pathFGDB, outNameJoin) | |
outName = stateName + '_' + eventType.replace(' ', '_') + '_' + year | |
outFeature = os.path.join(pathFGDB, outName) | |
try: | |
# Make feature layer for first attribute selection | |
arcpy.MakeFeatureLayer_management(inFeatures, layerNameJoin) | |
# Join the feature layer to the CSV | |
arcpy.AddJoin_management(layerNameJoin, joinField, joinTable, joinField) | |
message('...join procedure complete.') | |
# Copy the layer to a new permanent feature class | |
message('Creating new joined feature class.') | |
arcpy.CopyFeatures_management(layerNameJoin, outFeatureJoin) | |
# Make feature layer for first attribute selection | |
arcpy.MakeFeatureLayer_management(outFeatureJoin, layerName) | |
# Select by attributes | |
# State and Storm query | |
eventField = 'EVENT_TYPE' | |
stateField = 'STATE' | |
stateQuery = '"' + stateField + '" = ' + "'" + stateName + "'" | |
stormQuery = '"' + eventField + '" = ' + "'" + eventType + "'" | |
attributeQuery = stateQuery + ' AND ' + stormQuery | |
message('Selecting from user input varaibles: ' + stateName + ', ' + eventType) | |
arcpy.SelectLayerByAttribute_management(layerName, 'NEW_SELECTION', attributeQuery) | |
attributeCount = int(str(arcpy.GetCount_management(layerName))) | |
print attributeCount | |
if not attributeCount > 0: | |
arcpy.Delete_management(layerNameJoin) | |
arcpy.Delete_management(layerName) | |
arcpy.Delete_management(joinTable) | |
message('No features selected.') | |
continue | |
message('Features selected.') | |
# Copy the layer to a new permanent feature class | |
message('Creating new feature class.') | |
arcpy.CopyFeatures_management(layerName, outFeature) | |
# Clean up | |
message('Cleaning up extraneous files.') | |
arcpy.Delete_management(layerNameJoin) | |
arcpy.Delete_management(layerName) | |
arcpy.Delete_management(outFeatureJoin) | |
arcpy.Delete_management(inFeatures) | |
arcpy.Delete_management(joinTable) | |
except(arcpy.ExecuteError): | |
message('The NOAA location csv was not generated. No features were joined.') | |
arcpy.Delete_management(joinTable) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment