GCP

# Cell 1
import pandas as pd
import os, sys
response_data = pd.read_csv('Downloads/Fire_Rescue_Responses.csv')
response_data.head()
rd_addr_loc = response_data['Location Address'].str.split('\n', 1, expand=True)
rd_addr_loc.columns = ['Address', 'Location']

rd_date = response_data['Response Date'].str.split(' ', 1, expand=True)
rd_date.columns = ['Date', 'Time']
rd = pd.merge(response_data, rd_addr_loc, right_index=True, left_index=True)
rd = pd.merge(rd, rd_date, right_index=True, left_index=True)
rd = rd[['Date', 'Response Type', 'Location Name', 'Address', 'Location']]
rd.to_csv('Downloads/responses.csv')
rd.head()

Cell 1

Imports

import pandas as pd
import os, sys
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

Cell 2

response_data = pd.read_csv('Downloads/Fire_Rescue_Responses.csv')
response_data.head()
rd_addr_loc = response_data['Location Address'].str.split('\n', 1, expand=True)
rd_addr_loc.columns = ['Address', 'Location']

rd_date = response_data['Response Date'].str.split(' ', 1, expand=True)
rd_date.columns = ['Date', 'Time']
rd = pd.merge(response_data, rd_addr_loc, right_index=True, left_index=True)
rd = pd.merge(rd, rd_date, right_index=True, left_index=True)
rd = rd[['Date', 'Response Type', 'Location Name', 'Address', 'Location']]
rd.to_csv('Downloads/responses.csv')
rd.head()

Cell 3

import numpy as np
from IPython.display import display
raw_electric_data = pd.read_csv('Downloads/GRU_Customer_Electric_Consumption.csv')
display("Raw data")
display(raw_electric_data.head())
years = np.sort(raw_electric_data.Year.unique())
display(f"Years: {years}")
#Range: 2012-2018
location = raw_electric_data['Location'].str.split('\n', 0, expand=True)
location.columns = ['Address', 'Area', 'Geolocation']
#display(location.head())
electric_data_2 = pd.merge(raw_electric_data, location, right_index=True, left_index=True)
electric_data_3 = electric_data_2[['Address', 'Geolocation', 'KWH Consumption', ]]
electric_data_3.columns = ['Address', 'Geolocation', 'Consumption']
display("Grouping by median electric consumption")
consumption = electric_data_3.groupby('Address').agg({'Consumption':'median'}).reset_index()
#consumption.reset_index(level='Address')
display(consumption.head())
electric_data_4 = electric_data_3[['Address', 'Geolocation']]
display("Removing address duplicates")
electric_data_5 = electric_data_4.drop_duplicates('Address')
display(electric_data_5.head())
display("Merging location and consumption")
electric_data = pd.merge(electric_data_5, consumption, left_on = 'Address', right_on = 'Address', how = 'left')
display(electric_data.head())
electric_data.to_csv('Downloads/electric_consumption.csv')

Cell 4

responses = pd.read_csv('Downloads/responses.csv')
consumption = pd.read_csv('Downloads/electric_consumption.csv')
#display(responses.columns)
#display(consumption.columns)
display("Emergency Response Data")
display(responses.head())
num_calls = responses.shape[0]
display(f"Number of emergency calls: '{num_calls}'")
display("Electrical Consumption Data")
display(consumption.head())
num_households = consumption.shape[0]
display(f"Number of addresses with consumption data: '{num_households}'")
responses['address_lower'] = responses['Address'].str.lower()
consumption['address_lower'] = consumption['Address'].str.lower()
merged = responses.merge(consumption, left_on = 'address_lower', right_on = 'address_lower', how = 'left')
df = merged[['Date', 'Response Type', 'Location Name', 'Address_x', 'Location', 'Consumption']]
display("Combined Data")
display(df.head())
final = df.dropna()
num_calls = final.shape[0]
display(f"Number of final records with response and consumption data: {num_calls}")
display(final.head())
final.to_csv('Downloads/gainesville_emergency_response_and_consumption_data.csv')

Cell 5

#df.info()
#display(df.head())
#df['Address'] = df['Address Category']
#df = final
#df.columns = ['Date', 'Type', 'Name', 'Address', 'Location', 'Consumption']
#df['Type Category'] = df["Type"].astype('category')
#date = pd.to_datetime(df['Date'])
#df['Date'] = date

Name

#df['Name Category'] = df["Name"].astype('category')
#df['Name'] = df['Name Category']

Type

#df['Type'] = df['Type Category']
#df['Address'] = df['Address Category']

??? Location

#df = df[['Date', 'Type', 'Name', 'Address', 'Location', 'Consumption']]
#df['Address Category'] = df["Address"].astype('category')
df.info()
display(df.head())

Cell 6

typ_counts = sns.countplot(x="Type", data=df)
sns.set(style="ticks")
#display(consumption_dist = sns.distplot(df['Consumption']))

Cell 7

consumption_dist = sns.distplot(df['Consumption'])
plt.title("Raw Consumption Data Distibution")

Cell 8

display("Median Consumption:", df['Consumption'].median())
display("Maximum Consumption:", df['Consumption'].max())
display(df['Consumption'].describe())
display(df['Date'].describe())
display(df['Name'].describe())

<class 'pandas.core.frame.DataFrame'>
Int64Index: 33485 entries, 0 to 105505
Data columns (total 6 columns):
Date 33485 non-null datetime64[ns]
Type 33485 non-null category
Name 33485 non-null category
Address 33485 non-null category
Location 33485 non-null object
Consumption 33485 non-null float64
dtypes: category(3), datetime64ns, float64(1), object(1)

#display(df['Consumption'].head())
consumption_min = df['Consumption'].min()
display("Consumption min: ", consumption_min)
consumption_max = df['Consumption'].max()
display("Consumption max: ", consumption_max)
consumption_median = df['Consumption'].median()
display("Consumption median: ", consumption_median)
consumption_std = df['Consumption'].std()
display("Consumption std: ", consumption_std)
display("Consumption count: ", df['Consumption'].shape[0])
consumption_mode = df['Consumption'].mode()
display("Consumption mode: ", consumption_mode)

List of outliers

consumption_outliers = df['Consumption'] < ( df['Consumption'].median() + 3 * consumption_std )
display(consumption_outliers.describe())
display(df['Consumption'].shape[0])
display(consumption_outliers.shape[0])
from scipy import stats
df1 = df[((df['Consumption'] - df['Consumption'].mean()) / df['Consumption'].std()).abs() < 3]
df1['Consumption'].describe()

#display(df['Consumption'].head())
consumption_min = df['Consumption'].min()
display("Consumption min: ", consumption_min)
consumption_max = df['Consumption'].max()
display("Consumption max: ", consumption_max)
consumption_median = df['Consumption'].median()
display("Consumption median: ", consumption_median)
consumption_std = df['Consumption'].std()
display("Consumption std: ", consumption_std)
display("Consumption count: ", df['Consumption'].shape[0])
consumption_mode = df['Consumption'].mode()
display("Consumption mode: ", consumption_mode)

List of outliers

consumption_outliers = df['Consumption'] < ( df['Consumption'].median() + 3 * consumption_std )
display(consumption_outliers.describe())
display(df['Consumption'].shape[0])
display(consumption_outliers.shape[0])
pd.options.display.float_format = '{:,.2f}'.format
res_df = df[((df['Consumption'] - df['Consumption'].mean()) / df['Consumption'].std()).abs() < 2]
display("Residential: ", res_df['Consumption'].describe())
com_df = df[((df['Consumption'] - df['Consumption'].mean()) / df['Consumption'].std()).abs() >= 2]
display("Commercial: ", com_df['Consumption'].describe())
pd.reset_option('^display.float_format', silent=True)

Plot 1:

display(cons_res_dist = sns.distplot(res_df['Consumption']))
plt.title("Residential Consumption Data Distibution")

Plot 2

display(cons_res_dist = sns.distplot(com_df['Consumption']))
plt.title("Commercial Consumption Data Distibution")

res_ems = res_df[res_df['Type'] == 'EMS']
#display(res_ems.head())
buckets = res_ems.groupby('Consumption Categories')['Type'].count().reset_index()
#display(buckets.head())
buckets.columns = ['Consumption', 'Calls']

#buckets['Tiers'] =
buckets['Consumption'].to_string().split(', ')[1].split('\n')[1][2:]
#display(buckets.head())
#.split(', ', expand=True).reset_index()
#display(buckets.head(20))

#sns.set(style="whitegrid", color_codes=True)
g = sns.factorplot(x="Calls", y="Consumption", data=buckets, size=6, kind="bar", palette="muted")

res_df['Consumption Categories'] = pd.cut(res_df['Consumption'], 12)
#display(res_df.head())
#display(res_df.tail())
#display(res_df['Consumption Categories'].head())
display(res_df.info())

for call_type in res_df['Type'].unique():
display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
#display(res_ems.head())
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
display(g = sns.factorplot(x=call_type_name , y="Consumption", data=buckets, size=6, kind="bar", palette="muted"))

Cell 1

#display(df['Consumption'].head())
consumption_min = df['Consumption'].min()
display("Consumption min: ", consumption_min)
consumption_max = df['Consumption'].max()
display("Consumption max: ", consumption_max)
consumption_median = df['Consumption'].median()
display("Consumption median: ", consumption_median)
consumption_std = df['Consumption'].std()
display("Consumption std: ", consumption_std)
display("Consumption count: ", df['Consumption'].shape[0])
consumption_mode = df['Consumption'].mode()
display("Consumption mode: ", consumption_mode)

List of outliers

consumption_outliers = df['Consumption'] < ( df['Consumption'].median() + 1 * consumption_std )
display(consumption_outliers.describe())
display(df['Consumption'].shape[0])
display(consumption_outliers.shape[0])
pd.options.display.float_format = '{:,.2f}'.format
#res_df = df[((df['Consumption'] - df['Consumption'].median()) / df['Consumption'].std()).abs() < 1]
res_df = df[(df['Consumption'] <= 5000)]
display("Residential: ", res_df['Consumption'].describe())
#com_df = df[((df['Consumption'] - df['Consumption'].median()) / df['Consumption'].std()).abs() >= 1]
com_df = df[(df['Consumption'] > 5000) &(df['Consumption'] < 200000)]
display("Commercial: ", com_df['Consumption'].describe())
pd.reset_option('^display.float_format', silent=True)

Cell 2

for call_type in res_df['Type'].unique():
display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
#display(res_ems.head())
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
display(g = sns.factorplot(x=call_type_name , y="Consumption", data=buckets, size=6, kind="bar", palette="muted"))

call_corr = pd.DataFrame()
for call_type in res_df['Type'].unique():
#display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
#display(buckets)
if len(call_corr.axes[0]) == 0:
display("Starting from", call_type_name)
call_corr = buckets
#display(call_corr)
else:
display("Adding data for", call_type_name)
call_corr = pd.merge(call_corr, buckets, right_index=True, left_index=True)
#display(call_corr)
calls_df = call_corr[['Consumption', 'EMS Calls', 'FIRE Calls', 'ALM Calls', 'SVC Calls', 'HAZ Calls']]
#calls_df.columns = ['Consumption', 'EMS', 'FIRE', 'ALM', 'SVC', 'HAZ']
#display(calls_df)
calls_only_df = calls_df[['EMS Calls', 'FIRE Calls', 'ALM Calls', 'SVC Calls', 'HAZ Calls']]
calls_only_df.corr(method='spearman')

EMS Calls	FIRE Calls	ALM Calls	SVC Calls	HAZ Calls

EMS Calls 1.000000 0.848759 0.525009 0.716986 0.808898
FIRE Calls 0.848759 1.000000 0.696650 0.571377 0.820447
ALM Calls 0.525009 0.696650 1.000000 0.331825 0.443817
SVC Calls 0.716986 0.571377 0.331825 1.000000 0.699963
HAZ Calls 0.808898 0.820447 0.443817 0.699963 1.000000