-
-
Save moskalenko/94648276eabfb90b9ecd19e0c8c2028e to your computer and use it in GitHub Desktop.
# Cell 1 | |
import pandas as pd | |
import os, sys | |
response_data = pd.read_csv('Downloads/Fire_Rescue_Responses.csv') | |
response_data.head() | |
rd_addr_loc = response_data['Location Address'].str.split('\n', 1, expand=True) | |
rd_addr_loc.columns = ['Address', 'Location'] | |
rd_date = response_data['Response Date'].str.split(' ', 1, expand=True) | |
rd_date.columns = ['Date', 'Time'] | |
rd = pd.merge(response_data, rd_addr_loc, right_index=True, left_index=True) | |
rd = pd.merge(rd, rd_date, right_index=True, left_index=True) | |
rd = rd[['Date', 'Response Type', 'Location Name', 'Address', 'Location']] | |
rd.to_csv('Downloads/responses.csv') | |
rd.head() |
res_ems = res_df[res_df['Type'] == 'EMS']
#display(res_ems.head())
buckets = res_ems.groupby('Consumption Categories')['Type'].count().reset_index()
#display(buckets.head())
buckets.columns = ['Consumption', 'Calls']
#buckets['Tiers'] =
buckets['Consumption'].to_string().split(', ')[1].split('\n')[1][2:]
#display(buckets.head())
#.split(', ', expand=True).reset_index()
#display(buckets.head(20))
#sns.set(style="whitegrid", color_codes=True)
g = sns.factorplot(x="Calls", y="Consumption", data=buckets, size=6, kind="bar", palette="muted")
res_df['Consumption Categories'] = pd.cut(res_df['Consumption'], 12)
#display(res_df.head())
#display(res_df.tail())
#display(res_df['Consumption Categories'].head())
display(res_df.info())
for call_type in res_df['Type'].unique():
display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
#display(res_ems.head())
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
display(g = sns.factorplot(x=call_type_name , y="Consumption", data=buckets, size=6, kind="bar", palette="muted"))
Cell 1
#display(df['Consumption'].head())
consumption_min = df['Consumption'].min()
display("Consumption min: ", consumption_min)
consumption_max = df['Consumption'].max()
display("Consumption max: ", consumption_max)
consumption_median = df['Consumption'].median()
display("Consumption median: ", consumption_median)
consumption_std = df['Consumption'].std()
display("Consumption std: ", consumption_std)
display("Consumption count: ", df['Consumption'].shape[0])
consumption_mode = df['Consumption'].mode()
display("Consumption mode: ", consumption_mode)
List of outliers
consumption_outliers = df['Consumption'] < ( df['Consumption'].median() + 1 * consumption_std )
display(consumption_outliers.describe())
display(df['Consumption'].shape[0])
display(consumption_outliers.shape[0])
pd.options.display.float_format = '{:,.2f}'.format
#res_df = df[((df['Consumption'] - df['Consumption'].median()) / df['Consumption'].std()).abs() < 1]
res_df = df[(df['Consumption'] <= 5000)]
display("Residential: ", res_df['Consumption'].describe())
#com_df = df[((df['Consumption'] - df['Consumption'].median()) / df['Consumption'].std()).abs() >= 1]
com_df = df[(df['Consumption'] > 5000) &(df['Consumption'] < 200000)]
display("Commercial: ", com_df['Consumption'].describe())
pd.reset_option('^display.float_format', silent=True)
Cell 2
for call_type in res_df['Type'].unique():
display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
#display(res_ems.head())
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
display(g = sns.factorplot(x=call_type_name , y="Consumption", data=buckets, size=6, kind="bar", palette="muted"))
call_corr = pd.DataFrame()
for call_type in res_df['Type'].unique():
#display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
#display(buckets)
if len(call_corr.axes[0]) == 0:
display("Starting from", call_type_name)
call_corr = buckets
#display(call_corr)
else:
display("Adding data for", call_type_name)
call_corr = pd.merge(call_corr, buckets, right_index=True, left_index=True)
#display(call_corr)
calls_df = call_corr[['Consumption', 'EMS Calls', 'FIRE Calls', 'ALM Calls', 'SVC Calls', 'HAZ Calls']]
#calls_df.columns = ['Consumption', 'EMS', 'FIRE', 'ALM', 'SVC', 'HAZ']
#display(calls_df)
calls_only_df = calls_df[['EMS Calls', 'FIRE Calls', 'ALM Calls', 'SVC Calls', 'HAZ Calls']]
calls_only_df.corr(method='spearman')
EMS Calls FIRE Calls ALM Calls SVC Calls HAZ Calls
EMS Calls 1.000000 0.848759 0.525009 0.716986 0.808898
FIRE Calls 0.848759 1.000000 0.696650 0.571377 0.820447
ALM Calls 0.525009 0.696650 1.000000 0.331825 0.443817
SVC Calls 0.716986 0.571377 0.331825 1.000000 0.699963
HAZ Calls 0.808898 0.820447 0.443817 0.699963 1.000000
Plot 1:
display(cons_res_dist = sns.distplot(res_df['Consumption']))
plt.title("Residential Consumption Data Distibution")
Plot 2
display(cons_res_dist = sns.distplot(com_df['Consumption']))
plt.title("Commercial Consumption Data Distibution")