-
-
Save moskalenko/94648276eabfb90b9ecd19e0c8c2028e to your computer and use it in GitHub Desktop.
# Cell 1 | |
import pandas as pd | |
import os, sys | |
response_data = pd.read_csv('Downloads/Fire_Rescue_Responses.csv') | |
response_data.head() | |
rd_addr_loc = response_data['Location Address'].str.split('\n', 1, expand=True) | |
rd_addr_loc.columns = ['Address', 'Location'] | |
rd_date = response_data['Response Date'].str.split(' ', 1, expand=True) | |
rd_date.columns = ['Date', 'Time'] | |
rd = pd.merge(response_data, rd_addr_loc, right_index=True, left_index=True) | |
rd = pd.merge(rd, rd_date, right_index=True, left_index=True) | |
rd = rd[['Date', 'Response Type', 'Location Name', 'Address', 'Location']] | |
rd.to_csv('Downloads/responses.csv') | |
rd.head() |
#display(df['Consumption'].head())
consumption_min = df['Consumption'].min()
display("Consumption min: ", consumption_min)
consumption_max = df['Consumption'].max()
display("Consumption max: ", consumption_max)
consumption_median = df['Consumption'].median()
display("Consumption median: ", consumption_median)
consumption_std = df['Consumption'].std()
display("Consumption std: ", consumption_std)
display("Consumption count: ", df['Consumption'].shape[0])
consumption_mode = df['Consumption'].mode()
display("Consumption mode: ", consumption_mode)
List of outliers
consumption_outliers = df['Consumption'] < ( df['Consumption'].median() + 3 * consumption_std )
display(consumption_outliers.describe())
display(df['Consumption'].shape[0])
display(consumption_outliers.shape[0])
from scipy import stats
df1 = df[((df['Consumption'] - df['Consumption'].mean()) / df['Consumption'].std()).abs() < 3]
df1['Consumption'].describe()
#display(df['Consumption'].head())
consumption_min = df['Consumption'].min()
display("Consumption min: ", consumption_min)
consumption_max = df['Consumption'].max()
display("Consumption max: ", consumption_max)
consumption_median = df['Consumption'].median()
display("Consumption median: ", consumption_median)
consumption_std = df['Consumption'].std()
display("Consumption std: ", consumption_std)
display("Consumption count: ", df['Consumption'].shape[0])
consumption_mode = df['Consumption'].mode()
display("Consumption mode: ", consumption_mode)
List of outliers
consumption_outliers = df['Consumption'] < ( df['Consumption'].median() + 3 * consumption_std )
display(consumption_outliers.describe())
display(df['Consumption'].shape[0])
display(consumption_outliers.shape[0])
pd.options.display.float_format = '{:,.2f}'.format
res_df = df[((df['Consumption'] - df['Consumption'].mean()) / df['Consumption'].std()).abs() < 2]
display("Residential: ", res_df['Consumption'].describe())
com_df = df[((df['Consumption'] - df['Consumption'].mean()) / df['Consumption'].std()).abs() >= 2]
display("Commercial: ", com_df['Consumption'].describe())
pd.reset_option('^display.float_format', silent=True)
Plot 1:
display(cons_res_dist = sns.distplot(res_df['Consumption']))
plt.title("Residential Consumption Data Distibution")
Plot 2
display(cons_res_dist = sns.distplot(com_df['Consumption']))
plt.title("Commercial Consumption Data Distibution")
res_ems = res_df[res_df['Type'] == 'EMS']
#display(res_ems.head())
buckets = res_ems.groupby('Consumption Categories')['Type'].count().reset_index()
#display(buckets.head())
buckets.columns = ['Consumption', 'Calls']
#buckets['Tiers'] =
buckets['Consumption'].to_string().split(', ')[1].split('\n')[1][2:]
#display(buckets.head())
#.split(', ', expand=True).reset_index()
#display(buckets.head(20))
#sns.set(style="whitegrid", color_codes=True)
g = sns.factorplot(x="Calls", y="Consumption", data=buckets, size=6, kind="bar", palette="muted")
res_df['Consumption Categories'] = pd.cut(res_df['Consumption'], 12)
#display(res_df.head())
#display(res_df.tail())
#display(res_df['Consumption Categories'].head())
display(res_df.info())
for call_type in res_df['Type'].unique():
display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
#display(res_ems.head())
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
display(g = sns.factorplot(x=call_type_name , y="Consumption", data=buckets, size=6, kind="bar", palette="muted"))
Cell 1
#display(df['Consumption'].head())
consumption_min = df['Consumption'].min()
display("Consumption min: ", consumption_min)
consumption_max = df['Consumption'].max()
display("Consumption max: ", consumption_max)
consumption_median = df['Consumption'].median()
display("Consumption median: ", consumption_median)
consumption_std = df['Consumption'].std()
display("Consumption std: ", consumption_std)
display("Consumption count: ", df['Consumption'].shape[0])
consumption_mode = df['Consumption'].mode()
display("Consumption mode: ", consumption_mode)
List of outliers
consumption_outliers = df['Consumption'] < ( df['Consumption'].median() + 1 * consumption_std )
display(consumption_outliers.describe())
display(df['Consumption'].shape[0])
display(consumption_outliers.shape[0])
pd.options.display.float_format = '{:,.2f}'.format
#res_df = df[((df['Consumption'] - df['Consumption'].median()) / df['Consumption'].std()).abs() < 1]
res_df = df[(df['Consumption'] <= 5000)]
display("Residential: ", res_df['Consumption'].describe())
#com_df = df[((df['Consumption'] - df['Consumption'].median()) / df['Consumption'].std()).abs() >= 1]
com_df = df[(df['Consumption'] > 5000) &(df['Consumption'] < 200000)]
display("Commercial: ", com_df['Consumption'].describe())
pd.reset_option('^display.float_format', silent=True)
Cell 2
for call_type in res_df['Type'].unique():
display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
#display(res_ems.head())
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
display(g = sns.factorplot(x=call_type_name , y="Consumption", data=buckets, size=6, kind="bar", palette="muted"))
call_corr = pd.DataFrame()
for call_type in res_df['Type'].unique():
#display(call_type)
res_type = res_df[res_df['Type'] == call_type]
call_type_name = '{} Calls'.format(call_type)
buckets = res_type.groupby('Consumption Categories')['Type'].count().reset_index()
buckets.columns = ['Consumption', call_type_name]
#display(buckets)
if len(call_corr.axes[0]) == 0:
display("Starting from", call_type_name)
call_corr = buckets
#display(call_corr)
else:
display("Adding data for", call_type_name)
call_corr = pd.merge(call_corr, buckets, right_index=True, left_index=True)
#display(call_corr)
calls_df = call_corr[['Consumption', 'EMS Calls', 'FIRE Calls', 'ALM Calls', 'SVC Calls', 'HAZ Calls']]
#calls_df.columns = ['Consumption', 'EMS', 'FIRE', 'ALM', 'SVC', 'HAZ']
#display(calls_df)
calls_only_df = calls_df[['EMS Calls', 'FIRE Calls', 'ALM Calls', 'SVC Calls', 'HAZ Calls']]
calls_only_df.corr(method='spearman')
EMS Calls FIRE Calls ALM Calls SVC Calls HAZ Calls
EMS Calls 1.000000 0.848759 0.525009 0.716986 0.808898
FIRE Calls 0.848759 1.000000 0.696650 0.571377 0.820447
ALM Calls 0.525009 0.696650 1.000000 0.331825 0.443817
SVC Calls 0.716986 0.571377 0.331825 1.000000 0.699963
HAZ Calls 0.808898 0.820447 0.443817 0.699963 1.000000
<class 'pandas.core.frame.DataFrame'>
Int64Index: 33485 entries, 0 to 105505
Data columns (total 6 columns):
Date 33485 non-null datetime64[ns]
Type 33485 non-null category
Name 33485 non-null category
Address 33485 non-null category
Location 33485 non-null object
Consumption 33485 non-null float64
dtypes: category(3), datetime64ns, float64(1), object(1)