Skip to content

Instantly share code, notes, and snippets.

@Saurabh7

Saurabh7/evaluation3.ipynb

Created July 27, 2017 11:05
Show Gist options
  • Save Saurabh7/776063b2e7d26b41ea981bec8a3a5415 to your computer and use it in GitHub Desktop.
Save Saurabh7/776063b2e7d26b41ea981bec8a3a5415 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
evaluation3.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<script>jQuery(function() {if (jQuery(\"body.notebook_app\").length == 0) { jQuery(\".input_area\").toggle(); jQuery(\".prompt\").toggle();}});</script>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<button onclick=\"jQuery('.input_area').toggle(); jQuery('.prompt').toggle();\">Toggle code</button>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import IPython.core.display as di\n",
"\n",
"# This line will hide code by default when the notebook is exported as HTML\n",
"di.display_html('<script>jQuery(function() {if (jQuery(\"body.notebook_app\").length == 0) { jQuery(\".input_area\").toggle(); jQuery(\".prompt\").toggle();}});</script>', raw=True)\n",
"\n",
"# This line will add a button to toggle visibility of code blocks, for use with the HTML export version\n",
"di.display_html('''<button onclick=\"jQuery('.input_area').toggle(); jQuery('.prompt').toggle();\">Toggle code</button>''', raw=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Path analysis flow\n",
"\n",
"- Find check threshold.\n",
"- Find paths satisfy threshold.\n",
"- Filter according to impact factor."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Evaluation information:\n",
"imp_factor vs spend_prop\n",
"\n",
"- Evaluation metrics: precision, recall, MCC, f1 score\n",
"- check threshold"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from jarvis.mongo.mongo_paths import MongoPaths"
]
},
{
"cell_type": "code",
"execution_count": 61,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import display"
]
},
{
"cell_type": "code",
"execution_count": 75,
"metadata": {},
"outputs": [],
"source": [
"def evaluate(account_list, metric):\n",
"\n",
" document_list = list(MongoPaths._get_collection().find({'account_ids': account_list, 'impact_factor': 'v0', 'goal_metric': metric}))\n",
" df = pd.DataFrame(document_list)\n",
" df1 = df[['start_date', 'end_date', 'precision', 'recall', 'mcc', 'f1', 'check_threshold', 'true_spend_prop', 'goal_metric']][1:]\n",
"\n",
" document_list = list(MongoPaths._get_collection().find({'account_ids': account_list, 'impact_factor': 'spend_prop', 'goal_metric': metric}))\n",
" df = pd.DataFrame(document_list)\n",
" df2 = df[['start_date', 'end_date', 'precision', 'recall', 'mcc', 'f1', 'check_threshold', 'true_spend_prop', 'goal_metric']][1:]\n",
"\n",
" display((df1.merge(df2, on=['start_date', 'end_date'], how='inner', suffixes=('_imp_factor', '_spend_prop'))\n",
" [['start_date', 'end_date', 'precision_imp_factor', 'precision_spend_prop', 'recall_imp_factor',\n",
" 'recall_spend_prop','mcc_imp_factor', 'mcc_spend_prop', 'check_threshold_imp_factor', 'true_spend_prop_imp_factor', 'check_threshold_spend_prop', 'true_spend_prop_spend_prop']]))# 'true_spend_prop', 'goal_metric'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Zivame\n",
"#### GA: ROAS"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>start_date</th>\n",
" <th>end_date</th>\n",
" <th>precision_imp_factor</th>\n",
" <th>precision_spend_prop</th>\n",
" <th>recall_imp_factor</th>\n",
" <th>recall_spend_prop</th>\n",
" <th>mcc_imp_factor</th>\n",
" <th>mcc_spend_prop</th>\n",
" <th>check_threshold_imp_factor</th>\n",
" <th>true_spend_prop_imp_factor</th>\n",
" <th>check_threshold_spend_prop</th>\n",
" <th>true_spend_prop_spend_prop</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2017-07-26</td>\n",
" <td>2017-07-26</td>\n",
" <td>0.920111</td>\n",
" <td>0.538056</td>\n",
" <td>0.836282</td>\n",
" <td>0.796627</td>\n",
" <td>0.719417</td>\n",
" <td>-0.306508</td>\n",
" <td>0.179102</td>\n",
" <td>0.593846</td>\n",
" <td>0.179102</td>\n",
" <td>0.593846</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2017-07-21</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.832236</td>\n",
" <td>0.567648</td>\n",
" <td>0.650568</td>\n",
" <td>0.782327</td>\n",
" <td>0.489078</td>\n",
" <td>-0.004011</td>\n",
" <td>0.313907</td>\n",
" <td>0.554163</td>\n",
" <td>0.302588</td>\n",
" <td>0.568692</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2017-07-14</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.731919</td>\n",
" <td>0.720345</td>\n",
" <td>0.592037</td>\n",
" <td>0.684167</td>\n",
" <td>0.382595</td>\n",
" <td>0.427203</td>\n",
" <td>0.315738</td>\n",
" <td>0.499644</td>\n",
" <td>0.323203</td>\n",
" <td>0.492543</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2017-06-30</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.689149</td>\n",
" <td>0.672070</td>\n",
" <td>0.639975</td>\n",
" <td>0.643434</td>\n",
" <td>0.300730</td>\n",
" <td>0.272682</td>\n",
" <td>0.337541</td>\n",
" <td>0.539579</td>\n",
" <td>0.336658</td>\n",
" <td>0.540974</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2017-07-01</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.807772</td>\n",
" <td>0.674542</td>\n",
" <td>0.478670</td>\n",
" <td>0.691742</td>\n",
" <td>0.361224</td>\n",
" <td>0.283037</td>\n",
" <td>0.329181</td>\n",
" <td>0.548880</td>\n",
" <td>0.327564</td>\n",
" <td>0.551039</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-30</td>\n",
" <td>0.685342</td>\n",
" <td>0.649140</td>\n",
" <td>0.598908</td>\n",
" <td>0.607395</td>\n",
" <td>0.322593</td>\n",
" <td>0.275167</td>\n",
" <td>0.327601</td>\n",
" <td>0.503321</td>\n",
" <td>0.327601</td>\n",
" <td>0.503321</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>2017-05-01</td>\n",
" <td>2017-05-31</td>\n",
" <td>0.711754</td>\n",
" <td>0.682829</td>\n",
" <td>0.296060</td>\n",
" <td>0.353253</td>\n",
" <td>0.180245</td>\n",
" <td>0.172163</td>\n",
" <td>0.338024</td>\n",
" <td>0.546267</td>\n",
" <td>0.338024</td>\n",
" <td>0.546267</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-27</td>\n",
" <td>0.742584</td>\n",
" <td>0.649603</td>\n",
" <td>0.640397</td>\n",
" <td>0.646865</td>\n",
" <td>0.403185</td>\n",
" <td>0.270885</td>\n",
" <td>0.297782</td>\n",
" <td>0.518636</td>\n",
" <td>0.297782</td>\n",
" <td>0.518636</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>2017-07-17</td>\n",
" <td>2017-07-20</td>\n",
" <td>0.774678</td>\n",
" <td>0.749666</td>\n",
" <td>0.711644</td>\n",
" <td>0.848953</td>\n",
" <td>0.481790</td>\n",
" <td>0.543929</td>\n",
" <td>0.321044</td>\n",
" <td>0.525778</td>\n",
" <td>0.321044</td>\n",
" <td>0.525778</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" start_date end_date precision_imp_factor precision_spend_prop \\\n",
"0 2017-07-26 2017-07-26 0.920111 0.538056 \n",
"1 2017-07-21 2017-07-27 0.832236 0.567648 \n",
"2 2017-07-14 2017-07-27 0.731919 0.720345 \n",
"3 2017-06-30 2017-07-27 0.689149 0.672070 \n",
"4 2017-07-01 2017-07-27 0.807772 0.674542 \n",
"5 2017-06-01 2017-06-30 0.685342 0.649140 \n",
"6 2017-05-01 2017-05-31 0.711754 0.682829 \n",
"7 2017-06-01 2017-06-27 0.742584 0.649603 \n",
"8 2017-07-17 2017-07-20 0.774678 0.749666 \n",
"\n",
" recall_imp_factor recall_spend_prop mcc_imp_factor mcc_spend_prop \\\n",
"0 0.836282 0.796627 0.719417 -0.306508 \n",
"1 0.650568 0.782327 0.489078 -0.004011 \n",
"2 0.592037 0.684167 0.382595 0.427203 \n",
"3 0.639975 0.643434 0.300730 0.272682 \n",
"4 0.478670 0.691742 0.361224 0.283037 \n",
"5 0.598908 0.607395 0.322593 0.275167 \n",
"6 0.296060 0.353253 0.180245 0.172163 \n",
"7 0.640397 0.646865 0.403185 0.270885 \n",
"8 0.711644 0.848953 0.481790 0.543929 \n",
"\n",
" check_threshold_imp_factor true_spend_prop_imp_factor \\\n",
"0 0.179102 0.593846 \n",
"1 0.313907 0.554163 \n",
"2 0.315738 0.499644 \n",
"3 0.337541 0.539579 \n",
"4 0.329181 0.548880 \n",
"5 0.327601 0.503321 \n",
"6 0.338024 0.546267 \n",
"7 0.297782 0.518636 \n",
"8 0.321044 0.525778 \n",
"\n",
" check_threshold_spend_prop true_spend_prop_spend_prop \n",
"0 0.179102 0.593846 \n",
"1 0.302588 0.568692 \n",
"2 0.323203 0.492543 \n",
"3 0.336658 0.540974 \n",
"4 0.327564 0.551039 \n",
"5 0.327601 0.503321 \n",
"6 0.338024 0.546267 \n",
"7 0.297782 0.518636 \n",
"8 0.321044 0.525778 "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"evaluate(account_list = [15],\n",
" metric = 'ga_fb_roas')"
]
},
{
"cell_type": "code",
"execution_count": 77,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>start_date</th>\n",
" <th>end_date</th>\n",
" <th>precision_imp_factor</th>\n",
" <th>precision_spend_prop</th>\n",
" <th>recall_imp_factor</th>\n",
" <th>recall_spend_prop</th>\n",
" <th>mcc_imp_factor</th>\n",
" <th>mcc_spend_prop</th>\n",
" <th>check_threshold_imp_factor</th>\n",
" <th>true_spend_prop_imp_factor</th>\n",
" <th>check_threshold_spend_prop</th>\n",
" <th>true_spend_prop_spend_prop</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2017-07-26</td>\n",
" <td>2017-07-26</td>\n",
" <td>0.934384</td>\n",
" <td>0.959010</td>\n",
" <td>0.513679</td>\n",
" <td>1.000000</td>\n",
" <td>-0.131164</td>\n",
" <td>3.143130e-08</td>\n",
" <td>0.044739</td>\n",
" <td>0.959010</td>\n",
" <td>0.044739</td>\n",
" <td>0.959010</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2017-07-21</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.822489</td>\n",
" <td>0.613825</td>\n",
" <td>0.627749</td>\n",
" <td>0.826384</td>\n",
" <td>0.476278</td>\n",
" <td>1.664679e-01</td>\n",
" <td>0.250274</td>\n",
" <td>0.538019</td>\n",
" <td>0.237254</td>\n",
" <td>0.568101</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2017-07-14</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.763042</td>\n",
" <td>0.583053</td>\n",
" <td>0.504785</td>\n",
" <td>0.843198</td>\n",
" <td>0.337117</td>\n",
" <td>1.816185e-01</td>\n",
" <td>0.246700</td>\n",
" <td>0.536349</td>\n",
" <td>0.247414</td>\n",
" <td>0.533815</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2017-06-30</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.756774</td>\n",
" <td>0.635208</td>\n",
" <td>0.435135</td>\n",
" <td>0.695695</td>\n",
" <td>0.284500</td>\n",
" <td>2.179553e-01</td>\n",
" <td>0.272759</td>\n",
" <td>0.548027</td>\n",
" <td>0.274035</td>\n",
" <td>0.546300</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2017-07-01</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.768996</td>\n",
" <td>0.631995</td>\n",
" <td>0.563390</td>\n",
" <td>0.733805</td>\n",
" <td>0.370062</td>\n",
" <td>2.385150e-01</td>\n",
" <td>0.260981</td>\n",
" <td>0.541681</td>\n",
" <td>0.262346</td>\n",
" <td>0.540402</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-30</td>\n",
" <td>0.842640</td>\n",
" <td>0.658718</td>\n",
" <td>0.451960</td>\n",
" <td>0.660409</td>\n",
" <td>0.335368</td>\n",
" <td>1.324408e-01</td>\n",
" <td>0.264378</td>\n",
" <td>0.606814</td>\n",
" <td>0.264378</td>\n",
" <td>0.606814</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>2017-05-01</td>\n",
" <td>2017-05-31</td>\n",
" <td>0.726499</td>\n",
" <td>0.640488</td>\n",
" <td>0.287635</td>\n",
" <td>0.453172</td>\n",
" <td>0.157603</td>\n",
" <td>9.452705e-02</td>\n",
" <td>0.298306</td>\n",
" <td>0.585073</td>\n",
" <td>0.298306</td>\n",
" <td>0.585073</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-27</td>\n",
" <td>0.816983</td>\n",
" <td>0.655254</td>\n",
" <td>0.488567</td>\n",
" <td>0.643273</td>\n",
" <td>0.354522</td>\n",
" <td>1.851655e-01</td>\n",
" <td>0.276323</td>\n",
" <td>0.574820</td>\n",
" <td>0.276323</td>\n",
" <td>0.574820</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>2017-07-17</td>\n",
" <td>2017-07-20</td>\n",
" <td>0.708916</td>\n",
" <td>0.700727</td>\n",
" <td>0.798946</td>\n",
" <td>0.976300</td>\n",
" <td>0.417296</td>\n",
" <td>5.572679e-01</td>\n",
" <td>0.181681</td>\n",
" <td>0.543665</td>\n",
" <td>0.181681</td>\n",
" <td>0.543665</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" start_date end_date precision_imp_factor precision_spend_prop \\\n",
"0 2017-07-26 2017-07-26 0.934384 0.959010 \n",
"1 2017-07-21 2017-07-27 0.822489 0.613825 \n",
"2 2017-07-14 2017-07-27 0.763042 0.583053 \n",
"3 2017-06-30 2017-07-27 0.756774 0.635208 \n",
"4 2017-07-01 2017-07-27 0.768996 0.631995 \n",
"5 2017-06-01 2017-06-30 0.842640 0.658718 \n",
"6 2017-05-01 2017-05-31 0.726499 0.640488 \n",
"7 2017-06-01 2017-06-27 0.816983 0.655254 \n",
"8 2017-07-17 2017-07-20 0.708916 0.700727 \n",
"\n",
" recall_imp_factor recall_spend_prop mcc_imp_factor mcc_spend_prop \\\n",
"0 0.513679 1.000000 -0.131164 3.143130e-08 \n",
"1 0.627749 0.826384 0.476278 1.664679e-01 \n",
"2 0.504785 0.843198 0.337117 1.816185e-01 \n",
"3 0.435135 0.695695 0.284500 2.179553e-01 \n",
"4 0.563390 0.733805 0.370062 2.385150e-01 \n",
"5 0.451960 0.660409 0.335368 1.324408e-01 \n",
"6 0.287635 0.453172 0.157603 9.452705e-02 \n",
"7 0.488567 0.643273 0.354522 1.851655e-01 \n",
"8 0.798946 0.976300 0.417296 5.572679e-01 \n",
"\n",
" check_threshold_imp_factor true_spend_prop_imp_factor \\\n",
"0 0.044739 0.959010 \n",
"1 0.250274 0.538019 \n",
"2 0.246700 0.536349 \n",
"3 0.272759 0.548027 \n",
"4 0.260981 0.541681 \n",
"5 0.264378 0.606814 \n",
"6 0.298306 0.585073 \n",
"7 0.276323 0.574820 \n",
"8 0.181681 0.543665 \n",
"\n",
" check_threshold_spend_prop true_spend_prop_spend_prop \n",
"0 0.044739 0.959010 \n",
"1 0.237254 0.568101 \n",
"2 0.247414 0.533815 \n",
"3 0.274035 0.546300 \n",
"4 0.262346 0.540402 \n",
"5 0.264378 0.606814 \n",
"6 0.298306 0.585073 \n",
"7 0.276323 0.574820 \n",
"8 0.181681 0.543665 "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"evaluate(account_list = [15],\n",
" metric = 'ga_fb_cpt')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Faballey\n",
"#### GA: ROAS"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>start_date</th>\n",
" <th>end_date</th>\n",
" <th>precision_imp_factor</th>\n",
" <th>precision_spend_prop</th>\n",
" <th>recall_imp_factor</th>\n",
" <th>recall_spend_prop</th>\n",
" <th>mcc_imp_factor</th>\n",
" <th>mcc_spend_prop</th>\n",
" <th>check_threshold_imp_factor</th>\n",
" <th>true_spend_prop_imp_factor</th>\n",
" <th>check_threshold_spend_prop</th>\n",
" <th>true_spend_prop_spend_prop</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2017-07-26</td>\n",
" <td>2017-07-26</td>\n",
" <td>0.886358</td>\n",
" <td>0.696606</td>\n",
" <td>0.840305</td>\n",
" <td>0.906962</td>\n",
" <td>0.679905</td>\n",
" <td>0.397515</td>\n",
" <td>0.382831</td>\n",
" <td>0.588588</td>\n",
" <td>0.382831</td>\n",
" <td>0.588588</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2017-07-21</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.883327</td>\n",
" <td>0.746221</td>\n",
" <td>0.563081</td>\n",
" <td>0.736350</td>\n",
" <td>0.540084</td>\n",
" <td>0.511474</td>\n",
" <td>0.312959</td>\n",
" <td>0.472798</td>\n",
" <td>0.311917</td>\n",
" <td>0.473547</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2017-07-14</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.683056</td>\n",
" <td>0.648201</td>\n",
" <td>0.507114</td>\n",
" <td>0.733423</td>\n",
" <td>0.352455</td>\n",
" <td>0.431926</td>\n",
" <td>0.358513</td>\n",
" <td>0.427252</td>\n",
" <td>0.359253</td>\n",
" <td>0.427392</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2017-06-30</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.678968</td>\n",
" <td>0.647034</td>\n",
" <td>0.300314</td>\n",
" <td>0.433352</td>\n",
" <td>0.158040</td>\n",
" <td>0.163520</td>\n",
" <td>0.430056</td>\n",
" <td>0.537963</td>\n",
" <td>0.425013</td>\n",
" <td>0.538500</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2017-07-01</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.708748</td>\n",
" <td>0.648078</td>\n",
" <td>0.389634</td>\n",
" <td>0.490755</td>\n",
" <td>0.277763</td>\n",
" <td>0.262222</td>\n",
" <td>0.427192</td>\n",
" <td>0.475617</td>\n",
" <td>0.432358</td>\n",
" <td>0.472762</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-30</td>\n",
" <td>0.690650</td>\n",
" <td>0.687657</td>\n",
" <td>0.420023</td>\n",
" <td>0.494882</td>\n",
" <td>0.216280</td>\n",
" <td>0.241684</td>\n",
" <td>0.462156</td>\n",
" <td>0.535412</td>\n",
" <td>0.462156</td>\n",
" <td>0.535412</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>2017-05-01</td>\n",
" <td>2017-05-31</td>\n",
" <td>0.737476</td>\n",
" <td>0.690923</td>\n",
" <td>0.551959</td>\n",
" <td>0.606790</td>\n",
" <td>0.334329</td>\n",
" <td>0.297492</td>\n",
" <td>0.401100</td>\n",
" <td>0.532585</td>\n",
" <td>0.401100</td>\n",
" <td>0.532585</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-27</td>\n",
" <td>0.663437</td>\n",
" <td>0.657253</td>\n",
" <td>0.512935</td>\n",
" <td>0.568539</td>\n",
" <td>0.223687</td>\n",
" <td>0.235602</td>\n",
" <td>0.443338</td>\n",
" <td>0.529284</td>\n",
" <td>0.443338</td>\n",
" <td>0.529284</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>2017-07-17</td>\n",
" <td>2017-07-20</td>\n",
" <td>0.845847</td>\n",
" <td>0.748979</td>\n",
" <td>0.549655</td>\n",
" <td>0.866532</td>\n",
" <td>0.490922</td>\n",
" <td>0.598776</td>\n",
" <td>0.443231</td>\n",
" <td>0.483281</td>\n",
" <td>0.443231</td>\n",
" <td>0.483281</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" start_date end_date precision_imp_factor precision_spend_prop \\\n",
"0 2017-07-26 2017-07-26 0.886358 0.696606 \n",
"1 2017-07-21 2017-07-27 0.883327 0.746221 \n",
"2 2017-07-14 2017-07-27 0.683056 0.648201 \n",
"3 2017-06-30 2017-07-27 0.678968 0.647034 \n",
"4 2017-07-01 2017-07-27 0.708748 0.648078 \n",
"5 2017-06-01 2017-06-30 0.690650 0.687657 \n",
"6 2017-05-01 2017-05-31 0.737476 0.690923 \n",
"7 2017-06-01 2017-06-27 0.663437 0.657253 \n",
"8 2017-07-17 2017-07-20 0.845847 0.748979 \n",
"\n",
" recall_imp_factor recall_spend_prop mcc_imp_factor mcc_spend_prop \\\n",
"0 0.840305 0.906962 0.679905 0.397515 \n",
"1 0.563081 0.736350 0.540084 0.511474 \n",
"2 0.507114 0.733423 0.352455 0.431926 \n",
"3 0.300314 0.433352 0.158040 0.163520 \n",
"4 0.389634 0.490755 0.277763 0.262222 \n",
"5 0.420023 0.494882 0.216280 0.241684 \n",
"6 0.551959 0.606790 0.334329 0.297492 \n",
"7 0.512935 0.568539 0.223687 0.235602 \n",
"8 0.549655 0.866532 0.490922 0.598776 \n",
"\n",
" check_threshold_imp_factor true_spend_prop_imp_factor \\\n",
"0 0.382831 0.588588 \n",
"1 0.312959 0.472798 \n",
"2 0.358513 0.427252 \n",
"3 0.430056 0.537963 \n",
"4 0.427192 0.475617 \n",
"5 0.462156 0.535412 \n",
"6 0.401100 0.532585 \n",
"7 0.443338 0.529284 \n",
"8 0.443231 0.483281 \n",
"\n",
" check_threshold_spend_prop true_spend_prop_spend_prop \n",
"0 0.382831 0.588588 \n",
"1 0.311917 0.473547 \n",
"2 0.359253 0.427392 \n",
"3 0.425013 0.538500 \n",
"4 0.432358 0.472762 \n",
"5 0.462156 0.535412 \n",
"6 0.401100 0.532585 \n",
"7 0.443338 0.529284 \n",
"8 0.443231 0.483281 "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"evaluate(account_list = [261, 260],\n",
" metric = 'ga_fb_roas')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Zivame\n",
"#### FB: CTR"
]
},
{
"cell_type": "code",
"execution_count": 78,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>start_date</th>\n",
" <th>end_date</th>\n",
" <th>precision_imp_factor</th>\n",
" <th>precision_spend_prop</th>\n",
" <th>recall_imp_factor</th>\n",
" <th>recall_spend_prop</th>\n",
" <th>mcc_imp_factor</th>\n",
" <th>mcc_spend_prop</th>\n",
" <th>check_threshold_imp_factor</th>\n",
" <th>true_spend_prop_imp_factor</th>\n",
" <th>check_threshold_spend_prop</th>\n",
" <th>true_spend_prop_spend_prop</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2017-07-26</td>\n",
" <td>2017-07-26</td>\n",
" <td>0.860961</td>\n",
" <td>0.746381</td>\n",
" <td>0.810956</td>\n",
" <td>0.912654</td>\n",
" <td>0.573538</td>\n",
" <td>4.330212e-01</td>\n",
" <td>0.247640</td>\n",
" <td>0.634121</td>\n",
" <td>0.247640</td>\n",
" <td>0.634121</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2017-06-30</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.857280</td>\n",
" <td>0.603410</td>\n",
" <td>0.571517</td>\n",
" <td>1.000000</td>\n",
" <td>0.425693</td>\n",
" <td>3.868027e-08</td>\n",
" <td>0.237161</td>\n",
" <td>0.603459</td>\n",
" <td>0.237278</td>\n",
" <td>0.603410</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2017-07-01</td>\n",
" <td>2017-07-27</td>\n",
" <td>0.859320</td>\n",
" <td>0.616169</td>\n",
" <td>0.559470</td>\n",
" <td>1.000000</td>\n",
" <td>0.409155</td>\n",
" <td>3.490394e-08</td>\n",
" <td>0.232438</td>\n",
" <td>0.616245</td>\n",
" <td>0.232556</td>\n",
" <td>0.616169</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-30</td>\n",
" <td>0.775743</td>\n",
" <td>0.668102</td>\n",
" <td>0.443005</td>\n",
" <td>0.704796</td>\n",
" <td>0.324948</td>\n",
" <td>3.126413e-01</td>\n",
" <td>0.296685</td>\n",
" <td>0.529328</td>\n",
" <td>0.296685</td>\n",
" <td>0.529328</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2017-05-01</td>\n",
" <td>2017-05-31</td>\n",
" <td>0.838058</td>\n",
" <td>0.698344</td>\n",
" <td>0.502936</td>\n",
" <td>0.748914</td>\n",
" <td>0.361298</td>\n",
" <td>2.625473e-01</td>\n",
" <td>0.296641</td>\n",
" <td>0.603529</td>\n",
" <td>0.296641</td>\n",
" <td>0.603529</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>2017-06-01</td>\n",
" <td>2017-06-27</td>\n",
" <td>0.763700</td>\n",
" <td>0.659158</td>\n",
" <td>0.431089</td>\n",
" <td>0.714531</td>\n",
" <td>0.302683</td>\n",
" <td>2.945511e-01</td>\n",
" <td>0.297897</td>\n",
" <td>0.533757</td>\n",
" <td>0.297897</td>\n",
" <td>0.533757</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" start_date end_date precision_imp_factor precision_spend_prop \\\n",
"0 2017-07-26 2017-07-26 0.860961 0.746381 \n",
"1 2017-06-30 2017-07-27 0.857280 0.603410 \n",
"2 2017-07-01 2017-07-27 0.859320 0.616169 \n",
"3 2017-06-01 2017-06-30 0.775743 0.668102 \n",
"4 2017-05-01 2017-05-31 0.838058 0.698344 \n",
"5 2017-06-01 2017-06-27 0.763700 0.659158 \n",
"\n",
" recall_imp_factor recall_spend_prop mcc_imp_factor mcc_spend_prop \\\n",
"0 0.810956 0.912654 0.573538 4.330212e-01 \n",
"1 0.571517 1.000000 0.425693 3.868027e-08 \n",
"2 0.559470 1.000000 0.409155 3.490394e-08 \n",
"3 0.443005 0.704796 0.324948 3.126413e-01 \n",
"4 0.502936 0.748914 0.361298 2.625473e-01 \n",
"5 0.431089 0.714531 0.302683 2.945511e-01 \n",
"\n",
" check_threshold_imp_factor true_spend_prop_imp_factor \\\n",
"0 0.247640 0.634121 \n",
"1 0.237161 0.603459 \n",
"2 0.232438 0.616245 \n",
"3 0.296685 0.529328 \n",
"4 0.296641 0.603529 \n",
"5 0.297897 0.533757 \n",
"\n",
" check_threshold_spend_prop true_spend_prop_spend_prop \n",
"0 0.247640 0.634121 \n",
"1 0.237278 0.603410 \n",
"2 0.232556 0.616169 \n",
"3 0.296685 0.529328 \n",
"4 0.296641 0.603529 \n",
"5 0.297897 0.533757 "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"evaluate(account_list = [15],\n",
" metric = 'fb_ctr')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"account_list = [15]\n",
"metric = 'ga_fb_cpt'\n",
"imp_factor = 'v0'\n",
"document_list = list(MongoPaths._get_collection().find({'account_ids': account_list, 'impact_factor':imp_factor, 'goal_metric': metric}))\n",
"df = pd.DataFrame(document_list)\n",
"df[['start_date', 'end_date', 'precision', 'recall', 'mcc', 'f1', 'check_threshold', 'true_spend_prop']]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"account_list = [15]\n",
"metric = 'ga_fb_cpt'\n",
"imp_factor = 'spend_prop'\n",
"document_list = list(MongoPaths._get_collection().find({'account_ids': account_list, 'impact_factor':imp_factor, 'goal_metric': metric}))\n",
"df = pd.DataFrame(document_list)\n",
"df[['start_date', 'end_date', 'precision', 'recall', 'mcc', 'f1', 'check_threshold', 'true_spend_prop']]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"document_list = list(MongoPaths._get_collection().find({'account_ids': account_list, 'impact_factor': 'spend_prop', 'goal_metric': metric}))\n",
"pd.DataFrame(document_list)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def print_scores(account_list, metric):\n",
" print('\\n imp_factor:')\n",
" document_list = list(MongoPaths._get_collection().find({'account_ids': account_list, 'impact_factor': 'v0', 'goal_metric': metric}))\n",
"\n",
" print('precision:', pd.np.mean([doc['precision'] for doc in document_list]))\n",
" print('recall:',pd.np.mean([doc['recall'] for doc in document_list]))\n",
" print('f1:',pd.np.mean([doc['f1'] for doc in document_list]))\n",
" print('mcc:',pd.np.mean([doc['mcc'] for doc in document_list]))\n",
"\n",
" print('\\n Spend prop:')\n",
" document_list = list(MongoPaths._get_collection().find({'account_ids': account_list, 'impact_factor': 'spend_prop', 'goal_metric': metric}))\n",
"\n",
" print('precision:', pd.np.mean([doc['precision'] for doc in document_list]))\n",
" print('recall:',pd.np.mean([doc['recall'] for doc in document_list]))\n",
" print('f1:',pd.np.mean([doc['f1'] for doc in document_list]))\n",
" print('mcc:',pd.np.mean([doc['mcc'] for doc in document_list]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Cars24\")\n",
"print('ga_fb_cpt')\n",
"print_scores([136], 'ga_fb_cpt')\n",
"\n",
"print('-'*10)\n",
"print(\"Zivame\")\n",
"print('ga_fb_cpt')\n",
"print_scores([15], 'ga_fb_cpt')\n",
"print('ga_fb_roas')\n",
"print_scores([15], 'ga_fb_roas')\n",
"\n",
"print('-'*10)\n",
"print(\"Faballey\")\n",
"print('ga_fb_cpt')\n",
"print_scores([261, 260], 'ga_fb_cpt')\n",
"print('ga_fb_roas')\n",
"print_scores([261, 260], 'ga_fb_roas')\n",
"\n"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [jarvis1]",
"language": "python",
"name": "Python [jarvis1]"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment