Skip to content

Instantly share code, notes, and snippets.

@yiyuezhuo

yiyuezhuo/model_ensemble.ipynb

Created February 6, 2020 03:58
Show Gist options
  • Save yiyuezhuo/29d227880473183a090dc77022706fe8 to your computer and use it in GitHub Desktop.
Save yiyuezhuo/29d227880473183a090dc77022706fe8 to your computer and use it in GitHub Desktop.
Download ZIP
video ensemble result
Display the source blob
Display the rendered blob
Raw
model_ensemble.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append('/home/yiyuezhuo/windows/e/yiyuezhuo4/video_yyz')\n",
"import video_yyz\n",
"from os import environ\n",
"environ['SVD'] = '/home/yiyuezhuo/windows/e/yiyuezhuo4/steel_video_dataset'"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"notebook_utils 0.1\n"
]
}
],
"source": [
"from video_yyz.notebook_utils import EvalSuit, DisplaySuit"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from video_yyz.tools import SVD"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"root = str(SVD / 'video_sample_free')\n",
"device = 'cuda'"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"from collections import OrderedDict"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"suit_map = OrderedDict()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/5 [00:00<?, ?it/s]/home/yiyuezhuo/anaconda3/lib/python3.7/site-packages/torchvision/io/video.py:106: UserWarning: The pts_unit 'pts' gives wrong results and will be removed in a follow-up version. Please use pts_unit 'sec'.\n",
" warnings.warn(\"The pts_unit 'pts' gives wrong results and will be removed in a \" +\n",
"100%|██████████| 5/5 [00:01<00:00, 2.87it/s]\n",
"100%|██████████| 1520/1520 [00:28<00:00, 53.39it/s]\n"
]
}
],
"source": [
"suit_map['resnet18_word_bag'] = EvalSuit(\n",
" root=root,\n",
" model_name='resnet18_word_bag',\n",
" dataset_config=['test_video_dataset_1_rgb', 'transform_test_1', 'val2vl', 'video_uniform_2'],\n",
" checkpoint_path='/home/yiyuezhuo/windows/e/yiyuezhuo4/video_yyz_output/test_word_bag_1.pth',\n",
" device=device\n",
")\n",
"suit_map['resnet18_word_bag'].do_work()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 5/5 [00:01<00:00, 2.89it/s]\n",
"100%|██████████| 1520/1520 [01:27<00:00, 17.40it/s]\n"
]
}
],
"source": [
"suit_map['cnn_lstm_1'] = EvalSuit(\n",
" root=root,\n",
" model_name='cnn_lstm_1',\n",
" dataset_config=['test_video_dataset_1', 'transform_test_1', 'val2vl', 'video_uniform_2'],\n",
" checkpoint_path='/home/yiyuezhuo/windows/e/yiyuezhuo4/video_yyz_output/test_cnn_lstm_1.pth',\n",
" device=device\n",
")\n",
"suit_map['cnn_lstm_1'].do_work()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 5/5 [00:01<00:00, 2.92it/s]\n",
"100%|██████████| 1520/1520 [01:54<00:00, 13.22it/s]\n"
]
}
],
"source": [
"suit_map['r2plus1d_18_1'] = EvalSuit(\n",
" root=root,\n",
" model_name='r2plus1d_18_1',\n",
" dataset_config=['test_video_dataset_1', 'transform_test_1', 'val2vl', 'video_uniform_2'],\n",
" checkpoint_path='/home/yiyuezhuo/windows/e/yiyuezhuo4/video_yyz_output/test2.pth',\n",
" device=device\n",
")\n",
"suit_map['r2plus1d_18_1'].do_work()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 5/5 [00:01<00:00, 2.85it/s]\n",
"100%|██████████| 1520/1520 [01:15<00:00, 20.05it/s]\n"
]
}
],
"source": [
"suit_map['resnet18_flat_L5'] = EvalSuit(\n",
" root=root,\n",
" model_name='resnet18_flat_L5',\n",
" dataset_config=['test_video_dataset_1_L5', 'transform_test_optical_flow', 'val2vl', 'video_uniform_2'],\n",
" checkpoint_path='/home/yiyuezhuo/windows/e/yiyuezhuo4/video_yyz_output/test_optical_1_resume_3.pth',\n",
" device=device\n",
")\n",
"suit_map['resnet18_flat_L5'].do_work()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"acc (frame): 87.10526315789474\n",
"acc: 0.9078947368421053\n",
"confuse matrix:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>35</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>6</td>\n",
" <td>34</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred 1 2\n",
"target \n",
"0 0 1\n",
"1 35 0\n",
"2 6 34"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wrong example show:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>pred</th>\n",
" <th>target</th>\n",
" </tr>\n",
" <tr>\n",
" <th>video_index</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>17</td>\n",
" <td>1.015335e-06</td>\n",
" <td>0.656977</td>\n",
" <td>0.343022</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>21</td>\n",
" <td>2.391515e-06</td>\n",
" <td>0.550014</td>\n",
" <td>0.449983</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>32</td>\n",
" <td>2.241563e-06</td>\n",
" <td>0.556264</td>\n",
" <td>0.443733</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>35</td>\n",
" <td>8.607343e-03</td>\n",
" <td>0.200767</td>\n",
" <td>0.790626</td>\n",
" <td>2</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>38</td>\n",
" <td>4.572751e-07</td>\n",
" <td>0.605815</td>\n",
" <td>0.394184</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>52</td>\n",
" <td>1.240059e-04</td>\n",
" <td>0.731009</td>\n",
" <td>0.268867</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>54</td>\n",
" <td>8.074607e-07</td>\n",
" <td>0.572096</td>\n",
" <td>0.427903</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 pred target\n",
"video_index \n",
"17 1.015335e-06 0.656977 0.343022 1 2\n",
"21 2.391515e-06 0.550014 0.449983 1 2\n",
"32 2.241563e-06 0.556264 0.443733 1 2\n",
"35 8.607343e-03 0.200767 0.790626 2 0\n",
"38 4.572751e-07 0.605815 0.394184 1 2\n",
"52 1.240059e-04 0.731009 0.268867 1 2\n",
"54 8.074607e-07 0.572096 0.427903 1 2"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adjustment threshold=0.7\n",
"acc (adjusted): 0.8421052631578947\n",
"confuse matrix (adjusted):\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred_adjusted</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>35</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>11</td>\n",
" <td>29</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred_adjusted 1 2\n",
"target \n",
"0 0 1\n",
"1 35 0\n",
"2 11 29"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"suit_map['resnet18_word_bag'].display()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"acc (frame): 84.32894736842105\n",
"acc: 0.868421052631579\n",
"confuse matrix:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred</th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>30</td>\n",
" <td>5</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>3</td>\n",
" <td>36</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred 0 1 2\n",
"target \n",
"0 0 1 0\n",
"1 0 30 5\n",
"2 1 3 36"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wrong example show:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>pred</th>\n",
" <th>target</th>\n",
" </tr>\n",
" <tr>\n",
" <th>video_index</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>0.000101</td>\n",
" <td>0.361241</td>\n",
" <td>0.638657</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>16</td>\n",
" <td>0.000058</td>\n",
" <td>0.136170</td>\n",
" <td>0.863773</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>17</td>\n",
" <td>0.000163</td>\n",
" <td>0.693739</td>\n",
" <td>0.306098</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>19</td>\n",
" <td>0.000288</td>\n",
" <td>0.242501</td>\n",
" <td>0.757211</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>21</td>\n",
" <td>0.003507</td>\n",
" <td>0.538034</td>\n",
" <td>0.458458</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>35</td>\n",
" <td>0.339336</td>\n",
" <td>0.512140</td>\n",
" <td>0.148524</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>50</td>\n",
" <td>0.474040</td>\n",
" <td>0.108750</td>\n",
" <td>0.417210</td>\n",
" <td>0</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>52</td>\n",
" <td>0.153318</td>\n",
" <td>0.494772</td>\n",
" <td>0.351910</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>64</td>\n",
" <td>0.000389</td>\n",
" <td>0.026984</td>\n",
" <td>0.972627</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>71</td>\n",
" <td>0.000443</td>\n",
" <td>0.142230</td>\n",
" <td>0.857327</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 pred target\n",
"video_index \n",
"9 0.000101 0.361241 0.638657 2 1\n",
"16 0.000058 0.136170 0.863773 2 1\n",
"17 0.000163 0.693739 0.306098 1 2\n",
"19 0.000288 0.242501 0.757211 2 1\n",
"21 0.003507 0.538034 0.458458 1 2\n",
"35 0.339336 0.512140 0.148524 1 0\n",
"50 0.474040 0.108750 0.417210 0 2\n",
"52 0.153318 0.494772 0.351910 1 2\n",
"64 0.000389 0.026984 0.972627 2 1\n",
"71 0.000443 0.142230 0.857327 2 1"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adjustment threshold=0.7\n",
"acc (adjusted): 0.8289473684210527\n",
"confuse matrix (adjusted):\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred_adjusted</th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>31</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>2</td>\n",
" <td>6</td>\n",
" <td>32</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred_adjusted 0 1 2\n",
"target \n",
"0 0 1 0\n",
"1 0 31 4\n",
"2 2 6 32"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"suit_map['cnn_lstm_1'].display()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"acc (frame): 84.53947368421052\n",
"acc: 0.8947368421052632\n",
"confuse matrix:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>29</td>\n",
" <td>6</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>39</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred 1 2\n",
"target \n",
"0 0 1\n",
"1 29 6\n",
"2 1 39"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wrong example show:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>pred</th>\n",
" <th>target</th>\n",
" </tr>\n",
" <tr>\n",
" <th>video_index</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>0.000003</td>\n",
" <td>0.338940</td>\n",
" <td>0.661057</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>0.000008</td>\n",
" <td>0.027733</td>\n",
" <td>0.972259</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>17</td>\n",
" <td>0.000050</td>\n",
" <td>0.912524</td>\n",
" <td>0.087427</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>19</td>\n",
" <td>0.000017</td>\n",
" <td>0.025124</td>\n",
" <td>0.974859</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>35</td>\n",
" <td>0.102665</td>\n",
" <td>0.119582</td>\n",
" <td>0.777753</td>\n",
" <td>2</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>49</td>\n",
" <td>0.000083</td>\n",
" <td>0.265351</td>\n",
" <td>0.734566</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>64</td>\n",
" <td>0.000015</td>\n",
" <td>0.064886</td>\n",
" <td>0.935099</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>71</td>\n",
" <td>0.000080</td>\n",
" <td>0.003894</td>\n",
" <td>0.996027</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 pred target\n",
"video_index \n",
"8 0.000003 0.338940 0.661057 2 1\n",
"9 0.000008 0.027733 0.972259 2 1\n",
"17 0.000050 0.912524 0.087427 1 2\n",
"19 0.000017 0.025124 0.974859 2 1\n",
"35 0.102665 0.119582 0.777753 2 0\n",
"49 0.000083 0.265351 0.734566 2 1\n",
"64 0.000015 0.064886 0.935099 2 1\n",
"71 0.000080 0.003894 0.996027 2 1"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adjustment threshold=0.7\n",
"acc (adjusted): 0.881578947368421\n",
"confuse matrix (adjusted):\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred_adjusted</th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>30</td>\n",
" <td>5</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" <td>37</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred_adjusted 0 1 2\n",
"target \n",
"0 0 0 1\n",
"1 0 30 5\n",
"2 1 2 37"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"suit_map['r2plus1d_18_1'].display()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"acc (frame): 74.82894736842105\n",
"acc: 0.8552631578947368\n",
"confuse matrix:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>26</td>\n",
" <td>9</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>39</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred 1 2\n",
"target \n",
"0 0 1\n",
"1 26 9\n",
"2 1 39"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wrong example show:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>pred</th>\n",
" <th>target</th>\n",
" </tr>\n",
" <tr>\n",
" <th>video_index</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>0.037660</td>\n",
" <td>0.416357</td>\n",
" <td>0.545983</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>19</td>\n",
" <td>0.000279</td>\n",
" <td>0.302443</td>\n",
" <td>0.697278</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>22</td>\n",
" <td>0.081994</td>\n",
" <td>0.306770</td>\n",
" <td>0.611237</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>35</td>\n",
" <td>0.151521</td>\n",
" <td>0.124169</td>\n",
" <td>0.724310</td>\n",
" <td>2</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>45</td>\n",
" <td>0.070737</td>\n",
" <td>0.289060</td>\n",
" <td>0.640203</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>49</td>\n",
" <td>0.084995</td>\n",
" <td>0.271739</td>\n",
" <td>0.643265</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>55</td>\n",
" <td>0.000254</td>\n",
" <td>0.667907</td>\n",
" <td>0.331839</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>60</td>\n",
" <td>0.107757</td>\n",
" <td>0.299451</td>\n",
" <td>0.592792</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>62</td>\n",
" <td>0.000546</td>\n",
" <td>0.468990</td>\n",
" <td>0.530464</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>64</td>\n",
" <td>0.000932</td>\n",
" <td>0.201405</td>\n",
" <td>0.797663</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>71</td>\n",
" <td>0.001247</td>\n",
" <td>0.391394</td>\n",
" <td>0.607358</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 pred target\n",
"video_index \n",
"8 0.037660 0.416357 0.545983 2 1\n",
"19 0.000279 0.302443 0.697278 2 1\n",
"22 0.081994 0.306770 0.611237 2 1\n",
"35 0.151521 0.124169 0.724310 2 0\n",
"45 0.070737 0.289060 0.640203 2 1\n",
"49 0.084995 0.271739 0.643265 2 1\n",
"55 0.000254 0.667907 0.331839 1 2\n",
"60 0.107757 0.299451 0.592792 2 1\n",
"62 0.000546 0.468990 0.530464 2 1\n",
"64 0.000932 0.201405 0.797663 2 1\n",
"71 0.001247 0.391394 0.607358 2 1"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adjustment threshold=0.7\n",
"acc (adjusted): 0.7763157894736842\n",
"confuse matrix (adjusted):\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred_adjusted</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>34</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>15</td>\n",
" <td>25</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred_adjusted 1 2\n",
"target \n",
"0 0 1\n",
"1 34 1\n",
"2 15 25"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"suit_map['resnet18_flat_L5'].display()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"suit = suit_map['resnet18_flat_L5']"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>video_index</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0.151749</td>\n",
" <td>0.102701</td>\n",
" <td>0.745550</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0.000549</td>\n",
" <td>0.740080</td>\n",
" <td>0.259371</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>0.075996</td>\n",
" <td>0.234080</td>\n",
" <td>0.689924</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>0.000436</td>\n",
" <td>0.205252</td>\n",
" <td>0.794312</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>0.000168</td>\n",
" <td>0.462925</td>\n",
" <td>0.536907</td>\n",
" </tr>\n",
" <tr>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <td>71</td>\n",
" <td>0.001247</td>\n",
" <td>0.391394</td>\n",
" <td>0.607358</td>\n",
" </tr>\n",
" <tr>\n",
" <td>72</td>\n",
" <td>0.000009</td>\n",
" <td>0.993300</td>\n",
" <td>0.006690</td>\n",
" </tr>\n",
" <tr>\n",
" <td>73</td>\n",
" <td>0.001145</td>\n",
" <td>0.688453</td>\n",
" <td>0.310403</td>\n",
" </tr>\n",
" <tr>\n",
" <td>74</td>\n",
" <td>0.000034</td>\n",
" <td>0.917543</td>\n",
" <td>0.082423</td>\n",
" </tr>\n",
" <tr>\n",
" <td>75</td>\n",
" <td>0.000886</td>\n",
" <td>0.341844</td>\n",
" <td>0.657270</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>76 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" 0 1 2\n",
"video_index \n",
"0 0.151749 0.102701 0.745550\n",
"1 0.000549 0.740080 0.259371\n",
"2 0.075996 0.234080 0.689924\n",
"3 0.000436 0.205252 0.794312\n",
"4 0.000168 0.462925 0.536907\n",
"... ... ... ...\n",
"71 0.001247 0.391394 0.607358\n",
"72 0.000009 0.993300 0.006690\n",
"73 0.001145 0.688453 0.310403\n",
"74 0.000034 0.917543 0.082423\n",
"75 0.000886 0.341844 0.657270\n",
"\n",
"[76 rows x 3 columns]"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"suit.df_prob_mean"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"from itertools import combinations"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[(10, 20, 30, 40)]"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(combinations([10,20,30,40], 4))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"odict_keys(['resnet18_word_bag', 'cnn_lstm_1', 'r2plus1d_18_1', 'resnet18_flat_L5'])"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"suit_map.keys()"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"suit1 = suit_map['resnet18_word_bag']\n",
"suit2 = suit_map['cnn_lstm_1']"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(0.9078947368421053, 0.868421052631579)"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"suit1.acc, suit2.acc"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"df_prob_mean = (suit1.df_prob_mean + suit2.df_prob_mean) / 2\n",
"target_ser = suit1.target_ser\n",
"suit12 = DisplaySuit(df_prob_mean=df_prob_mean, target_ser=target_ser)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"suit12.prepare_display()"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.8947368421052632"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"suit12.acc"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"acc: 0.8947368421052632\n",
"confuse matrix:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>31</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>3</td>\n",
" <td>37</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred 1 2\n",
"target \n",
"0 0 1\n",
"1 31 4\n",
"2 3 37"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wrong example show:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>pred</th>\n",
" <th>target</th>\n",
" </tr>\n",
" <tr>\n",
" <th>video_index</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>16</td>\n",
" <td>0.000030</td>\n",
" <td>0.417766</td>\n",
" <td>0.582205</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>17</td>\n",
" <td>0.000082</td>\n",
" <td>0.675358</td>\n",
" <td>0.324560</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>19</td>\n",
" <td>0.000145</td>\n",
" <td>0.461307</td>\n",
" <td>0.538548</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>21</td>\n",
" <td>0.001755</td>\n",
" <td>0.544024</td>\n",
" <td>0.454221</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>35</td>\n",
" <td>0.173972</td>\n",
" <td>0.356453</td>\n",
" <td>0.469575</td>\n",
" <td>2</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>52</td>\n",
" <td>0.076721</td>\n",
" <td>0.612891</td>\n",
" <td>0.310388</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>64</td>\n",
" <td>0.000195</td>\n",
" <td>0.295149</td>\n",
" <td>0.704656</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>71</td>\n",
" <td>0.000222</td>\n",
" <td>0.392896</td>\n",
" <td>0.606882</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 pred target\n",
"video_index \n",
"16 0.000030 0.417766 0.582205 2 1\n",
"17 0.000082 0.675358 0.324560 1 2\n",
"19 0.000145 0.461307 0.538548 2 1\n",
"21 0.001755 0.544024 0.454221 1 2\n",
"35 0.173972 0.356453 0.469575 2 0\n",
"52 0.076721 0.612891 0.310388 1 2\n",
"64 0.000195 0.295149 0.704656 2 1\n",
"71 0.000222 0.392896 0.606882 2 1"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Adjustment threshold=0.7\n",
"acc (adjusted): 0.8552631578947368\n",
"confuse matrix (adjusted):\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>pred_adjusted</th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>target</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>34</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>2</td>\n",
" <td>7</td>\n",
" <td>31</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"pred_adjusted 0 1 2\n",
"target \n",
"0 0 1 0\n",
"1 0 34 1\n",
"2 2 7 31"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"suit12.display()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['resnet18_word_bag'] 0.9078947368421053\n",
"['cnn_lstm_1'] 0.868421052631579\n",
"['r2plus1d_18_1'] 0.8947368421052632\n",
"['resnet18_flat_L5'] 0.8552631578947368\n",
"['resnet18_word_bag', 'cnn_lstm_1'] 0.8947368421052632\n",
"['resnet18_word_bag', 'r2plus1d_18_1'] 0.9078947368421053\n",
"['resnet18_word_bag', 'resnet18_flat_L5'] 0.9210526315789473\n",
"['cnn_lstm_1', 'r2plus1d_18_1'] 0.9078947368421053\n",
"['cnn_lstm_1', 'resnet18_flat_L5'] 0.9078947368421053\n",
"['r2plus1d_18_1', 'resnet18_flat_L5'] 0.868421052631579\n",
"['resnet18_word_bag', 'cnn_lstm_1', 'r2plus1d_18_1'] 0.9078947368421053\n",
"['resnet18_word_bag', 'cnn_lstm_1', 'resnet18_flat_L5'] 0.9078947368421053\n",
"['resnet18_word_bag', 'r2plus1d_18_1', 'resnet18_flat_L5'] 0.9078947368421053\n",
"['cnn_lstm_1', 'r2plus1d_18_1', 'resnet18_flat_L5'] 0.8947368421052632\n",
"['resnet18_word_bag', 'cnn_lstm_1', 'r2plus1d_18_1', 'resnet18_flat_L5'] 0.9210526315789473\n"
]
}
],
"source": [
"emsemble_map = {}\n",
"\n",
"for n in range(1, 5):\n",
" for key_suit_list in combinations(suit_map.items(), n):\n",
" df_prob_mean_list = []\n",
" key_list = []\n",
" target_ser = None\n",
" for key, suit in key_suit_list:\n",
" key_list.append(key)\n",
" df_prob_mean_list.append(suit.df_prob_mean)\n",
" target_ser = suit.target_ser\n",
" df_prob_mean = sum(df_prob_mean_list) / n\n",
" suit_ensemble = DisplaySuit(df_prob_mean=df_prob_mean, target_ser=target_ser)\n",
" suit_ensemble.prepare_display()\n",
" emsemble_map[tuple(key_list)] = suit_ensemble\n",
" print(key_list, suit_ensemble.acc)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>combination</th>\n",
" <th>acc</th>\n",
" <th>acc_adjusted</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>resnet18_word_bag</td>\n",
" <td>0.907895</td>\n",
" <td>0.842105</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>cnn_lstm_1</td>\n",
" <td>0.868421</td>\n",
" <td>0.828947</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>r2plus1d_18_1</td>\n",
" <td>0.894737</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>resnet18_flat_L5</td>\n",
" <td>0.855263</td>\n",
" <td>0.776316</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1</td>\n",
" <td>0.894737</td>\n",
" <td>0.855263</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>resnet18_word_bag, r2plus1d_18_1</td>\n",
" <td>0.907895</td>\n",
" <td>0.947368</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>resnet18_word_bag, resnet18_flat_L5</td>\n",
" <td>0.921053</td>\n",
" <td>0.842105</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>cnn_lstm_1, r2plus1d_18_1</td>\n",
" <td>0.907895</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>cnn_lstm_1, resnet18_flat_L5</td>\n",
" <td>0.907895</td>\n",
" <td>0.802632</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>r2plus1d_18_1, resnet18_flat_L5</td>\n",
" <td>0.868421</td>\n",
" <td>0.894737</td>\n",
" </tr>\n",
" <tr>\n",
" <td>10</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1</td>\n",
" <td>0.907895</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" <tr>\n",
" <td>11</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1, resnet18_flat_L5</td>\n",
" <td>0.907895</td>\n",
" <td>0.828947</td>\n",
" </tr>\n",
" <tr>\n",
" <td>12</td>\n",
" <td>resnet18_word_bag, r2plus1d_18_1, resnet18_fla...</td>\n",
" <td>0.907895</td>\n",
" <td>0.894737</td>\n",
" </tr>\n",
" <tr>\n",
" <td>13</td>\n",
" <td>cnn_lstm_1, r2plus1d_18_1, resnet18_flat_L5</td>\n",
" <td>0.894737</td>\n",
" <td>0.868421</td>\n",
" </tr>\n",
" <tr>\n",
" <td>14</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1, ...</td>\n",
" <td>0.921053</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" combination acc acc_adjusted\n",
"0 resnet18_word_bag 0.907895 0.842105\n",
"1 cnn_lstm_1 0.868421 0.828947\n",
"2 r2plus1d_18_1 0.894737 0.881579\n",
"3 resnet18_flat_L5 0.855263 0.776316\n",
"4 resnet18_word_bag, cnn_lstm_1 0.894737 0.855263\n",
"5 resnet18_word_bag, r2plus1d_18_1 0.907895 0.947368\n",
"6 resnet18_word_bag, resnet18_flat_L5 0.921053 0.842105\n",
"7 cnn_lstm_1, r2plus1d_18_1 0.907895 0.881579\n",
"8 cnn_lstm_1, resnet18_flat_L5 0.907895 0.802632\n",
"9 r2plus1d_18_1, resnet18_flat_L5 0.868421 0.894737\n",
"10 resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1 0.907895 0.881579\n",
"11 resnet18_word_bag, cnn_lstm_1, resnet18_flat_L5 0.907895 0.828947\n",
"12 resnet18_word_bag, r2plus1d_18_1, resnet18_fla... 0.907895 0.894737\n",
"13 cnn_lstm_1, r2plus1d_18_1, resnet18_flat_L5 0.894737 0.868421\n",
"14 resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1, ... 0.921053 0.881579"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df_em = pd.DataFrame([[', '.join(key), suit.acc, suit.acc_adjusted] for key, suit in emsemble_map.items()])\n",
"df_em.columns = ['combination', 'acc', 'acc_adjusted']\n",
"df_em"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>combination</th>\n",
" <th>acc</th>\n",
" <th>acc_adjusted</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>3</td>\n",
" <td>resnet18_flat_L5</td>\n",
" <td>0.855263</td>\n",
" <td>0.776316</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1</td>\n",
" <td>cnn_lstm_1</td>\n",
" <td>0.868421</td>\n",
" <td>0.828947</td>\n",
" </tr>\n",
" <tr>\n",
" <td>9</td>\n",
" <td>r2plus1d_18_1, resnet18_flat_L5</td>\n",
" <td>0.868421</td>\n",
" <td>0.894737</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
" <td>r2plus1d_18_1</td>\n",
" <td>0.894737</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" <tr>\n",
" <td>4</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1</td>\n",
" <td>0.894737</td>\n",
" <td>0.855263</td>\n",
" </tr>\n",
" <tr>\n",
" <td>13</td>\n",
" <td>cnn_lstm_1, r2plus1d_18_1, resnet18_flat_L5</td>\n",
" <td>0.894737</td>\n",
" <td>0.868421</td>\n",
" </tr>\n",
" <tr>\n",
" <td>0</td>\n",
" <td>resnet18_word_bag</td>\n",
" <td>0.907895</td>\n",
" <td>0.842105</td>\n",
" </tr>\n",
" <tr>\n",
" <td>5</td>\n",
" <td>resnet18_word_bag, r2plus1d_18_1</td>\n",
" <td>0.907895</td>\n",
" <td>0.947368</td>\n",
" </tr>\n",
" <tr>\n",
" <td>7</td>\n",
" <td>cnn_lstm_1, r2plus1d_18_1</td>\n",
" <td>0.907895</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" <tr>\n",
" <td>8</td>\n",
" <td>cnn_lstm_1, resnet18_flat_L5</td>\n",
" <td>0.907895</td>\n",
" <td>0.802632</td>\n",
" </tr>\n",
" <tr>\n",
" <td>10</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1</td>\n",
" <td>0.907895</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" <tr>\n",
" <td>11</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1, resnet18_flat_L5</td>\n",
" <td>0.907895</td>\n",
" <td>0.828947</td>\n",
" </tr>\n",
" <tr>\n",
" <td>12</td>\n",
" <td>resnet18_word_bag, r2plus1d_18_1, resnet18_fla...</td>\n",
" <td>0.907895</td>\n",
" <td>0.894737</td>\n",
" </tr>\n",
" <tr>\n",
" <td>6</td>\n",
" <td>resnet18_word_bag, resnet18_flat_L5</td>\n",
" <td>0.921053</td>\n",
" <td>0.842105</td>\n",
" </tr>\n",
" <tr>\n",
" <td>14</td>\n",
" <td>resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1, ...</td>\n",
" <td>0.921053</td>\n",
" <td>0.881579</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" combination acc acc_adjusted\n",
"3 resnet18_flat_L5 0.855263 0.776316\n",
"1 cnn_lstm_1 0.868421 0.828947\n",
"9 r2plus1d_18_1, resnet18_flat_L5 0.868421 0.894737\n",
"2 r2plus1d_18_1 0.894737 0.881579\n",
"4 resnet18_word_bag, cnn_lstm_1 0.894737 0.855263\n",
"13 cnn_lstm_1, r2plus1d_18_1, resnet18_flat_L5 0.894737 0.868421\n",
"0 resnet18_word_bag 0.907895 0.842105\n",
"5 resnet18_word_bag, r2plus1d_18_1 0.907895 0.947368\n",
"7 cnn_lstm_1, r2plus1d_18_1 0.907895 0.881579\n",
"8 cnn_lstm_1, resnet18_flat_L5 0.907895 0.802632\n",
"10 resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1 0.907895 0.881579\n",
"11 resnet18_word_bag, cnn_lstm_1, resnet18_flat_L5 0.907895 0.828947\n",
"12 resnet18_word_bag, r2plus1d_18_1, resnet18_fla... 0.907895 0.894737\n",
"6 resnet18_word_bag, resnet18_flat_L5 0.921053 0.842105\n",
"14 resnet18_word_bag, cnn_lstm_1, r2plus1d_18_1, ... 0.921053 0.881579"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df_em.sort_values(by=['acc'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment