Skip to content

Instantly share code, notes, and snippets.

@monajalal

monajalal/loocv_864images.ipynb Secret

Created November 27, 2018 04:05
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save monajalal/322f1e86496701d4b7b55a4d0fd80241 to your computer and use it in GitHub Desktop.
Save monajalal/322f1e86496701d4b7b55a4d0fd80241 to your computer and use it in GitHub Desktop.
Download ZIP
loocv but on more images and 9 categories
Display the source blob
Display the rendered blob
Raw
loocv_864images.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from __future__ import print_function, division\n",
"\n",
"import torch\n",
"from torch.autograd import Variable\n",
"from sklearn.metrics import accuracy_score\n",
"from sklearn.metrics import confusion_matrix\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"import torch.nn.functional as F\n",
"from torch.optim import lr_scheduler\n",
"import numpy as np\n",
"import torchvision\n",
"from torchvision import datasets, models, transforms\n",
"import matplotlib.pyplot as plt\n",
"import time\n",
"import os\n",
"import copy\n",
"\n",
"import torch.utils.data as data_utils\n",
"from torch.utils import data"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"torch.manual_seed(2809)\n",
"\n",
"\n",
"def train_model(model, criterion, optimizer, scheduler,\n",
" dataloader, num_epochs=25):\n",
"\n",
" since = time.time()\n",
" for epoch in range(num_epochs):\n",
" print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
" print('-' * 10)\n",
" \n",
" # Each epoch has a training and validation phase\n",
" scheduler.step()\n",
" model.train() # Set model to training mode\n",
" \n",
" \n",
" running_loss = 0.0\n",
" running_corrects = 0\n",
" \n",
" # Iterate over data.\n",
" for inputs, labels in dataloader:\n",
" inputs = inputs.to(device)\n",
" labels = labels.to(device)\n",
" \n",
" # zero the parameter gradients\n",
" optimizer.zero_grad()\n",
" \n",
" # forward\n",
" # track history if only in train\n",
" with torch.set_grad_enabled(True):\n",
" outputs = model(inputs)\n",
" _, preds = torch.max(outputs, 1)\n",
" loss = criterion(outputs, labels)\n",
" # backward + optimize only if in training phase\n",
" loss.backward()\n",
" optimizer.step()\n",
" # statistics\n",
" running_loss += loss.item() * inputs.size(0)\n",
" running_corrects += torch.sum(preds == labels.data)\n",
" \n",
" epoch_loss = running_loss / dataset_size['train']\n",
" epoch_acc = running_corrects.double() / dataset_size['train']\n",
" \n",
" print('Loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))\n",
" \n",
" time_elapsed = time.time() - since\n",
" print('Training complete in {:.0f}m {:.0f}s'.format(\n",
" time_elapsed // 60, time_elapsed % 60))\n",
" \n",
" return model"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"data_transforms = {\n",
" 'train': transforms.Compose([\n",
" transforms.RandomResizedCrop(224),\n",
" transforms.RandomHorizontalFlip(),\n",
" transforms.RandomRotation(20),\n",
" transforms.ToTensor(),\n",
" transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
" ]),\n",
" 'test': transforms.Compose([\n",
" transforms.Resize(224),\n",
" transforms.ToTensor(),\n",
" transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
" ])\n",
"}\n",
"\n",
"\n",
"data_dir = \"images\"\n",
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
"\n",
"\n",
"model_ft = models.resnet50(pretrained=True)\n",
"num_ftrs = model_ft.fc.in_features\n",
"model_ft.fc = nn.Linear(num_ftrs, 2)\n",
"model_ft = model_ft.to(device)\n",
"\n",
"# Save a clone of initial model to restore later\n",
"initial_model = copy.deepcopy(model_ft)\n",
"\n",
"criterion = nn.CrossEntropyLoss()\n",
"\n",
"# Observe that all parameters are being optimized\n",
"optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n",
"\n",
"# Decay LR by a factor of 0.1 every 7 epochs\n",
"exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)\n",
"\n",
"#model_ft = model_ft.cuda()\n",
"nb_samples = 864\n",
"nb_classes = 9\n",
"\n",
"image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),\n",
" data_transforms[x])\n",
" for x in ['train']}\n",
"\n",
"dataset_size = {x: len(image_datasets[x]) for x in ['train']}\n",
"class_names = image_datasets['train'].classes"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using sample 0 as test data\n",
"Resetting model\n",
"Epoch 0/24\n",
"----------\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7fa90b043c18>>\n",
"Traceback (most recent call last):\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
" self._shutdown_workers()\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
" self.worker_result_queue.get()\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 337, in get\n",
" return _ForkingPickler.loads(res)\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
" fd = df.detach()\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py\", line 57, in detach\n",
" with _resource_sharer.get_connection(self._id) as conn:\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
" c = Client(address, authkey=process.current_process().authkey)\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 493, in Client\n",
" answer_challenge(c, authkey)\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 737, in answer_challenge\n",
" response = connection.recv_bytes(256) # reject large message\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n",
" buf = self._recv_bytes(maxlength)\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
" buf = self._recv(4)\n",
" File \"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n",
" chunk = read(handle, remaining)\n",
"ConnectionResetError: [Errno 104] Connection reset by peer\n"
]
},
{
"ename": "RuntimeError",
"evalue": "CuDNN error: CUDNN_STATUS_EXECUTION_FAILED",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-4-9365fbc2464a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 34\u001b[0m model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, sample, target, num_epochs=10)'''\n\u001b[1;32m 35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m \u001b[0mmodel_ft\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_ft\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_ft\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexp_lr_scheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0;31m# Test on LOO sample\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-2-305560153b80>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(model, criterion, optimizer, scheduler, dataloader, num_epochs)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0;31m# backward + optimize only if in training phase\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 36\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0;31m# statistics\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \"\"\"\n\u001b[0;32m---> 93\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 88\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 89\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mRuntimeError\u001b[0m: CuDNN error: CUDNN_STATUS_EXECUTION_FAILED"
]
}
],
"source": [
"# LOOCV\n",
"loocv_preds = []\n",
"loocv_targets = []\n",
"loocv_probs = []\n",
"\n",
"for idx in range(nb_samples):\n",
"\n",
" print('Using sample {} as test data'.format(idx))\n",
"\n",
" print('Resetting model')\n",
" model_ft = copy.deepcopy(initial_model)\n",
" # Observe that all parameters are being optimized\n",
" optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n",
"\n",
" # Decay LR by a factor of 0.1 every 7 epochs\n",
" exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)\n",
"\n",
" # Get all indices and remove test sample\n",
" train_indices = list(range(len(image_datasets['train'])))\n",
" del train_indices[idx]\n",
"\n",
" # Create new sampler\n",
" sampler = data.SubsetRandomSampler(train_indices)\n",
" dataloader = data.DataLoader(\n",
" image_datasets['train'],\n",
" num_workers=2,\n",
" batch_size=1,\n",
" sampler=sampler\n",
" )\n",
"\n",
" # Train model\n",
" '''for batch_idx, (sample, target) in enumerate(dataloader):\n",
" print('Batch {}'.format(batch_idx))\n",
" model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, sample, target, num_epochs=10)'''\n",
"\n",
" model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, dataloader, num_epochs=25)\n",
"\n",
" # Test on LOO sample\n",
" model_ft.eval()\n",
" test_data, test_target = image_datasets['train'][idx]\n",
" # Apply test preprocessing on data\n",
" print(type(test_data))\n",
" test_data = data_transforms['test'](transforms.ToPILImage()(test_data))\n",
" test_data = test_data.cuda()\n",
" test_target = torch.tensor(test_target)\n",
" test_target = test_target.cuda()\n",
" test_data.unsqueeze_(0)\n",
" test_target.unsqueeze_(0)\n",
" print(test_data.shape)\n",
" output = model_ft(test_data)\n",
" pred = torch.argmax(output, 1)\n",
" loocv_preds.append(pred)\n",
" loocv_targets.append(test_target.item())\n",
" \n",
" print(\"idx: \", idx)\n",
" gpu_tensor_probs = F.softmax(output, 1)\n",
" cpu_numpy_probs = gpu_tensor_probs.data.cpu().numpy()\n",
" loocv_probs.append(cpu_numpy_probs.tolist())\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"loocv preds: []\n",
"loocv targets: []\n",
"acc score: nan\n",
"confusion matrix: \n",
" []\n",
"Class probabilities: []\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/numpy/lib/function_base.py:356: RuntimeWarning: Mean of empty slice.\n",
" avg = a.mean(axis)\n",
"/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars\n",
" ret = ret.dtype.type(ret / rcount)\n"
]
}
],
"source": [
"print(\"loocv preds: \", loocv_preds)\n",
"print(\"loocv targets: \", loocv_targets)\n",
"print(\"acc score: \", accuracy_score(loocv_targets, loocv_preds))\n",
"print(\"confusion matrix: \\n\", confusion_matrix(loocv_targets, loocv_preds))\n",
"print('Class probabilities:', loocv_probs)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment