Created
May 5, 2020 15:40
-
-
Save oguiza/d86a5c60f7796e2a587c39ce8a9abc07 to your computer and use it in GitHub Desktop.
Proba_Metrics.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "metadata": { | |
| "colab_type": "text", | |
| "id": "qcMubba3e7e2" | |
| }, | |
| "cell_type": "markdown", | |
| "source": "# Import libraries" | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 52 | |
| }, | |
| "colab_type": "code", | |
| "id": "JIpVVG_gMx50", | |
| "outputId": "25a0c603-f643-4734-e8b3-4c05b7ccaca6", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "!pip install git+https://github.com/fastai/fastcore.git@master -q\n!pip install git+https://github.com/fastai/fastai2.git@master -q", | |
| "execution_count": 1, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": " Building wheel for fastcore (setup.py) ... \u001b[?25l\u001b[?25hdone\n Building wheel for fastai2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": {}, | |
| "colab_type": "code", | |
| "id": "F-vR9U_vRzbB", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "from fastai2.vision.all import *\nfrom fastai2.metrics import *\nfrom sklearn import metrics as skm", | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "metadata": { | |
| "colab_type": "text", | |
| "id": "BOog1g8qZqjj" | |
| }, | |
| "cell_type": "markdown", | |
| "source": "# Scenarios\nIn sklearn there are 3 scenarios for **roc_auc_score** (each of them calculated slightly differently):\n\n* **Binary**:\n * `targets`: shape = `(n_samples, )` \n * `preds`: pass through **softmax** and then [:, -1], shape = `(n_samples,)`\n \n* **Multiclass**:\n * `targets`: shape = `(n_samples, )` \n * `preds`: pass through **softmax**, shape = `(n_samples, n_classes)`\n * `multi_class `= ‘ovr' or ‘ovo' (1)\n \n* **Multilabel**:\n * `targets`: shape = `(n_samples, n_classes)` \n * `preds`: pass through **sigmoid**, shape = `(n_samples, n_classes)`\n\n(1) ‘ovr’: average AUC of each class against the rest . 'ovo’ : average AUC of all possible pairwise combinations of classes.\n\nsklearn's **average_precision_score** implementation is restricted to binary or multilabel classification tasks. So it cannot be used in multiclass cases." | |
| }, | |
| { | |
| "metadata": { | |
| "colab_type": "text", | |
| "id": "eBH9OuqoZLZ3" | |
| }, | |
| "cell_type": "markdown", | |
| "source": "# Proposed code" | |
| }, | |
| { | |
| "metadata": { | |
| "colab": {}, | |
| "colab_type": "code", | |
| "id": "T5p8UL7wU2bx", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "class AccumMetric(Metric):\n \"Stores predictions and targets on CPU in accumulate to perform final calculations with `func`.\"\n def __init__(self, func, dim_argmax=None, sigmoid=False, softmax=False, proba=False, thresh=None, to_np=False, invert_arg=False,\n flatten=True, **kwargs):\n store_attr(self,'func,dim_argmax,sigmoid,softmax,proba,thresh,flatten')\n self.to_np,self.invert_args,self.kwargs = to_np,invert_arg,kwargs\n\n def reset(self): self.targs,self.preds = [],[]\n\n def accumulate(self, learn):\n pred = learn.pred.argmax(dim=self.dim_argmax) if (self.dim_argmax and not self.proba) else learn.pred\n if self.sigmoid: pred = torch.sigmoid(pred)\n if self.thresh: pred = (pred >= self.thresh)\n if self.softmax: \n pred = F.softmax(pred, dim=-1)\n if learn.dls.c == 2: pred = pred[:, -1]\n targ = learn.y\n pred,targ = to_detach(pred),to_detach(targ)\n if self.flatten: pred,targ = flatten_check(pred,targ)\n self.preds.append(pred)\n self.targs.append(targ)\n\n @property\n def value(self):\n if len(self.preds) == 0: return\n preds,targs = torch.cat(self.preds),torch.cat(self.targs)\n if self.to_np: preds,targs = preds.numpy(),targs.numpy()\n return self.func(targs, preds, **self.kwargs) if self.invert_args else self.func(preds, targs, **self.kwargs)\n\n @property\n def name(self): return self.func.func.__name__ if hasattr(self.func, 'func') else self.func.__name__\n\ndef skm_to_fastai(func, is_class=True, thresh=None, axis=-1, sigmoid=None, softmax=False, proba=False, **kwargs):\n \"Convert `func` from sklearn.metrics to a fastai metric\"\n dim_argmax = axis if is_class and thresh is None else None\n sigmoid = sigmoid if sigmoid is not None else (is_class and thresh is not None)\n return AccumMetric(func, dim_argmax=dim_argmax, sigmoid=sigmoid, softmax=softmax, proba=proba, thresh=thresh,\n to_np=True, invert_arg=True, **kwargs)\n\ndef APScore(axis=-1, average='macro', pos_label=1, sample_weight=None):\n \"Average Precision for binary single-label classification problems\"\n return skm_to_fastai(skm.average_precision_score, axis=axis, flatten=False, softmax=True, proba=True,\n average=average, pos_label=pos_label, sample_weight=sample_weight)\n \ndef APScoreMulti(axis=-1, average='macro', pos_label=1, sample_weight=None):\n \"Average Precision for multi-label classification problems\"\n return skm_to_fastai(skm.average_precision_score, axis=axis, flatten=False, sigmoid=True, proba=True,\n average=average, pos_label=pos_label, sample_weight=sample_weight)\n \ndef RocAuc(axis=-1, average='macro', sample_weight=None, max_fpr=None, multi_class='raise', labels=None):\n \"Area Under the Receiver Operating Characteristic Curve for single-label classification problems\"\n \"\"\"use default multi_class ('raise') for binary-class, and 'ovr'(average AUC of each class against the rest) \n or 'ovo' (average AUC of all possible pairwise combinations of classes) for multi-class tasks\"\"\"\n return skm_to_fastai(skm.roc_auc_score, axis=axis, flatten=False, softmax=True, proba=True,\n average=average, sample_weight=sample_weight, max_fpr=max_fpr, multi_class=multi_class, labels=labels)\n \ndef RocAucMulti(axis=-1, average='macro', sample_weight=None, max_fpr=None):\n \"Area Under the Receiver Operating Characteristic Curve for multi-label classification problems\"\n return skm_to_fastai(skm.roc_auc_score, axis=axis, flatten=False, sigmoid=True, proba=True,\n average=average, sample_weight=sample_weight, max_fpr=max_fpr)", | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "metadata": { | |
| "colab_type": "text", | |
| "id": "bMBMavMhLa8l" | |
| }, | |
| "cell_type": "markdown", | |
| "source": "# Binary:" | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 16 | |
| }, | |
| "colab_type": "code", | |
| "id": "uajPS5RlxjrI", | |
| "outputId": "8e5a85ad-68de-44d6-af77-02f1aada73ce", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "path = untar_data(URLs.MNIST_TINY)\ndls = ImageDataLoaders.from_folder(path)", | |
| "execution_count": 4, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": "", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 137 | |
| }, | |
| "colab_type": "code", | |
| "id": "IYb6cStePQ9c", | |
| "outputId": "e520c706-3633-4028-a5f7-a24e424841f7", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "learn = cnn_learner(dls, resnet18, pretrained=False, metrics=[APScore(), RocAuc()])\nlearn.fit_one_cycle(3, 0.1)", | |
| "execution_count": 5, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": "<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: left;\">\n <th>epoch</th>\n <th>train_loss</th>\n <th>valid_loss</th>\n <th>average_precision_score</th>\n <th>roc_auc_score</th>\n <th>time</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <td>0</td>\n <td>1.653272</td>\n <td>1666.066528</td>\n <td>0.505007</td>\n <td>0.500000</td>\n <td>00:01</td>\n </tr>\n <tr>\n <td>1</td>\n <td>1.086577</td>\n <td>231.428848</td>\n <td>0.505007</td>\n <td>0.500000</td>\n <td>00:01</td>\n </tr>\n <tr>\n <td>2</td>\n <td>0.797317</td>\n <td>10.656080</td>\n <td>0.733888</td>\n <td>0.815029</td>\n <td>00:01</td>\n </tr>\n </tbody>\n</table>", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 16 | |
| }, | |
| "colab_type": "code", | |
| "id": "JVldLRaEM8sE", | |
| "outputId": "2c8705e6-61e2-4e7f-9642-0d3bb0a90246", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "valid_probas, valid_targets, valid_preds = learn.get_preds(dl=dls.valid, with_decoded=True)", | |
| "execution_count": 6, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": "", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 52 | |
| }, | |
| "colab_type": "code", | |
| "id": "BaG-LDJwQncp", | |
| "outputId": "daa82d0b-343b-4623-8317-82a3e78d8dba", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "# APScore and RocAuc calculated based on probas\nprint(f'avg precision : {skm.average_precision_score(valid_targets, valid_probas[:, 1]):8.6f}')\nprint(f'roc auc : {skm.roc_auc_score(valid_targets, valid_probas[:, 1]):8.6f}')", | |
| "execution_count": 7, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": "avg precision : 0.733888\nroc auc : 0.815029\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 52 | |
| }, | |
| "colab_type": "code", | |
| "id": "8hgemM8exHj9", | |
| "outputId": "c25bfa28-ff7b-448d-a457-0167543a95e4", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "# APScore and RocAuc as they were previously on fastai2 calculated based on preds - these were wrong!!\nprint(f'avg precision : {skm.average_precision_score(valid_targets, valid_preds):8.6f}')\nprint(f'roc auc : {skm.roc_auc_score(valid_targets, valid_preds):8.6f}')", | |
| "execution_count": 8, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": "avg precision : 0.596284\nroc auc : 0.654624\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab_type": "text", | |
| "id": "OPk9hKfsLf63" | |
| }, | |
| "cell_type": "markdown", | |
| "source": "# Multiclass:" | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 34 | |
| }, | |
| "colab_type": "code", | |
| "id": "2pCXFq9Z-nCR", | |
| "outputId": "ab0198d6-aab9-439f-b2f1-b4969088b7b5", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "bs = 64\npath = untar_data(URLs.PETS); path\nPath.BASE_PATH = path\npath_anno = path/'annotations'\npath_img = path/'images'\nfnames = get_image_files(path_img)\ndls = ImageDataLoaders.from_name_re(\n path, fnames, pat=r'(.+)_\\d+.jpg$', item_tfms=Resize(460), bs=bs,\n batch_tfms=[*aug_transforms(size=224, min_scale=0.75), Normalize.from_stats(*imagenet_stats)])\ndls.c", | |
| "execution_count": 9, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": "", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "data": { | |
| "text/plain": "37" | |
| }, | |
| "execution_count": 9, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "execute_result" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 201, | |
| "referenced_widgets": [ | |
| "7f540ad08833493db60f23aae7640818", | |
| "60f5701faaeb4cd7bbf48ac0d9eec63c", | |
| "a1493e86ec424e81a4486ffb11b27267", | |
| "d8f640d0ddc04f93a1bed3e6fd5d5c67", | |
| "b2d4ad59d12c4be6a44378bf4b39dcd7", | |
| "b8e279fd96664ba6b0e54273e3ca4585", | |
| "ede3695f26ce4ab0815495f04260743f", | |
| "3693a06704f14253ba15b4e41613de2d" | |
| ] | |
| }, | |
| "colab_type": "code", | |
| "id": "Y8IWRml8AKh6", | |
| "outputId": "bd6d2156-f513-499f-f80f-158224354542", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "learn = cnn_learner(dls, resnet34, metrics=[accuracy, RocAuc(multi_class='ovo'), RocAuc(multi_class='ovr')]).to_fp16()\nlearn.fit_one_cycle(1)", | |
| "execution_count": 10, | |
| "outputs": [ | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": "Downloading: \"https://download.pytorch.org/models/resnet34-333f7ec4.pth\" to /root/.cache/torch/checkpoints/resnet34-333f7ec4.pth\n" | |
| }, | |
| { | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "7f540ad08833493db60f23aae7640818", | |
| "version_major": 2, | |
| "version_minor": 0 | |
| }, | |
| "text/plain": "HBox(children=(IntProgress(value=0, max=87306240), HTML(value='')))" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": "\n" | |
| }, | |
| { | |
| "data": { | |
| "text/html": "<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: left;\">\n <th>epoch</th>\n <th>train_loss</th>\n <th>valid_loss</th>\n <th>accuracy</th>\n <th>roc_auc_score</th>\n <th>roc_auc_score</th>\n <th>time</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <td>0</td>\n <td>1.248549</td>\n <td>0.344744</td>\n <td>0.899188</td>\n <td>0.997780</td>\n <td>0.997788</td>\n <td>01:04</td>\n </tr>\n </tbody>\n</table>", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2854: UserWarning: The default behavior for interpolate/upsample with float scale_factor will change in 1.6.0 to align with other frameworks/libraries, and use scale_factor directly, instead of relying on the computed output size. If you wish to keep the old behavior, please set recompute_scale_factor=True. See the documentation of nn.Upsample for details. \n warnings.warn(\"The default behavior for interpolate/upsample with float scale_factor will change \"\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 72 | |
| }, | |
| "colab_type": "code", | |
| "id": "OR3Og5iLKjTS", | |
| "outputId": "08d5951f-b9e0-40dd-fb26-7b4251277a3f", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "valid_probas, valid_targets, valid_preds = learn.get_preds(dl=dls.valid, with_decoded=True)", | |
| "execution_count": 11, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": "", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:2854: UserWarning: The default behavior for interpolate/upsample with float scale_factor will change in 1.6.0 to align with other frameworks/libraries, and use scale_factor directly, instead of relying on the computed output size. If you wish to keep the old behavior, please set recompute_scale_factor=True. See the documentation of nn.Upsample for details. \n warnings.warn(\"The default behavior for interpolate/upsample with float scale_factor will change \"\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 52 | |
| }, | |
| "colab_type": "code", | |
| "id": "Fc7jATbAA6Us", | |
| "outputId": "b6a429fe-db58-4b40-d2b2-4fb36eb65ba4", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "print(f'roc auc (\"ovo\") : {skm.roc_auc_score(valid_targets, valid_probas, multi_class=\"ovo\"):8.6f}')\nprint(f'roc auc (\"ovr\") : {skm.roc_auc_score(valid_targets, valid_probas, multi_class=\"ovr\"):8.6f}')", | |
| "execution_count": 12, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": "roc auc (\"ovo\") : 0.997780\nroc auc (\"ovr\") : 0.997788\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab_type": "text", | |
| "id": "dOxwFh48LlRh" | |
| }, | |
| "cell_type": "markdown", | |
| "source": "# Multilabel:" | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 16 | |
| }, | |
| "colab_type": "code", | |
| "id": "FvcXWlcMMLZh", | |
| "outputId": "8b085400-e464-4fff-cf10-61e9c5931019", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "path = untar_data(URLs.PASCAL_2007)\ndf = pd.read_csv(path/'train.csv')", | |
| "execution_count": 13, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": "", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": {}, | |
| "colab_type": "code", | |
| "id": "PxJiPxLJKPgM", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "def splitter(df):\n train = df.index[~df['is_valid']].tolist()\n valid = df.index[df['is_valid']].tolist()\n return train,valid\n\ndef get_x(r): return path/'train'/r['fname']\ndef get_y(r): return r['labels'].split(' ')\n\ndef accuracy_multi(inp, targ, thresh=0.5, sigmoid=True):\n \"Compute accuracy when `inp` and `targ` are the same size.\"\n if sigmoid: inp = inp.sigmoid()\n return ((inp>thresh)==targ.bool()).float().mean()\n\ndblock = DataBlock(blocks=(ImageBlock, MultiCategoryBlock),\n splitter=splitter,\n get_x=get_x, \n get_y=get_y,\n item_tfms = RandomResizedCrop(128, min_scale=0.35))\ndls = dblock.dataloaders(df)", | |
| "execution_count": 0, | |
| "outputs": [] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 145, | |
| "referenced_widgets": [ | |
| "04c1250abdcf448cabcbbdb9b617223b", | |
| "7a60c275ba4b4a85af07b9025d7cf0d9", | |
| "bbc7411ce46e433b819a277dfd97aab2", | |
| "1fcb550c5a0543e4bab36eb8df98522d", | |
| "15b42940fc8c47d9b6c583941698832f", | |
| "e6ddaca7a1cf43e4a6cdeda8fe4374c3", | |
| "1361287ddb334448b8671ee173d83675", | |
| "b5825f6c2bf54bcb8b361d9aa52b22b0" | |
| ] | |
| }, | |
| "colab_type": "code", | |
| "id": "lCGzcv60LoE4", | |
| "outputId": "b0cc7ea3-e5f7-4c34-8693-d6d305ca2879", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "learn = cnn_learner(dls, resnet18, metrics=[RocAucMulti(), APScoreMulti()])\nlearn.fit_one_cycle(1)", | |
| "execution_count": 15, | |
| "outputs": [ | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /root/.cache/torch/checkpoints/resnet18-5c106cde.pth\n" | |
| }, | |
| { | |
| "data": { | |
| "application/vnd.jupyter.widget-view+json": { | |
| "model_id": "04c1250abdcf448cabcbbdb9b617223b", | |
| "version_major": 2, | |
| "version_minor": 0 | |
| }, | |
| "text/plain": "HBox(children=(IntProgress(value=0, max=46827520), HTML(value='')))" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": "\n" | |
| }, | |
| { | |
| "data": { | |
| "text/html": "<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: left;\">\n <th>epoch</th>\n <th>train_loss</th>\n <th>valid_loss</th>\n <th>roc_auc_score</th>\n <th>average_precision_score</th>\n <th>time</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <td>0</td>\n <td>0.928385</td>\n <td>0.686431</td>\n <td>0.803554</td>\n <td>0.365506</td>\n <td>00:31</td>\n </tr>\n </tbody>\n</table>", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 16 | |
| }, | |
| "colab_type": "code", | |
| "id": "7dUNtO0KN-wg", | |
| "outputId": "4236e213-1128-4883-e163-2c7443b85682", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "valid_probas, valid_targets, valid_preds = learn.get_preds(dl=dls.valid, with_decoded=True)", | |
| "execution_count": 16, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": "", | |
| "text/plain": "<IPython.core.display.HTML object>" | |
| }, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "output_type": "display_data" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 52 | |
| }, | |
| "colab_type": "code", | |
| "id": "OzMqlHtQnlhl", | |
| "outputId": "b543d86a-1fe3-4230-824e-f7d143d9fafe", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "print(f'avg precision : {skm.average_precision_score(valid_targets, valid_probas):8.6f}')\nprint(f'roc auc : {skm.roc_auc_score(valid_targets, valid_probas):8.6f}')", | |
| "execution_count": 17, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": "avg precision : 0.365506\nroc auc : 0.803554\n" | |
| } | |
| ] | |
| }, | |
| { | |
| "metadata": { | |
| "colab": {}, | |
| "colab_type": "code", | |
| "id": "zOuJqu54V2rJ", | |
| "trusted": false | |
| }, | |
| "cell_type": "code", | |
| "source": "", | |
| "execution_count": 0, | |
| "outputs": [] | |
| } | |
| ], | |
| "metadata": { | |
| "accelerator": "GPU", | |
| "colab": { | |
| "collapsed_sections": [ | |
| "OPk9hKfsLf63", | |
| "dOxwFh48LlRh" | |
| ], | |
| "name": "Proba_Metrics.ipynb", | |
| "provenance": [] | |
| }, | |
| "gist": { | |
| "id": "", | |
| "data": { | |
| "description": "Proba_Metrics.ipynb", | |
| "public": true | |
| } | |
| }, | |
| "kernelspec": { | |
| "name": "python3", | |
| "display_name": "Python 3", | |
| "language": "python" | |
| }, | |
| "language_info": { | |
| "name": "python", | |
| "version": "3.7.3", | |
| "mimetype": "text/x-python", | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "pygments_lexer": "ipython3", | |
| "nbconvert_exporter": "python", | |
| "file_extension": ".py" | |
| }, | |
| "toc": { | |
| "nav_menu": {}, | |
| "number_sections": true, | |
| "sideBar": true, | |
| "skip_h1_title": false, | |
| "base_numbering": 1, | |
| "title_cell": "Table of Contents", | |
| "title_sidebar": "Contents", | |
| "toc_cell": false, | |
| "toc_position": {}, | |
| "toc_section_display": true, | |
| "toc_window_display": false | |
| }, | |
| "widgets": { | |
| "application/vnd.jupyter.widget-state+json": { | |
| "04c1250abdcf448cabcbbdb9b617223b": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "HBoxModel", | |
| "state": { | |
| "_dom_classes": [], | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "HBoxModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_view_module_version": "1.5.0", | |
| "_view_name": "HBoxView", | |
| "box_style": "", | |
| "children": [ | |
| "IPY_MODEL_bbc7411ce46e433b819a277dfd97aab2", | |
| "IPY_MODEL_1fcb550c5a0543e4bab36eb8df98522d" | |
| ], | |
| "layout": "IPY_MODEL_7a60c275ba4b4a85af07b9025d7cf0d9" | |
| } | |
| }, | |
| "1361287ddb334448b8671ee173d83675": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "DescriptionStyleModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "DescriptionStyleModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "StyleView", | |
| "description_width": "" | |
| } | |
| }, | |
| "15b42940fc8c47d9b6c583941698832f": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "ProgressStyleModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "ProgressStyleModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "StyleView", | |
| "bar_color": null, | |
| "description_width": "initial" | |
| } | |
| }, | |
| "1fcb550c5a0543e4bab36eb8df98522d": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "HTMLModel", | |
| "state": { | |
| "_dom_classes": [], | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "HTMLModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_view_module_version": "1.5.0", | |
| "_view_name": "HTMLView", | |
| "description": "", | |
| "description_tooltip": null, | |
| "layout": "IPY_MODEL_b5825f6c2bf54bcb8b361d9aa52b22b0", | |
| "placeholder": "", | |
| "style": "IPY_MODEL_1361287ddb334448b8671ee173d83675", | |
| "value": " 44.7M/44.7M [00:00<00:00, 48.3MB/s]" | |
| } | |
| }, | |
| "3693a06704f14253ba15b4e41613de2d": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.2.0", | |
| "_model_name": "LayoutModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "LayoutView", | |
| "align_content": null, | |
| "align_items": null, | |
| "align_self": null, | |
| "border": null, | |
| "bottom": null, | |
| "display": null, | |
| "flex": null, | |
| "flex_flow": null, | |
| "grid_area": null, | |
| "grid_auto_columns": null, | |
| "grid_auto_flow": null, | |
| "grid_auto_rows": null, | |
| "grid_column": null, | |
| "grid_gap": null, | |
| "grid_row": null, | |
| "grid_template_areas": null, | |
| "grid_template_columns": null, | |
| "grid_template_rows": null, | |
| "height": null, | |
| "justify_content": null, | |
| "justify_items": null, | |
| "left": null, | |
| "margin": null, | |
| "max_height": null, | |
| "max_width": null, | |
| "min_height": null, | |
| "min_width": null, | |
| "object_fit": null, | |
| "object_position": null, | |
| "order": null, | |
| "overflow": null, | |
| "overflow_x": null, | |
| "overflow_y": null, | |
| "padding": null, | |
| "right": null, | |
| "top": null, | |
| "visibility": null, | |
| "width": null | |
| } | |
| }, | |
| "60f5701faaeb4cd7bbf48ac0d9eec63c": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.2.0", | |
| "_model_name": "LayoutModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "LayoutView", | |
| "align_content": null, | |
| "align_items": null, | |
| "align_self": null, | |
| "border": null, | |
| "bottom": null, | |
| "display": null, | |
| "flex": null, | |
| "flex_flow": null, | |
| "grid_area": null, | |
| "grid_auto_columns": null, | |
| "grid_auto_flow": null, | |
| "grid_auto_rows": null, | |
| "grid_column": null, | |
| "grid_gap": null, | |
| "grid_row": null, | |
| "grid_template_areas": null, | |
| "grid_template_columns": null, | |
| "grid_template_rows": null, | |
| "height": null, | |
| "justify_content": null, | |
| "justify_items": null, | |
| "left": null, | |
| "margin": null, | |
| "max_height": null, | |
| "max_width": null, | |
| "min_height": null, | |
| "min_width": null, | |
| "object_fit": null, | |
| "object_position": null, | |
| "order": null, | |
| "overflow": null, | |
| "overflow_x": null, | |
| "overflow_y": null, | |
| "padding": null, | |
| "right": null, | |
| "top": null, | |
| "visibility": null, | |
| "width": null | |
| } | |
| }, | |
| "7a60c275ba4b4a85af07b9025d7cf0d9": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.2.0", | |
| "_model_name": "LayoutModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "LayoutView", | |
| "align_content": null, | |
| "align_items": null, | |
| "align_self": null, | |
| "border": null, | |
| "bottom": null, | |
| "display": null, | |
| "flex": null, | |
| "flex_flow": null, | |
| "grid_area": null, | |
| "grid_auto_columns": null, | |
| "grid_auto_flow": null, | |
| "grid_auto_rows": null, | |
| "grid_column": null, | |
| "grid_gap": null, | |
| "grid_row": null, | |
| "grid_template_areas": null, | |
| "grid_template_columns": null, | |
| "grid_template_rows": null, | |
| "height": null, | |
| "justify_content": null, | |
| "justify_items": null, | |
| "left": null, | |
| "margin": null, | |
| "max_height": null, | |
| "max_width": null, | |
| "min_height": null, | |
| "min_width": null, | |
| "object_fit": null, | |
| "object_position": null, | |
| "order": null, | |
| "overflow": null, | |
| "overflow_x": null, | |
| "overflow_y": null, | |
| "padding": null, | |
| "right": null, | |
| "top": null, | |
| "visibility": null, | |
| "width": null | |
| } | |
| }, | |
| "7f540ad08833493db60f23aae7640818": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "HBoxModel", | |
| "state": { | |
| "_dom_classes": [], | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "HBoxModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_view_module_version": "1.5.0", | |
| "_view_name": "HBoxView", | |
| "box_style": "", | |
| "children": [ | |
| "IPY_MODEL_a1493e86ec424e81a4486ffb11b27267", | |
| "IPY_MODEL_d8f640d0ddc04f93a1bed3e6fd5d5c67" | |
| ], | |
| "layout": "IPY_MODEL_60f5701faaeb4cd7bbf48ac0d9eec63c" | |
| } | |
| }, | |
| "a1493e86ec424e81a4486ffb11b27267": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "IntProgressModel", | |
| "state": { | |
| "_dom_classes": [], | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "IntProgressModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_view_module_version": "1.5.0", | |
| "_view_name": "ProgressView", | |
| "bar_style": "success", | |
| "description": "100%", | |
| "description_tooltip": null, | |
| "layout": "IPY_MODEL_b8e279fd96664ba6b0e54273e3ca4585", | |
| "max": 87306240, | |
| "min": 0, | |
| "orientation": "horizontal", | |
| "style": "IPY_MODEL_b2d4ad59d12c4be6a44378bf4b39dcd7", | |
| "value": 87306240 | |
| } | |
| }, | |
| "b2d4ad59d12c4be6a44378bf4b39dcd7": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "ProgressStyleModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "ProgressStyleModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "StyleView", | |
| "bar_color": null, | |
| "description_width": "initial" | |
| } | |
| }, | |
| "b5825f6c2bf54bcb8b361d9aa52b22b0": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.2.0", | |
| "_model_name": "LayoutModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "LayoutView", | |
| "align_content": null, | |
| "align_items": null, | |
| "align_self": null, | |
| "border": null, | |
| "bottom": null, | |
| "display": null, | |
| "flex": null, | |
| "flex_flow": null, | |
| "grid_area": null, | |
| "grid_auto_columns": null, | |
| "grid_auto_flow": null, | |
| "grid_auto_rows": null, | |
| "grid_column": null, | |
| "grid_gap": null, | |
| "grid_row": null, | |
| "grid_template_areas": null, | |
| "grid_template_columns": null, | |
| "grid_template_rows": null, | |
| "height": null, | |
| "justify_content": null, | |
| "justify_items": null, | |
| "left": null, | |
| "margin": null, | |
| "max_height": null, | |
| "max_width": null, | |
| "min_height": null, | |
| "min_width": null, | |
| "object_fit": null, | |
| "object_position": null, | |
| "order": null, | |
| "overflow": null, | |
| "overflow_x": null, | |
| "overflow_y": null, | |
| "padding": null, | |
| "right": null, | |
| "top": null, | |
| "visibility": null, | |
| "width": null | |
| } | |
| }, | |
| "b8e279fd96664ba6b0e54273e3ca4585": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.2.0", | |
| "_model_name": "LayoutModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "LayoutView", | |
| "align_content": null, | |
| "align_items": null, | |
| "align_self": null, | |
| "border": null, | |
| "bottom": null, | |
| "display": null, | |
| "flex": null, | |
| "flex_flow": null, | |
| "grid_area": null, | |
| "grid_auto_columns": null, | |
| "grid_auto_flow": null, | |
| "grid_auto_rows": null, | |
| "grid_column": null, | |
| "grid_gap": null, | |
| "grid_row": null, | |
| "grid_template_areas": null, | |
| "grid_template_columns": null, | |
| "grid_template_rows": null, | |
| "height": null, | |
| "justify_content": null, | |
| "justify_items": null, | |
| "left": null, | |
| "margin": null, | |
| "max_height": null, | |
| "max_width": null, | |
| "min_height": null, | |
| "min_width": null, | |
| "object_fit": null, | |
| "object_position": null, | |
| "order": null, | |
| "overflow": null, | |
| "overflow_x": null, | |
| "overflow_y": null, | |
| "padding": null, | |
| "right": null, | |
| "top": null, | |
| "visibility": null, | |
| "width": null | |
| } | |
| }, | |
| "bbc7411ce46e433b819a277dfd97aab2": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "IntProgressModel", | |
| "state": { | |
| "_dom_classes": [], | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "IntProgressModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_view_module_version": "1.5.0", | |
| "_view_name": "ProgressView", | |
| "bar_style": "success", | |
| "description": "100%", | |
| "description_tooltip": null, | |
| "layout": "IPY_MODEL_e6ddaca7a1cf43e4a6cdeda8fe4374c3", | |
| "max": 46827520, | |
| "min": 0, | |
| "orientation": "horizontal", | |
| "style": "IPY_MODEL_15b42940fc8c47d9b6c583941698832f", | |
| "value": 46827520 | |
| } | |
| }, | |
| "d8f640d0ddc04f93a1bed3e6fd5d5c67": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "HTMLModel", | |
| "state": { | |
| "_dom_classes": [], | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "HTMLModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/controls", | |
| "_view_module_version": "1.5.0", | |
| "_view_name": "HTMLView", | |
| "description": "", | |
| "description_tooltip": null, | |
| "layout": "IPY_MODEL_3693a06704f14253ba15b4e41613de2d", | |
| "placeholder": "", | |
| "style": "IPY_MODEL_ede3695f26ce4ab0815495f04260743f", | |
| "value": " 83.3M/83.3M [00:03<00:00, 27.5MB/s]" | |
| } | |
| }, | |
| "e6ddaca7a1cf43e4a6cdeda8fe4374c3": { | |
| "model_module": "@jupyter-widgets/base", | |
| "model_name": "LayoutModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/base", | |
| "_model_module_version": "1.2.0", | |
| "_model_name": "LayoutModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "LayoutView", | |
| "align_content": null, | |
| "align_items": null, | |
| "align_self": null, | |
| "border": null, | |
| "bottom": null, | |
| "display": null, | |
| "flex": null, | |
| "flex_flow": null, | |
| "grid_area": null, | |
| "grid_auto_columns": null, | |
| "grid_auto_flow": null, | |
| "grid_auto_rows": null, | |
| "grid_column": null, | |
| "grid_gap": null, | |
| "grid_row": null, | |
| "grid_template_areas": null, | |
| "grid_template_columns": null, | |
| "grid_template_rows": null, | |
| "height": null, | |
| "justify_content": null, | |
| "justify_items": null, | |
| "left": null, | |
| "margin": null, | |
| "max_height": null, | |
| "max_width": null, | |
| "min_height": null, | |
| "min_width": null, | |
| "object_fit": null, | |
| "object_position": null, | |
| "order": null, | |
| "overflow": null, | |
| "overflow_x": null, | |
| "overflow_y": null, | |
| "padding": null, | |
| "right": null, | |
| "top": null, | |
| "visibility": null, | |
| "width": null | |
| } | |
| }, | |
| "ede3695f26ce4ab0815495f04260743f": { | |
| "model_module": "@jupyter-widgets/controls", | |
| "model_name": "DescriptionStyleModel", | |
| "state": { | |
| "_model_module": "@jupyter-widgets/controls", | |
| "_model_module_version": "1.5.0", | |
| "_model_name": "DescriptionStyleModel", | |
| "_view_count": null, | |
| "_view_module": "@jupyter-widgets/base", | |
| "_view_module_version": "1.2.0", | |
| "_view_name": "StyleView", | |
| "description_width": "" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 1 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment