F1 score using wandb.sklearn.plot_summary_metrics

sklearn_metrics_logging.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "relative-audio",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mwandb\u001b[0m (use `wandb login --relogin` to force relogin)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import wandb\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "sensitive-federation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Finishing last run (ID:1hrrw9b6) before initializing another..."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<br/>Waiting for W&B process to finish, PID 28252<br/>Program ended successfully."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find user logs for this run at: <code>/Users/morganmcguire/ML/testing/wandb/run-20210406_204831-1hrrw9b6/logs/debug.log</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find internal logs for this run at: <code>/Users/morganmcguire/ML/testing/wandb/run-20210406_204831-1hrrw9b6/logs/debug-internal.log</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                    <br/>Synced <strong style=\"color:#cdcd00\">crimson-smoke-5</strong>: <a href=\"https://wandb.ai/wandb/testing/runs/1hrrw9b6\" target=\"_blank\">https://wandb.ai/wandb/testing/runs/1hrrw9b6</a><br/>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "...Successfully finished last run (ID:1hrrw9b6). Initializing new run:<br/><br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Tracking run with wandb version 0.10.25<br/>\n",
       "                Syncing run <strong style=\"color:#cdcd00\">glowing-gorge-6</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://wandb.ai/wandb/testing\" target=\"_blank\">https://wandb.ai/wandb/testing</a><br/>\n",
       "                Run page: <a href=\"https://wandb.ai/wandb/testing/runs/3imjt0io\" target=\"_blank\">https://wandb.ai/wandb/testing/runs/3imjt0io</a><br/>\n",
       "                Run data is saved locally in <code>/Users/morganmcguire/ML/testing/wandb/run-20210406_204845-3imjt0io</code><br/><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h1>Run(3imjt0io)</h1><iframe src=\"https://wandb.ai/wandb/testing/runs/3imjt0io\" style=\"border:none;width:100%;height:400px\"></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7fbd712ddcd0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn import linear_model\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "wandb.init(project=\"testing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "infinite-gabriel",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import some data to play with\n",
    "# iris = datasets.load_iris()\n",
    "# X = iris.data[:, :2]  # we only take the first two features.\n",
    "# y = iris.target\n",
    "\n",
    "X, y = datasets.make_classification(n_classes=2, random_state=0)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "secret-container",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = linear_model.LogisticRegression()\n",
    "\n",
    "# Fit model\n",
    "classifier.fit(X_train, y_train)\n",
    "\n",
    "# Predict classes\n",
    "y_pred = classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dangerous-ordinance",
   "metadata": {},
   "outputs": [],
   "source": [
    "# wandb logs\n",
    "wandb.sklearn.plot_summary_metrics(classifier, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dangerous-baker",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9032258064516129"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1 = f1_score(y_test, y_pred)\n",
    "wandb.log({'f1': f1})\n",
    "f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "worthy-allowance",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "separate-owner",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import base \n",
    "base.is_classifier(classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "alpine-niger",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "\n",
    "metric_name = []\n",
    "metric_value = []\n",
    "\n",
    "def round_2(n):\n",
    "    return round(n, 2)\n",
    "\n",
    "if base.is_classifier(classifier):\n",
    "    y_pred = classifier.predict(X_test)\n",
    "    y_probas = classifier.predict_proba(X_test)\n",
    "\n",
    "    metric_name.append(\"accuracy_score\")\n",
    "    metric_value.append(round_2(sklearn.metrics.accuracy_score(y_test, y_pred)))\n",
    "    metric_name.append(\"precision\")\n",
    "    metric_value.append(\n",
    "        round_2(\n",
    "            sklearn.metrics.precision_score(y_test, y_pred, average=\"weighted\")\n",
    "        )\n",
    "    )\n",
    "    metric_name.append(\"recall\")\n",
    "    metric_value.append(\n",
    "        round_2(\n",
    "            sklearn.metrics.recall_score(y_test, y_pred, average=\"weighted\")\n",
    "        )\n",
    "    )\n",
    "    metric_name.append(\"f1_score\")\n",
    "    metric_value.append(\n",
    "        round_2(sklearn.metrics.f1_score(y_test, y_pred, average=\"weighted\"))\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "strategic-norway",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['accuracy_score', 'precision', 'recall', 'f1_score'],\n",
       " [0.91, 0.91, 0.91, 0.91])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric_name, metric_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "accepting-links",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9089233347297864, 0.91)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wgt_f1 = f1_score(y_test, y_pred, average=\"weighted\")\n",
    "\n",
    "wgt_f1, round(wgt_f1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "placed-protest",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9032258064516129, 0.9)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_test, y_pred)\n",
    "\n",
    "f1, round(f1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bottom-charger",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}