JasonTam/approx_auc_dist.ipynb

## approx_auc_dist.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "np.random.seed(322)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_obs = 1000000\n",
    "pos = np.random.randn(n_obs)+0.5\n",
    "neg = np.random.randn(n_obs)-0.5\n",
    "scores = np.concatenate([pos, neg])\n",
    "indicator = np.concatenate([\n",
    "    np.ones_like(pos),\n",
    "    np.zeros_like(neg)]).astype(bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.760026926259\n",
      "CPU times: user 594 ms, sys: 110 ms, total: 704 ms\n",
      "Wall time: 724 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "exact_auc = roc_auc_score(indicator, scores)\n",
    "print(exact_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 81.1 ms, sys: 14.8 ms, total: 95.9 ms\n",
      "Wall time: 85.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pos_ind = np.where(indicator)[0]\n",
    "neg_ind = np.where(~indicator)[0]\n",
    "\n",
    "n_samp = 100\n",
    "samp_aucs = []\n",
    "for ii in range(1000):\n",
    "    pos_samp_inds = np.random.choice(pos_ind, n_samp)\n",
    "    neg_samp_inds = np.random.choice(neg_ind, n_samp)\n",
    "    pos_samp = scores[pos_samp_inds]\n",
    "    neg_samp = scores[neg_samp_inds]\n",
    "    samp_auc = (pos_samp > neg_samp).mean()\n",
    "    samp_aucs.append(samp_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x114f99a90>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd0XNdh5/HvHQx6n0EHQRSCRSxiA4tqZMlFbpHjxF4l\nm2STk0TJxont7G4cb5zj7Nmz8XESZ0984jhZrRKnyLK0UW8kRYqkuih2EkRhJwEQlSB6m3L3jxkq\nFE0SAxIzb8rvcw4O2pSfnoAfHu/c+66x1iIiIonD5XQAERGZGxW3iEiCUXGLiCQYFbeISIJRcYuI\nJBgVt4hIglFxi4gkGBW3iEiCUXGLiCQYdzQetKSkxNbV1UXjoUVEktL+/fsHrLWlkdw2KsVdV1fH\nvn37ovHQIiJJyRhzLtLbaqhERCTBqLhFRBKMiltEJMGouEVEEoyKW0Qkwai4RUQSjIpbRCTBqLhF\noiwQtGiLQJlPUVmAI5LKAkHLi4e7+Nf3ztFxaZKLY9NUFWbzxXXV/ML6GhZ6c5yOKAlOxS0yj949\nNcC3XzjGyb4xaoqzWVlVQGF2Oqf6x/mbnSf54e5TfO9Lq/nC2mqno0oCU3GLzJNn9nfyR88coTQ/\nk689sJiN9R5cxnz4/Ytj0/xw9ym+/tQh+ken+a17GxxMK4lMxS0yD364+yR/sbWdldUF/MHHl5CT\n8dO/Wt68TL756WX87a6T/Nmrrbhcht+4u96BtJLo9OKkyC36t30d/MXWdu5c5OWPPrXsmqV9WXqa\ni6/ev5j1tcX8+ZY2jveOxjCpJAsVt8gtONQxxLeea2ZFVQG/e18j7rTZf6VcLsNv3dNAVoaLP3jq\nEL5AMAZJJZmouEVuUv/oNL/9r/soyknnqw8sJs1lZr9TWGF2Or9xVwPHLozwg50no5hSkpGKW+Qm\nffuFZi6N+/gvn1hCQVb6nO+/sd7DnYu8/N3uU/SOTEUhoSQrFbfITdjR0suW5h5+bl01td7cm36c\nLzfV4A8G+fs3Ts1jOkl2Km6RORqf9vPtF5qpKc7mc6sqb+mxyguyuGdxKU/sOU/fqM66JTIqbpE5\n+v7rJ7gwPMVv3tMQ0YuRs/nCmmp8gSCPvnF6HtJJKlBxi8xB19AkP3rnDPctKWVJef68PGZFYRZ3\nNZbw+J5zDIxNz8tjSnJTcYvMwd/uOom18PPrF8zr4z60ppopX5Cn9nbM6+NKclJxi0SoY3CC/7e3\ng48tK6MkL3NeH7u6KJvllfk8ufc8waCuJCg3puIWidAPdp7EGHhodVVUHv/+ZeV0DE7y3umLUXl8\nSR4qbpEIdAxO8PT+Th5YVo53ns+2L9tQ5yE/081PPjgflceX5KHiFonAv7x3Fovlc7ff2vS/G8lw\nu7hrcQnbjvUwOD4TteeRxKfiFpnFxIyfp/Z2sLHeE7Wz7cvuX1qGL2B5Zn9nVJ9HEpuKW2QWzx3s\nYmTKz6dWVET9uWo8OTSW5fHsQRW3XJ+KW+QGrLX80ztnqfPmsHSe5m3P5s5FXlq7RznVPxaT55PE\no+IWuYH3Tl/kRN8Yn1xRgTGRX/3vVmyq92KAlw93x+T5JPGouEVu4Md7zpOf6eauRSUxe05PbgbL\nKvN58XCXdoeXa1Jxi1zH8ISP7cd6ubOxhAx3bH9VNjd4OdU/Trt2yJFrUHGLXMcrR7uZCQS5Z3Hs\nzrYv21jnwWU0XCLXpuIWuY5n9neyoDibhpKbv972zSrKyWB5VQEvHbmg4RL5KSpukWs4OzDO/vOX\nuKexJGYvSl5tc72XcxcnaOvRcIl8lIpb5BqePdCJAe5qjP0wyWXraosBeL2117EMEp9U3CJXsdby\n7IEuVlYXRn2l5I0U52TQWJrL9hYVt3xURMVtjPkDY8wxY0yzMeYnxpisaAcTccqhjiE6hyYdPdu+\nbO3CYg53DmtbM/mIWYvbGFMNfBVostauBNKAh6MdTMQpW5p7cLsMTeGhCietD2fY2drncBKJJ5EO\nlbiBbGOMG8gBLkQvkohzrLW8cqSbVdWF5Ga6nY7DQk8OpXmZbNc4t1xh1uK21nYB3wPOA93AsLX2\ntatvZ4x5xBizzxizr7+/f/6TisTA0a5huoYm2VjvcToKAMYY1tUW8/aJASZnAk7HkTgRyVBJMfAQ\nUA9UAbnGmF+++nbW2kettU3W2qbS0tL5TyoSA68c7SbNZWiqjY/iBli3sIhpf5B3Tg44HUXiRCRD\nJR8Hzlhr+621PuBZ4M7oxhKJPWstrx7pZmVVAXlZzg+TXLa8soDs9DR2aLhEwiIp7vPAZmNMjgmt\nRHgAaI1uLJHYO3ZhhI5Lk2yq9zod5SPcaS5W1xSyvbVXGwkLENkY9x7gaeAAcDR8n0ejnEsk5rYd\n68FlYH2d87NJrra+1sPFsRkOdw45HUXiQESzSqy1f2qtXWatXWmt/RVr7XS0g4nE2vaWXpZW5FOQ\nle50lJ+yZkERLgOva1qgoJWTIkBoF/e2nlHWL4yfFyWvlJflZmlFvlZRCqDiFgH48IW/dbVFDie5\nvvULPbT3jtIxOOF0FHGYilsE2NHSS3VxNpWF2U5Hua7Lf1Q0u0RU3JLyhid97DkzyPqF8fei5JUq\nC7OpLs5mh4ZLUp6KW1LeG8f78Qfth9cFiWfraorYc2aQ0Smf01HEQSpuSXk7WnopyHbTWJrndJRZ\nrVtYjD9oeeuEVlGmMhW3pDR/IMju9r7QdDuXMzvdzMXi8nzyMt2aFpjiVNyS0g51DDEy5WdNTfwP\nkwCkuQyrFxSyq72PgFZRpiwVt6S0Xe19uAysWlDodJSIrV1YzOC4VlGmMhW3pLRdbf0fDj8kitXh\nVZTaXCF1qbglZfWNTNHSPcKamvhddHMtl1dRvt6maYGpSsUtKWv38dCGH2sTrLgB1tYU09o9yoWh\nSaejiANU3JKydrf34clNZ6Enx+koc7YuvFhoZ5uGS1KRiltSki8Q5K0TA6xeUEToMvOJpaooi/KC\nTBV3ilJxS0o6cO4So1N+VifgMAmE9qJcW1PMOye1F2UqUnFLStp9vJ80l2FVdeJMA7za2vBelO+e\n0irKVKPilpS0q62PJeV55GQkzjTAq90W3ovydQ2XpBwVt6ScnuEp2npGE2a15PWkp7lYVV3I6629\nWKtVlKlExS0pZ3d76Aw10eZvX8vahUX0jkzT0j3idBSJIRW3pJzd7f14czOoKY7fTRMidfmPjy46\nlVpU3JJSZvxB3jrZz+qaxJwGeLWinAway/K0K06KUXFLStl/7hLj04GkGCa5rKm2mCOdw3QPaxVl\nqlBxS0rZ3d6H22VYWZW40wCv1lQX2pleW5qlDhW3pJRd7X0srcgnOyPN6Sjzproom6rCLLYdU3Gn\nChW3pIwLQ5Mc7x1LqmGSy9bXFvP+6YsMT2ovylSg4paUsbs9dDXAZCzupjoP/qD9cKqjJDcVt6SM\nXe19lORlUF2U+NMAr9ZYlkdRdjqvabgkJai4JSVM+wO8c3KANUkyDfBqLmNYX1vMrvY+pny66FSy\nU3FLSth39hITM4GEvRpgJJrqipmYCfDeqYtOR5EoU3FLSkjGaYBXW1FVSHZ6Gq+19DgdRaJMxS0p\nYWdbH7dVFpCVnjzTAK+WnuZidU0h21t6CQZ10alkpuKWpNcxOMGp/vGknE1ytaZaDwNjMxzsGHI6\nikSRiluS3q7wFLlE3BR4rtYuLMLtMhouSXIqbkl6O9v6qCjMojIJpwFeLSfDzfKqArY19+ga3UlM\nxS1JbTI8yyIVhkkua6ot5uzFCU71jzkdRaIkouI2xhQZY542xrQZY1qNMXdEO5jIfHj31ADT/mBK\nDJNctr42dNGprc0aLklWkZ5xfx/Yaq1dBqwGWqMXSWT+7GzrIyvdxW2VBU5HiRlPbgZLyvPYouJO\nWrMWtzGmELgX+AcAa+2MtVYvWUvcs9ays62PlVWFpKel1qjgxjovxy6McO7iuNNRJAoi+WmuB/qB\nHxljDhpjHjPG5EY5l8gta+8dpXt4ijULU2eY5LKN9aHhkleP6qw7GUVS3G5gHfB31tq1wDjwzatv\nZIx5xBizzxizr7+/f55jiszdrrbQz+HaBN/N/WaU5meyqDSXV492Ox1FoiCS4u4EOq21e8KfP02o\nyD/CWvuotbbJWttUWlo6nxlFbsrOtl7qvDl4cjOcjuKIjfVejnYN0zE44XQUmWezFre1tgfoMMYs\nDX/pAaAlqqlEbtHwhI8D54ZYk4Jn25dtqtfskmQV6Ss2vw/82BhzBFgDfCd6kURu3Rsn+glYy9oU\nHN++rLwgi/oSDZckI3ckN7LWHgKaopxFZN7sauujIMtNY2me01EctbHew1N7O7gwNElVCqwcTRWp\nNUdKUkIgaNnV3sftC4pwuZJv04S52BTeAV5zupOLiluSzqGOIYYmfCm1zP16KouyqfXmsEXDJUlF\nxS1JZ1dbHy4DqxeouAE21nnYd+4SPcNTTkeReaLilqSzs62PJeX55GVF9BJO0tvU4AVg2zENlyQL\nFbcklc5LE7R0j7BuYepOA7xadVE2NcXZml2SRFTcklRebw1tmtBUq+K+0sZ6Dx+cGaRvVMMlyUDF\nLUnltZYeqopSY9OEudhU78WixTjJQsUtSWNkysee04Os1zDJT6nx5FDjyebFQxecjiLzQMUtSWN3\nez/+oP1wIwH5qM31Xvadu0T38KTTUeQWqbglaexo6aUwO53FZam9WvJ67lgUml3yyhG9SJnoVNyS\nFHyBILva+1hbo9WS11NZmE1DSS4vHtZwSaJTcUtS+ODMIKNTftZrNskN3bHIy5HOYe2Mk+BU3JIU\ntrf0kpHmYtWCQqejxLXN4cU4L2u4JKGpuCXhWWt5raWHVdWFZLrTnI4T10ryMllans9LGi5JaCpu\nSXit3aNcGJrSMEmENjd4aesZ5UTvqNNR5CapuCXhbW/pxUBKb5owF5sbPLgMvKThkoSl4paEt721\nh8XleRTlpObeknNVlJPB8soCXj58AWut03HkJqi4JaF1D0/S3KWLSs3V5gYvpwfGaekecTqK3AQV\ntyS0HR9eVEqrJediQ72HNJfhpcMaLklEKm5JaNtbeqkszKKqKMvpKAmlICudVdWFvKThkoSk4paE\nNTrl491TA6xbWIwxWi05V3c0eOkamuTA+SGno8gcqbglYb15fAB/wOra2zepqa6YjDQXLx7qcjqK\nzJGKWxLWjtZe8rPcLC7PdzpKQsrJcLN2YREvHenGHwg6HUfmQMUtCckXCLKzLXRRqTRdVOqm3dVY\nwuD4DO+cuuh0FJkDFbckpH1nLzE86dO1t2/RmpoicjLStMFCglFxS0La3tJLeprhdl1U6pakp7nY\nVO9h67FupnwBp+NIhFTcknCstWxv6WFldSFZ6bqo1K26c1EJ49MBdrb1OR1FIqTiloTT1jNKx6VJ\nXVRqniyvLKA4J50XNLskYai4JeFsbe7BoNWS88XlMtzR4GVnWx/Dkz6n40gEVNyScLYe62FZZT6F\n2elOR0kadzaW4AtYtjX3OB1FIqDiloRydmCc9p5RnW3Ps4aSXCoKsnhewyUJQcUtCWXbsdAZ4YY6\nFfd8MsZwZ6OX905dpG9kyuk4MgsVtySUrc091JfkUpqf6XSUpHPnohIs2mAhEai4JWH0jkxxsGNI\nZ9tRUl2UTX1JLi8c1HBJvFNxS8J4LTxMslHFHTV3LvJypGuYMwPjTkeRG4i4uI0xacaYg8aYl6MZ\nSOR6tjb3UFWURXVxttNRktYdDV4MaAl8nJvLGffXgNZoBRG5kaGJGd4/Pahhkijz5mVyW2UBzx/q\n0gYLcSyi4jbGLAA+CzwW3Tgi17ajtY+AtSruGLiz0cuZgXGOXdB+lPEq0jPuvwa+AeiiveKIrc09\neHMzaCjJdTpK0ttU58XtMloCH8dmLW5jzOeAPmvt/llu94gxZp8xZl9/f/+8BRQZn/bz1ol+NtR5\ntEVZDORlubl9QREvHLpAIKjhkngUyRn3XcDPGmPOAk8C9xtjHr/6RtbaR621TdbaptLS0nmOKans\njeP9TPuDbKjTRaVi5a5GL32j03xwZtDpKHINsxa3tfa/W2sXWGvrgIeBndbaX456MpGwrc09FGS5\nWVpR4HSUlLFuYTFZbhcvHtZwSTzSPG6Ja1O+ADtae1lf69EWZTGUlZ7G+joPrxztZtqvDRbizZyK\n21q721r7uWiFEbna7vY+JmYC3LHI63SUlHPXIi8jk37ePD7gdBS5is64Ja69fKSbwux0lldqmCTW\nVi0oJD/LzYuHtRgn3qi4JW5NzPh5vbWPDXXFGiZxgNvlYlO9l+3Hehif9jsdR66g4pa4tbOtj0lf\ngDsaNEzilLsavUz5g2xv6XU6ilxBxS1x65Uj3RTnpLNMs0kcs6Q8n5K8DF7UYpy4ouKWuDQ27Wdn\nWx8b6jy4NEziGJcxbKz38uaJAUamtB9lvFBxS1x6vbWXaX9QwyRxYFO9B3/QskPDJXFDxS1x6eUj\n3XhzM1hSke90lJTXWJaHNzeDV49qZ5x4oeKWuDMy5WN3ex8b6z24dG0Sx4WGSzy8eXyAUQ2XxAUV\nt8SdHS29+AKWzRomiRub6r3MBIK83trndBRBxS1x6OUj3ZTkZbC4LM/pKBK2uDwPj4ZL4oaKW+LK\n8ISPN4/3s7nBq0u4xhGXMWys87C7vZ8xLcZxnIpb4sq2lh78QQ2TxKNN9Z7wcIlmlzhNxS1x5YWD\nXZQXZGqnmzi0pCKf4px0tmi4xHEqbokb3cOTvHvqInc3lmiYJA65jGFDnYdd7f26donDVNwSN144\ndAEL3N2oHZTi1aYGL9P+ILvaNbvESSpuiQvWWp7Z38mS8jwqCrOcjiPXsaw8n6LsdM0ucZiKW+LC\nsQsjnOgb09l2nHO5DBvqPexs7WNiRsMlTlFxS1x47mAXbpfRtUkSwKZ6D1P+ILva+p2OkrJU3OI4\nXyDIC4e6WLuwiLwst9NxZBa3VRRQkO1m67Eep6OkLBW3OG5XWx8DYzP8zJIyp6NIBFwuw/qFHl5v\n7WXKp42EnaDiFsc9ubeD4px01tQUOR1FIrSx3sPETIB3T2kjYSeouMVRPcNT7G7v494lpdpXMoGs\nrCogNyONLUc1XOIEFbc46pkDnQQt3KdhkoTiTnOxdmExr7X04g8EnY6TclTc4phg0PLU3g6WVxZo\n7nYC2ljnYXjSxwdnBp2OknJU3OKY989c5PzgBPct1dztRHR7TSGZbhdbmjVcEmsqbnHM4++fIz/T\nzaZ6zd1ORJnuNFbXFLH1WA/BoHU6TkpRcYsjeoan2Nbcy88sLSXDrR/DRLWxzkP/6DQHO4acjpJS\n9Bsjjnjig/MEreXjt5U7HUVuwdqFRbhdhq3NunZJLKm4JeZm/EF+suc8q2uKKC/Qi5KJLCfDzarq\nQrY092CthktiRcUtMbftWA/9Y9N8YrnOtpPBhnoPnZcmaekecTpKylBxS8z987tnKcvPZM0CrZRM\nButri3EZ2KrZJTGj4paYOtQxxL5zl/jUigpcWimZFAqy0rmtskDTAmNIxS0x9dhbp8nJSNPc7SSz\nsc7Dyb4xTvaNOR0lJai4JWY6L02w5WgP9y8rIydDl29NJk11HgxoZ5wYUXFLzPzonbNg4MEVFU5H\nkXnmyc1gWWU+Lx+54HSUlDBrcRtjaowxu4wxLcaYY8aYr8UimCSXkSkfT+49z+Z6D968TKfjSBRs\nrvdyvHeME72jTkdJepGccfuB/2qtXQ5sBr5ijFke3ViSbP7l3bOMTwf47O1VTkeRKNlYHxoueUXD\nJVE3a3Fba7uttQfCH48CrUB1tINJ8piY8fPY22dYW1NEfUmu03EkSopyLg+XqLijbU5j3MaYOmAt\nsCcaYSQ5PbHnPEMTPr6wVn/vk93mBi8n+8Y4ruGSqIq4uI0xecAzwNettT+1RMoY84gxZp8xZl9/\nv3Z/lpBpf4BH3zzN8soClpTnOx1HomxjnQeXgZcP60XKaIqouI0x6YRK+8fW2mevdRtr7aPW2iZr\nbVNpqeboSsjT+zvpG53moTUa204FRTkZLK8q4PlDF3TtkiiKZFaJAf4BaLXW/u/oR5JkMe0P8IOd\nJ2ksy2NVdaHTcSRG7m4s4fzgBAfO61Kv0RLJGfddwK8A9xtjDoXfPhPlXJIEntrbQffwFF9av4DQ\n339JBRvqPGS6XTx/sMvpKEkrklklb1trjbX2dmvtmvDbq7EIJ4lryhfgb3aeZFlFvs62U0xOhpt1\ntcW8ePgCM35tJBwNWjkpUfH4++foH53mS001OttOQXc3ljA86eON45qoEA0qbpl3o1M+frj7FCuq\nClheWeB0HHHA7QsKKch289zBTqejJCUVt8y7//PGaQbHZ3h4w0Kno4hD3C4XdzSUsKOlj6GJGafj\nJB0Vt8yrnuEpHnv7NHcs8tJYlud0HHHQx5aWMhMI8swBvUg531TcMq/+6rV2/EHLw001TkcRh9V6\nc2ksy+OJPec0p3ueqbhl3rR2j/D0/k4+ubyCMm0CLMADy8o41T/O3rOXnI6SVFTcMm++82oruZlu\nfm6NrkkiIZsbvORkpPHEnnNOR0kqKm6ZF28e7+etEwN8YU01eVna3UZCstLTuLuxhFeP9nBpXC9S\nzhcVt9yyQNDynVdbKcvP5JMryp2OI3Hm/mVlzASCPLWvw+koSUPFLbfsmQOdtPWM8h821JCeph8p\n+ahaby4rqwv4x7fPMO0POB0nKei3TG7J6JSPv9zaTmNZHnc0eJ2OI3Hq87dX0Tc6zQuHdLnX+aDi\nllvy/R0nGBib5j/dUael7XJdq6oLqfXm8OibpwkGNTXwVqm45aYd7x3lR++c5b6lZVpsIzdkjOHz\nt1dxsm+MXe19TsdJeCpuuSnWWv70hWNkZ6Tx8EYttpHZbWrwUJqfyQ92ntSCnFuk4pab8tzBLt47\nfZEvNy2gICvd6TiSANwuFw+tqeJgxxDbjvU6HSehqbhlzi6OTfM/X25hSXkeDyzT9D+J3H1Lyqgu\nzubPt7bhC+ha3TdLxS1z9r9eaWVsys9v3t2Ay6UXJCVyaS7DwxtqODMwzlN7Na/7Zqm4ZU52t/fx\n3MEufnZ1FTWeHKfjSAJav7CYZRX5/PWO44xN+52Ok5BU3BKxoYkZvvH0EaqLs3lI1yORm2SM4T9u\nquXi2Azf3dLqdJyEpOKWiFhr+dbzzVwcn+Er9zWS4daPjty8xrI8Pr2qksffP8+7JwecjpNw9Nsn\nEXnh0AVeOdLNL6xbQH1JrtNxJAl8uWkBlYVZfOPpI4xryGROVNwyq9P9Y/zJ880sKc/jZ1dXOR1H\nkkSmO41H7m2ga2iSb79wTHO750DFLTc0Pu3nt/91P8bA79+/WLNIZF4tqyjgi+sW8MyBTh5764zT\ncRKGiluuy1rLHz1zhFP9Y/z+/Yspyct0OpIkoS+uq2ZTvYfvvNrKzjYtzImEiluu6wc7T/LykW6+\n3FTDqupCp+NIknIZw3++bxF1Jbl85YmDvKMXK2el4pZremLPef5q+3HuWVyicW2Jukx3Gt/41FJK\n8jL4tR99wNbmHqcjxTUVt/yULUe7+ZPnj7K2pohH7m3Q5VolJopyMvj2Z1dQ683ld3+8n79/45Qu\nAXsdKm75iKf3d/J7TxyksSyPrz6wGLdLPyISO3lZbr71mdtoqvXw3S1t/NL/fZ+uoUmnY8Ud/VYK\nEHoh8rG3TvPf/u0wy6sK+OaDt5GVnuZ0LElBWelpfP3ji3nk3gYOdQ5x//d282evtDCozYY/pO24\nhfFpP3/yfDPPHexiU72Hr3ysUXtHiqOMMXxsaRkrqwr4t/2d/MPbZ3h8z3k+t6qSL2+ooam2OKWH\n8Ew0Jr03NTXZffv2zfvjyvxr7hrma08e5HT/OD+/fgE/t6Zac7Ul7nRdmuTlIxfYc+Yik74gpfmZ\n3Lu4lHuXlHBXY0lSTFU1xuy31jZFdFsVd2oanvDxvdfaefz9cxTmpPOV+xpZqSl/EuemfAE+ODPI\noY4hmruGGQ0vlV9RVcDdi0u4p7GUprrihBzmU3HLdV0cm+af3j3LP797lrFpP59YXsGX1i8gN1Oj\nZpJYgkHLmYvjHOkc5mjXEMd7xwgELZluFxvqPPzMklI+uaKcWm9iXFtHxS0f4Q8EefvkAM8d7GJb\ncw/T/iBNdcX8/LoFCfNDLTKbKV+Alu4RjnYN09w1TOel0GyU2yry+fSqSj69soLF5fkOp7w+FXeK\nm/IFaO0e4UjnMO+cHOC90xcZnfKTl+lmc4OHB1dUUl2c7XRMkajqH51m79lB9py5yPHeMQAWleby\n2VWVfHpVJcsq8uPqBc55L25jzIPA94E04DFr7XdvdHsVd/RM+wMMT/gYGJuhb3SKvpHp0PvRac4P\nTnBmYJzOwUkC4f+vpfmZrKouZG1NEatrijRbRFLS4PgM+84OsufMIG09IwQt1Jfk8plVFXxyeQWr\nqgsdf1F+XovbGJMGHAc+AXQCe4FftNa2XO8+Ku7ZTfkCDE34GJqcCb2fCL+f9DE04WP4w6/7uBT+\n3vCkj0lf4JqPl5fppjQ/k7L8TKqKsqn35lJfmos3NyOuzipEnDY0McPes5f44OxFWi6ESrw4J517\nFpeyod7D+oXFLCnPwx3jk5y5FHckr0htBE5aa0+HH/xJ4CHgusU9H6y1BIKWgLVYy4cfB4MWf9Ay\nORNgyhdgYibApC/8NhP+fMbPxEyA8fDHofcBxqf9BIKWoLVYIGhDz5PpTiM7I40st4us9Cs+zkgj\nI81FpttFepqLjCveZ4T/p84EgvgDFl8gyEwgyPi0n9EpP6NTPkYm/YxO+xid8jMy6Qt/3c/IlI9p\n//V3uHa7DAVZ6eRmpZGX4SYv0015QRZ5maGP87Lc5Ge5Kc7JoDgnncLsDO1IIxKhopwMPrG8nE8s\nL2dk0seRrmEOdwzx5ol+Xjx8AYD0NENjWR4NpXlUFGRRXpBJeUEWZflZFOWkk52eFuqK9DQy00N9\nEMsz9kiKuxq4cjvmTmBTNMLc/j+2MT4TIDCP1yfISnd9eJAz3S7SXAZX+Az08vuZQJAZf5BpfyD8\nPnjDYo3mgZJKAAAEtklEQVToed0ucjPd5GSkkZPhJjczjaqibHIz0sjJDBVvfmZ66H34LS/8eabb\npbNkkRjIy3RTVZTNgysqsNbSOzJNa88IZwfGOTc4wYFzl7g0PsNUhH1QXpDJnj/+eJRTz+PKSWPM\nI8Aj4U/HjDHt8/XYN1ACxPM1IJXv1sV7xnjPB/GfMWnynQPMt276eWojvWEkxd0F1Fzx+YLw1z7C\nWvso8GikTzwfjDH7Ih0TcoLy3bp4zxjv+SD+Myrf3EUyMLoXWGyMqTfGZAAPAy9GN5aIiFzPrGfc\n1lq/Meb3gG2EpgP+o7X2WNSTiYjINUU0xm2tfRV4NcpZbkZMh2ZugvLdunjPGO/5IP4zKt8cRWXl\npIiIRI8m/4qIJJi4LG5jzIPGmHZjzEljzDev8f37jDHDxphD4bdvX/G9s8aYo+GvR2X55mz5rsh4\nyBhzzBjzxlzuGwcZHT+Gxpg/vOL/b7MxJmCM8UT63xYHGePhGBYaY14yxhwO/z/+9UjvGwf5on78\nIsxYbIx5zhhzxBjzgTFmZaT3jSprbVy9EXoB9BTQAGQAh4HlV93mPuDl69z/LFDicL4iQitLF4Y/\nL4v0vk5njJdjeNXtPw/sjLdjeL2M8XIMgT8G/jz8cSkwGL5t1I/hreSLxfGbQ8a/BP40/PEy4PVY\n/hxe7y0ez7g/XGJvrZ0BLi+xjxeR5Psl4Flr7XkAa23fHO7rdMZYmOtx+EXgJzd5XycyxkIk+SyQ\nb0LLcPMIFaM/wvs6mS9WIsm4HNgJYK1tA+qMMeUR3jdq4rG4r7XEvvoat7sz/M+XLcaYFVd83QI7\njDH7w6s5nci3BCg2xuwO5/jVOdzX6YwQH8cQAGNMDvAg8Mxc7+tgRoiPY/gD4DbgAnAU+Jq1Nhjh\nfZ3MB9E/fpFmPAx8EcAYs5HQ6sYFEd43ahJ125MDhP6JP2aM+QzwPLA4/L27rbVdxpgyYLsxps1a\n+2aM87mB9cADQDbwnjHm/RhnmM01M1prjxMfx/CyzwPvWGsHHXr+SFwrYzwcw08Bh4D7gUXhHG/F\nOMONXDOftXaE+Dh+AN8Fvm+MOUToj8tB4NqX6IyheDzjnnWJvbV2xFo7Fv74VSDdGFMS/rwr/L4P\neI7QP2limo/QX99t1tpxa+0A8CawOsL7Op0xXo7hZQ/z0SGIeDqGl12dMV6O4a8TGg6z1tqTwBlC\n47SxOIa3ki8Wxy+ijOGu+XVr7RrgVwmNxZ+O5L5RFavB9EjfCJ0Jngbq+fdB/xVX3aaCf5+DvhE4\nDxggF8gPfz0XeBd40IF8twGvh2+bAzQDKyO5bxxkjItjGL5dIaFxz9y53tfhjHFxDIG/A/5H+ONy\nQsVSEotjeIv5on785pCxiH9/wfS3gH+J5c/hdbPH6onmeEA/Q2jzhlPAt8Jf+x3gd8If/x5wLHyw\n3gfuDH+9Ify1w+Hvf8uJfOHP/5DQrI1m4Os3um88ZYyzY/hrwJOR3DeeMsbLMQSqgNcI/RO/Gfjl\nWB7Dm80Xq+MXYcY7wt9vB54FimP9c3itN62cFBFJMPE4xi0iIjeg4hYRSTAqbhGRBKPiFhFJMCpu\nEZEEo+IWEUkwKm4RkQSj4hYRSTD/Hxm+OGyJKG2WAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114fac908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.kdeplot(samp_aucs, shade=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cautionary Note: Obviously, this is a special case where the positive & negative scores are balanced and drawn from unit-scale normal distributions with different means"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [ipykernel_py3]",
   "language": "python",
   "name": "Python [ipykernel_py3]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
	{
	"cells": [
	{
	"cell_type": "code",
	"execution_count": 1,
	"metadata": {},
	"outputs": [],
	"source": [
	"import numpy as np\n",
	"from sklearn.metrics import roc_auc_score\n",
	"import matplotlib.pyplot as plt\n",
	"import seaborn as sns\n",
	"%matplotlib inline\n",
	"\n",
	"np.random.seed(322)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 2,
	"metadata": {
	"collapsed": true
	},
	"outputs": [],
	"source": [
	"n_obs = 1000000\n",
	"pos = np.random.randn(n_obs)+0.5\n",
	"neg = np.random.randn(n_obs)-0.5\n",
	"scores = np.concatenate([pos, neg])\n",
	"indicator = np.concatenate([\n",
	" np.ones_like(pos),\n",
	" np.zeros_like(neg)]).astype(bool)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 3,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"0.760026926259\n",
	"CPU times: user 594 ms, sys: 110 ms, total: 704 ms\n",
	"Wall time: 724 ms\n"
	]
	}
	],
	"source": [
	"%%time\n",
	"exact_auc = roc_auc_score(indicator, scores)\n",
	"print(exact_auc)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 4,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"CPU times: user 81.1 ms, sys: 14.8 ms, total: 95.9 ms\n",
	"Wall time: 85.1 ms\n"
	]
	}
	],
	"source": [
	"%%time\n",
	"pos_ind = np.where(indicator)[0]\n",
	"neg_ind = np.where(~indicator)[0]\n",
	"\n",
	"n_samp = 100\n",
	"samp_aucs = []\n",
	"for ii in range(1000):\n",
	" pos_samp_inds = np.random.choice(pos_ind, n_samp)\n",
	" neg_samp_inds = np.random.choice(neg_ind, n_samp)\n",
	" pos_samp = scores[pos_samp_inds]\n",
	" neg_samp = scores[neg_samp_inds]\n",
	" samp_auc = (pos_samp > neg_samp).mean()\n",
	" samp_aucs.append(samp_auc)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 5,
	"metadata": {},
	"outputs": [
	{
	"data": {
	"text/plain": [
	"<matplotlib.axes._subplots.AxesSubplot at 0x114f99a90>"
	]
	},
	"execution_count": 5,
	"metadata": {},
	"output_type": "execute_result"
	},
	{
	"data": {
	"image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd0XNdh5/HvHQx6n0EHQRSCRSxiA4tqZMlFbpHjxF4l\nm2STk0TJxont7G4cb5zj7Nmz8XESZ0984jhZrRKnyLK0UW8kRYqkuih2EkRhJwEQlSB6m3L3jxkq\nFE0SAxIzb8rvcw4O2pSfnoAfHu/c+66x1iIiIonD5XQAERGZGxW3iEiCUXGLiCQYFbeISIJRcYuI\nJBgVt4hIglFxi4gkGBW3iEiCUXGLiCQYdzQetKSkxNbV1UXjoUVEktL+/fsHrLWlkdw2KsVdV1fH\nvn37ovHQIiJJyRhzLtLbaqhERCTBqLhFRBKMiltEJMGouEVEEoyKW0Qkwai4RUQSjIpbRCTBqLhF\noiwQtGiLQJlPUVmAI5LKAkHLi4e7+Nf3ztFxaZKLY9NUFWbzxXXV/ML6GhZ6c5yOKAlOxS0yj949\nNcC3XzjGyb4xaoqzWVlVQGF2Oqf6x/mbnSf54e5TfO9Lq/nC2mqno0oCU3GLzJNn9nfyR88coTQ/\nk689sJiN9R5cxnz4/Ytj0/xw9ym+/tQh+ken+a17GxxMK4lMxS0yD364+yR/sbWdldUF/MHHl5CT\n8dO/Wt68TL756WX87a6T/Nmrrbhcht+4u96BtJLo9OKkyC36t30d/MXWdu5c5OWPPrXsmqV9WXqa\ni6/ev5j1tcX8+ZY2jveOxjCpJAsVt8gtONQxxLeea2ZFVQG/e18j7rTZf6VcLsNv3dNAVoaLP3jq\nEL5AMAZJJZmouEVuUv/oNL/9r/soyknnqw8sJs1lZr9TWGF2Or9xVwPHLozwg50no5hSkpGKW+Qm\nffuFZi6N+/gvn1hCQVb6nO+/sd7DnYu8/N3uU/SOTEUhoSQrFbfITdjR0suW5h5+bl01td7cm36c\nLzfV4A8G+fs3Ts1jOkl2Km6RORqf9vPtF5qpKc7mc6sqb+mxyguyuGdxKU/sOU/fqM66JTIqbpE5\n+v7rJ7gwPMVv3tMQ0YuRs/nCmmp8gSCPvnF6HtJJKlBxi8xB19AkP3rnDPctKWVJef68PGZFYRZ3\nNZbw+J5zDIxNz8tjSnJTcYvMwd/uOom18PPrF8zr4z60ppopX5Cn9nbM6+NKclJxi0SoY3CC/7e3\ng48tK6MkL3NeH7u6KJvllfk8ufc8waCuJCg3puIWidAPdp7EGHhodVVUHv/+ZeV0DE7y3umLUXl8\nSR4qbpEIdAxO8PT+Th5YVo53ns+2L9tQ5yE/081PPjgflceX5KHiFonAv7x3Fovlc7ff2vS/G8lw\nu7hrcQnbjvUwOD4TteeRxKfiFpnFxIyfp/Z2sLHeE7Wz7cvuX1qGL2B5Zn9nVJ9HEpuKW2QWzx3s\nYmTKz6dWVET9uWo8OTSW5fHsQRW3XJ+KW+QGrLX80ztnqfPmsHSe5m3P5s5FXlq7RznVPxaT55PE\no+IWuYH3Tl/kRN8Yn1xRgTGRX/3vVmyq92KAlw93x+T5JPGouEVu4Md7zpOf6eauRSUxe05PbgbL\nKvN58XCXdoeXa1Jxi1zH8ISP7cd6ubOxhAx3bH9VNjd4OdU/Trt2yJFrUHGLXMcrR7uZCQS5Z3Hs\nzrYv21jnwWU0XCLXpuIWuY5n9neyoDibhpKbv972zSrKyWB5VQEvHbmg4RL5KSpukWs4OzDO/vOX\nuKexJGYvSl5tc72XcxcnaOvRcIl8lIpb5BqePdCJAe5qjP0wyWXraosBeL2117EMEp9U3CJXsdby\n7IEuVlYXRn2l5I0U52TQWJrL9hYVt3xURMVtjPkDY8wxY0yzMeYnxpisaAcTccqhjiE6hyYdPdu+\nbO3CYg53DmtbM/mIWYvbGFMNfBVostauBNKAh6MdTMQpW5p7cLsMTeGhCietD2fY2drncBKJJ5EO\nlbiBbGOMG8gBLkQvkohzrLW8cqSbVdWF5Ga6nY7DQk8OpXmZbNc4t1xh1uK21nYB3wPOA93AsLX2\ntatvZ4x5xBizzxizr7+/f/6TisTA0a5huoYm2VjvcToKAMYY1tUW8/aJASZnAk7HkTgRyVBJMfAQ\nUA9UAbnGmF+++nbW2kettU3W2qbS0tL5TyoSA68c7SbNZWiqjY/iBli3sIhpf5B3Tg44HUXiRCRD\nJR8Hzlhr+621PuBZ4M7oxhKJPWstrx7pZmVVAXlZzg+TXLa8soDs9DR2aLhEwiIp7vPAZmNMjgmt\nRHgAaI1uLJHYO3ZhhI5Lk2yq9zod5SPcaS5W1xSyvbVXGwkLENkY9x7gaeAAcDR8n0ejnEsk5rYd\n68FlYH2d87NJrra+1sPFsRkOdw45HUXiQESzSqy1f2qtXWatXWmt/RVr7XS0g4nE2vaWXpZW5FOQ\nle50lJ+yZkERLgOva1qgoJWTIkBoF/e2nlHWL4yfFyWvlJflZmlFvlZRCqDiFgH48IW/dbVFDie5\nvvULPbT3jtIxOOF0FHGYilsE2NHSS3VxNpWF2U5Hua7Lf1Q0u0RU3JLyhid97DkzyPqF8fei5JUq\nC7OpLs5mh4ZLUp6KW1LeG8f78Qfth9cFiWfraorYc2aQ0Smf01HEQSpuSXk7WnopyHbTWJrndJRZ\nrVtYjD9oeeuEVlGmMhW3pDR/IMju9r7QdDuXMzvdzMXi8nzyMt2aFpjiVNyS0g51DDEy5WdNTfwP\nkwCkuQyrFxSyq72PgFZRpiwVt6S0Xe19uAysWlDodJSIrV1YzOC4VlGmMhW3pLRdbf0fDj8kitXh\nVZTaXCF1qbglZfWNTNHSPcKamvhddHMtl1dRvt6maYGpSsUtKWv38dCGH2sTrLgB1tYU09o9yoWh\nSaejiANU3JKydrf34clNZ6Enx+koc7YuvFhoZ5uGS1KRiltSki8Q5K0TA6xeUEToMvOJpaooi/KC\nTBV3ilJxS0o6cO4So1N+VifgMAmE9qJcW1PMOye1F2UqUnFLStp9vJ80l2FVdeJMA7za2vBelO+e\n0irKVKPilpS0q62PJeV55GQkzjTAq90W3ovydQ2XpBwVt6ScnuEp2npGE2a15PWkp7lYVV3I6629\nWKtVlKlExS0pZ3d76Aw10eZvX8vahUX0jkzT0j3idBSJIRW3pJzd7f14czOoKY7fTRMidfmPjy46\nlVpU3JJSZvxB3jrZz+qaxJwGeLWinAway/K0K06KUXFLStl/7hLj04GkGCa5rKm2mCOdw3QPaxVl\nqlBxS0rZ3d6H22VYWZW40wCv1lQX2pleW5qlDhW3pJRd7X0srcgnOyPN6Sjzproom6rCLLYdU3Gn\nChW3pIwLQ5Mc7x1LqmGSy9bXFvP+6YsMT2ovylSg4paUsbs9dDXAZCzupjoP/qD9cKqjJDcVt6SM\nXe19lORlUF2U+NMAr9ZYlkdRdjqvabgkJai4JSVM+wO8c3KANUkyDfBqLmNYX1vMrvY+pny66FSy\nU3FLSth39hITM4GEvRpgJJrqipmYCfDeqYtOR5EoU3FLSkjGaYBXW1FVSHZ6Gq+19DgdRaJMxS0p\nYWdbH7dVFpCVnjzTAK+WnuZidU0h21t6CQZ10alkpuKWpNcxOMGp/vGknE1ytaZaDwNjMxzsGHI6\nikSRiluS3q7wFLlE3BR4rtYuLMLtMhouSXIqbkl6O9v6qCjMojIJpwFeLSfDzfKqArY19+ga3UlM\nxS1JbTI8yyIVhkkua6ot5uzFCU71jzkdRaIkouI2xhQZY542xrQZY1qNMXdEO5jIfHj31ADT/mBK\nDJNctr42dNGprc0aLklWkZ5xfx/Yaq1dBqwGWqMXSWT+7GzrIyvdxW2VBU5HiRlPbgZLyvPYouJO\nWrMWtzGmELgX+AcAa+2MtVYvWUvcs9ays62PlVWFpKel1qjgxjovxy6McO7iuNNRJAoi+WmuB/qB\nHxljDhpjHjPG5EY5l8gta+8dpXt4ijULU2eY5LKN9aHhkleP6qw7GUVS3G5gHfB31tq1wDjwzatv\nZIx5xBizzxizr7+/f55jiszdrrbQz+HaBN/N/WaU5meyqDSXV492Ox1FoiCS4u4EOq21e8KfP02o\nyD/CWvuotbbJWttUWlo6nxlFbsrOtl7qvDl4cjOcjuKIjfVejnYN0zE44XQUmWezFre1tgfoMMYs\nDX/pAaAlqqlEbtHwhI8D54ZYk4Jn25dtqtfskmQV6Ss2vw/82BhzBFgDfCd6kURu3Rsn+glYy9oU\nHN++rLwgi/oSDZckI3ckN7LWHgKaopxFZN7sauujIMtNY2me01EctbHew1N7O7gwNElVCqwcTRWp\nNUdKUkIgaNnV3sftC4pwuZJv04S52BTeAV5zupOLiluSzqGOIYYmfCm1zP16KouyqfXmsEXDJUlF\nxS1JZ1dbHy4DqxeouAE21nnYd+4SPcNTTkeReaLilqSzs62PJeX55GVF9BJO0tvU4AVg2zENlyQL\nFbcklc5LE7R0j7BuYepOA7xadVE2NcXZml2SRFTcklRebw1tmtBUq+K+0sZ6Dx+cGaRvVMMlyUDF\nLUnltZYeqopSY9OEudhU78WixTjJQsUtSWNkysee04Os1zDJT6nx5FDjyebFQxecjiLzQMUtSWN3\nez/+oP1wIwH5qM31Xvadu0T38KTTUeQWqbglaexo6aUwO53FZam9WvJ67lgUml3yyhG9SJnoVNyS\nFHyBILva+1hbo9WS11NZmE1DSS4vHtZwSaJTcUtS+ODMIKNTftZrNskN3bHIy5HOYe2Mk+BU3JIU\ntrf0kpHmYtWCQqejxLXN4cU4L2u4JKGpuCXhWWt5raWHVdWFZLrTnI4T10ryMllans9LGi5JaCpu\nSXit3aNcGJrSMEmENjd4aesZ5UTvqNNR5CapuCXhbW/pxUBKb5owF5sbPLgMvKThkoSl4paEt721\nh8XleRTlpObeknNVlJPB8soCXj58AWut03HkJqi4JaF1D0/S3KWLSs3V5gYvpwfGaekecTqK3AQV\ntyS0HR9eVEqrJediQ72HNJfhpcMaLklEKm5JaNtbeqkszKKqKMvpKAmlICudVdWFvKThkoSk4paE\nNTrl491TA6xbWIwxWi05V3c0eOkamuTA+SGno8gcqbglYb15fAB/wOra2zepqa6YjDQXLx7qcjqK\nzJGKWxLWjtZe8rPcLC7PdzpKQsrJcLN2YREvHenGHwg6HUfmQMUtCckXCLKzLXRRqTRdVOqm3dVY\nwuD4DO+cuuh0FJkDFbckpH1nLzE86dO1t2/RmpoicjLStMFCglFxS0La3tJLeprhdl1U6pakp7nY\nVO9h67FupnwBp+NIhFTcknCstWxv6WFldSFZ6bqo1K26c1EJ49MBdrb1OR1FIqTiloTT1jNKx6VJ\nXVRqniyvLKA4J50XNLskYai4JeFsbe7BoNWS88XlMtzR4GVnWx/Dkz6n40gEVNyScLYe62FZZT6F\n2elOR0kadzaW4AtYtjX3OB1FIqDiloRydmCc9p5RnW3Ps4aSXCoKsnhewyUJQcUtCWXbsdAZ4YY6\nFfd8MsZwZ6OX905dpG9kyuk4MgsVtySUrc091JfkUpqf6XSUpHPnohIs2mAhEai4JWH0jkxxsGNI\nZ9tRUl2UTX1JLi8c1HBJvFNxS8J4LTxMslHFHTV3LvJypGuYMwPjTkeRG4i4uI0xacaYg8aYl6MZ\nSOR6tjb3UFWURXVxttNRktYdDV4MaAl8nJvLGffXgNZoBRG5kaGJGd4/Pahhkijz5mVyW2UBzx/q\n0gYLcSyi4jbGLAA+CzwW3Tgi17ajtY+AtSruGLiz0cuZgXGOXdB+lPEq0jPuvwa+AeiiveKIrc09\neHMzaCjJdTpK0ttU58XtMloCH8dmLW5jzOeAPmvt/llu94gxZp8xZl9/f/+8BRQZn/bz1ol+NtR5\ntEVZDORlubl9QREvHLpAIKjhkngUyRn3XcDPGmPOAk8C9xtjHr/6RtbaR621TdbaptLS0nmOKans\njeP9TPuDbKjTRaVi5a5GL32j03xwZtDpKHINsxa3tfa/W2sXWGvrgIeBndbaX456MpGwrc09FGS5\nWVpR4HSUlLFuYTFZbhcvHtZwSTzSPG6Ja1O+ADtae1lf69EWZTGUlZ7G+joPrxztZtqvDRbizZyK\n21q721r7uWiFEbna7vY+JmYC3LHI63SUlHPXIi8jk37ePD7gdBS5is64Ja69fKSbwux0lldqmCTW\nVi0oJD/LzYuHtRgn3qi4JW5NzPh5vbWPDXXFGiZxgNvlYlO9l+3Hehif9jsdR66g4pa4tbOtj0lf\ngDsaNEzilLsavUz5g2xv6XU6ilxBxS1x65Uj3RTnpLNMs0kcs6Q8n5K8DF7UYpy4ouKWuDQ27Wdn\nWx8b6jy4NEziGJcxbKz38uaJAUamtB9lvFBxS1x6vbWXaX9QwyRxYFO9B3/QskPDJXFDxS1x6eUj\n3XhzM1hSke90lJTXWJaHNzeDV49qZ5x4oeKWuDMy5WN3ex8b6z24dG0Sx4WGSzy8eXyAUQ2XxAUV\nt8SdHS29+AKWzRomiRub6r3MBIK83trndBRBxS1x6OUj3ZTkZbC4LM/pKBK2uDwPj4ZL4oaKW+LK\n8ISPN4/3s7nBq0u4xhGXMWys87C7vZ8xLcZxnIpb4sq2lh78QQ2TxKNN9Z7wcIlmlzhNxS1x5YWD\nXZQXZGqnmzi0pCKf4px0tmi4xHEqbokb3cOTvHvqInc3lmiYJA65jGFDnYdd7f26donDVNwSN144\ndAEL3N2oHZTi1aYGL9P+ILvaNbvESSpuiQvWWp7Z38mS8jwqCrOcjiPXsaw8n6LsdM0ucZiKW+LC\nsQsjnOgb09l2nHO5DBvqPexs7WNiRsMlTlFxS1x47mAXbpfRtUkSwKZ6D1P+ILva+p2OkrJU3OI4\nXyDIC4e6WLuwiLwst9NxZBa3VRRQkO1m67Eep6OkLBW3OG5XWx8DYzP8zJIyp6NIBFwuw/qFHl5v\n7WXKp42EnaDiFsc9ubeD4px01tQUOR1FIrSx3sPETIB3T2kjYSeouMVRPcNT7G7v494lpdpXMoGs\nrCogNyONLUc1XOIEFbc46pkDnQQt3KdhkoTiTnOxdmExr7X04g8EnY6TclTc4phg0PLU3g6WVxZo\n7nYC2ljnYXjSxwdnBp2OknJU3OKY989c5PzgBPct1dztRHR7TSGZbhdbmjVcEmsqbnHM4++fIz/T\nzaZ6zd1ORJnuNFbXFLH1WA/BoHU6TkpRcYsjeoan2Nbcy88sLSXDrR/DRLWxzkP/6DQHO4acjpJS\n9Bsjjnjig/MEreXjt5U7HUVuwdqFRbhdhq3NunZJLKm4JeZm/EF+suc8q2uKKC/Qi5KJLCfDzarq\nQrY092CthktiRcUtMbftWA/9Y9N8YrnOtpPBhnoPnZcmaekecTpKylBxS8z987tnKcvPZM0CrZRM\nButri3EZ2KrZJTGj4paYOtQxxL5zl/jUigpcWimZFAqy0rmtskDTAmNIxS0x9dhbp8nJSNPc7SSz\nsc7Dyb4xTvaNOR0lJai4JWY6L02w5WgP9y8rIydDl29NJk11HgxoZ5wYUXFLzPzonbNg4MEVFU5H\nkXnmyc1gWWU+Lx+54HSUlDBrcRtjaowxu4wxLcaYY8aYr8UimCSXkSkfT+49z+Z6D968TKfjSBRs\nrvdyvHeME72jTkdJepGccfuB/2qtXQ5sBr5ijFke3ViSbP7l3bOMTwf47O1VTkeRKNlYHxoueUXD\nJVE3a3Fba7uttQfCH48CrUB1tINJ8piY8fPY22dYW1NEfUmu03EkSopyLg+XqLijbU5j3MaYOmAt\nsCcaYSQ5PbHnPEMTPr6wVn/vk93mBi8n+8Y4ruGSqIq4uI0xecAzwNettT+1RMoY84gxZp8xZl9/\nv3Z/lpBpf4BH3zzN8soClpTnOx1HomxjnQeXgZcP60XKaIqouI0x6YRK+8fW2mevdRtr7aPW2iZr\nbVNpqeboSsjT+zvpG53moTUa204FRTkZLK8q4PlDF3TtkiiKZFaJAf4BaLXW/u/oR5JkMe0P8IOd\nJ2ksy2NVdaHTcSRG7m4s4fzgBAfO61Kv0RLJGfddwK8A9xtjDoXfPhPlXJIEntrbQffwFF9av4DQ\n339JBRvqPGS6XTx/sMvpKEkrklklb1trjbX2dmvtmvDbq7EIJ4lryhfgb3aeZFlFvs62U0xOhpt1\ntcW8ePgCM35tJBwNWjkpUfH4++foH53mS001OttOQXc3ljA86eON45qoEA0qbpl3o1M+frj7FCuq\nClheWeB0HHHA7QsKKch289zBTqejJCUVt8y7//PGaQbHZ3h4w0Kno4hD3C4XdzSUsKOlj6GJGafj\nJB0Vt8yrnuEpHnv7NHcs8tJYlud0HHHQx5aWMhMI8swBvUg531TcMq/+6rV2/EHLw001TkcRh9V6\nc2ksy+OJPec0p3ueqbhl3rR2j/D0/k4+ubyCMm0CLMADy8o41T/O3rOXnI6SVFTcMm++82oruZlu\nfm6NrkkiIZsbvORkpPHEnnNOR0kqKm6ZF28e7+etEwN8YU01eVna3UZCstLTuLuxhFeP9nBpXC9S\nzhcVt9yyQNDynVdbKcvP5JMryp2OI3Hm/mVlzASCPLWvw+koSUPFLbfsmQOdtPWM8h821JCeph8p\n+ahaby4rqwv4x7fPMO0POB0nKei3TG7J6JSPv9zaTmNZHnc0eJ2OI3Hq87dX0Tc6zQuHdLnX+aDi\nllvy/R0nGBib5j/dUael7XJdq6oLqfXm8OibpwkGNTXwVqm45aYd7x3lR++c5b6lZVpsIzdkjOHz\nt1dxsm+MXe19TsdJeCpuuSnWWv70hWNkZ6Tx8EYttpHZbWrwUJqfyQ92ntSCnFuk4pab8tzBLt47\nfZEvNy2gICvd6TiSANwuFw+tqeJgxxDbjvU6HSehqbhlzi6OTfM/X25hSXkeDyzT9D+J3H1Lyqgu\nzubPt7bhC+ha3TdLxS1z9r9eaWVsys9v3t2Ay6UXJCVyaS7DwxtqODMwzlN7Na/7Zqm4ZU52t/fx\n3MEufnZ1FTWeHKfjSAJav7CYZRX5/PWO44xN+52Ok5BU3BKxoYkZvvH0EaqLs3lI1yORm2SM4T9u\nquXi2Azf3dLqdJyEpOKWiFhr+dbzzVwcn+Er9zWS4daPjty8xrI8Pr2qksffP8+7JwecjpNw9Nsn\nEXnh0AVeOdLNL6xbQH1JrtNxJAl8uWkBlYVZfOPpI4xryGROVNwyq9P9Y/zJ880sKc/jZ1dXOR1H\nkkSmO41H7m2ga2iSb79wTHO750DFLTc0Pu3nt/91P8bA79+/WLNIZF4tqyjgi+sW8MyBTh5764zT\ncRKGiluuy1rLHz1zhFP9Y/z+/Yspyct0OpIkoS+uq2ZTvYfvvNrKzjYtzImEiluu6wc7T/LykW6+\n3FTDqupCp+NIknIZw3++bxF1Jbl85YmDvKMXK2el4pZremLPef5q+3HuWVyicW2Jukx3Gt/41FJK\n8jL4tR99wNbmHqcjxTUVt/yULUe7+ZPnj7K2pohH7m3Q5VolJopyMvj2Z1dQ683ld3+8n79/45Qu\nAXsdKm75iKf3d/J7TxyksSyPrz6wGLdLPyISO3lZbr71mdtoqvXw3S1t/NL/fZ+uoUmnY8Ud/VYK\nEHoh8rG3TvPf/u0wy6sK+OaDt5GVnuZ0LElBWelpfP3ji3nk3gYOdQ5x//d282evtDCozYY/pO24\nhfFpP3/yfDPPHexiU72Hr3ysUXtHiqOMMXxsaRkrqwr4t/2d/MPbZ3h8z3k+t6qSL2+ooam2OKWH\n8Ew0Jr03NTXZffv2zfvjyvxr7hrma08e5HT/OD+/fgE/t6Zac7Ul7nRdmuTlIxfYc+Yik74gpfmZ\n3Lu4lHuXlHBXY0lSTFU1xuy31jZFdFsVd2oanvDxvdfaefz9cxTmpPOV+xpZqSl/EuemfAE+ODPI\noY4hmruGGQ0vlV9RVcDdi0u4p7GUprrihBzmU3HLdV0cm+af3j3LP797lrFpP59YXsGX1i8gN1Oj\nZpJYgkHLmYvjHOkc5mjXEMd7xwgELZluFxvqPPzMklI+uaKcWm9iXFtHxS0f4Q8EefvkAM8d7GJb\ncw/T/iBNdcX8/LoFCfNDLTKbKV+Alu4RjnYN09w1TOel0GyU2yry+fSqSj69soLF5fkOp7w+FXeK\nm/IFaO0e4UjnMO+cHOC90xcZnfKTl+lmc4OHB1dUUl2c7XRMkajqH51m79lB9py5yPHeMQAWleby\n2VWVfHpVJcsq8uPqBc55L25jzIPA94E04DFr7XdvdHsVd/RM+wMMT/gYGJuhb3SKvpHp0PvRac4P\nTnBmYJzOwUkC4f+vpfmZrKouZG1NEatrijRbRFLS4PgM+84OsufMIG09IwQt1Jfk8plVFXxyeQWr\nqgsdf1F+XovbGJMGHAc+AXQCe4FftNa2XO8+Ku7ZTfkCDE34GJqcCb2fCL+f9DE04WP4w6/7uBT+\n3vCkj0lf4JqPl5fppjQ/k7L8TKqKsqn35lJfmos3NyOuzipEnDY0McPes5f44OxFWi6ESrw4J517\nFpeyod7D+oXFLCnPwx3jk5y5FHckr0htBE5aa0+HH/xJ4CHgusU9H6y1BIKWgLVYy4cfB4MWf9Ay\nORNgyhdgYibApC/8NhP+fMbPxEyA8fDHofcBxqf9BIKWoLVYIGhDz5PpTiM7I40st4us9Cs+zkgj\nI81FpttFepqLjCveZ4T/p84EgvgDFl8gyEwgyPi0n9EpP6NTPkYm/YxO+xid8jMy6Qt/3c/IlI9p\n//V3uHa7DAVZ6eRmpZGX4SYv0015QRZ5maGP87Lc5Ge5Kc7JoDgnncLsDO1IIxKhopwMPrG8nE8s\nL2dk0seRrmEOdwzx5ol+Xjx8AYD0NENjWR4NpXlUFGRRXpBJeUEWZflZFOWkk52eFuqK9DQy00N9\nEMsz9kiKuxq4cjvmTmBTNMLc/j+2MT4TIDCP1yfISnd9eJAz3S7SXAZX+Az08vuZQJAZf5BpfyD8\nPnjDYo3mgZJKAAAEtklEQVToed0ucjPd5GSkkZPhJjczjaqibHIz0sjJDBVvfmZ66H34LS/8eabb\npbNkkRjIy3RTVZTNgysqsNbSOzJNa88IZwfGOTc4wYFzl7g0PsNUhH1QXpDJnj/+eJRTz+PKSWPM\nI8Aj4U/HjDHt8/XYN1ACxPM1IJXv1sV7xnjPB/GfMWnynQPMt276eWojvWEkxd0F1Fzx+YLw1z7C\nWvso8GikTzwfjDH7Ih0TcoLy3bp4zxjv+SD+Myrf3EUyMLoXWGyMqTfGZAAPAy9GN5aIiFzPrGfc\n1lq/Meb3gG2EpgP+o7X2WNSTiYjINUU0xm2tfRV4NcpZbkZMh2ZugvLdunjPGO/5IP4zKt8cRWXl\npIiIRI8m/4qIJJi4LG5jzIPGmHZjzEljzDev8f37jDHDxphD4bdvX/G9s8aYo+GvR2X55mz5rsh4\nyBhzzBjzxlzuGwcZHT+Gxpg/vOL/b7MxJmCM8UT63xYHGePhGBYaY14yxhwO/z/+9UjvGwf5on78\nIsxYbIx5zhhzxBjzgTFmZaT3jSprbVy9EXoB9BTQAGQAh4HlV93mPuDl69z/LFDicL4iQitLF4Y/\nL4v0vk5njJdjeNXtPw/sjLdjeL2M8XIMgT8G/jz8cSkwGL5t1I/hreSLxfGbQ8a/BP40/PEy4PVY\n/hxe7y0ez7g/XGJvrZ0BLi+xjxeR5Psl4Flr7XkAa23fHO7rdMZYmOtx+EXgJzd5XycyxkIk+SyQ\nb0LLcPMIFaM/wvs6mS9WIsm4HNgJYK1tA+qMMeUR3jdq4rG4r7XEvvoat7sz/M+XLcaYFVd83QI7\njDH7w6s5nci3BCg2xuwO5/jVOdzX6YwQH8cQAGNMDvAg8Mxc7+tgRoiPY/gD4DbgAnAU+Jq1Nhjh\nfZ3MB9E/fpFmPAx8EcAYs5HQ6sYFEd43ahJ125MDhP6JP2aM+QzwPLA4/L27rbVdxpgyYLsxps1a\n+2aM87mB9cADQDbwnjHm/RhnmM01M1prjxMfx/CyzwPvWGsHHXr+SFwrYzwcw08Bh4D7gUXhHG/F\nOMONXDOftXaE+Dh+AN8Fvm+MOUToj8tB4NqX6IyheDzjnnWJvbV2xFo7Fv74VSDdGFMS/rwr/L4P\neI7QP2limo/QX99t1tpxa+0A8CawOsL7Op0xXo7hZQ/z0SGIeDqGl12dMV6O4a8TGg6z1tqTwBlC\n47SxOIa3ki8Wxy+ijOGu+XVr7RrgVwmNxZ+O5L5RFavB9EjfCJ0Jngbq+fdB/xVX3aaCf5+DvhE4\nDxggF8gPfz0XeBd40IF8twGvh2+bAzQDKyO5bxxkjItjGL5dIaFxz9y53tfhjHFxDIG/A/5H+ONy\nQsVSEotjeIv5on785pCxiH9/wfS3gH+J5c/hdbPH6onmeEA/Q2jzhlPAt8Jf+x3gd8If/x5wLHyw\n3gfuDH+9Ify1w+Hvf8uJfOHP/5DQrI1m4Os3um88ZYyzY/hrwJOR3DeeMsbLMQSqgNcI/RO/Gfjl\nWB7Dm80Xq+MXYcY7wt9vB54FimP9c3itN62cFBFJMPE4xi0iIjeg4hYRSTAqbhGRBKPiFhFJMCpu\nEZEEo+IWEUkwKm4RkQSj4hYRSTD/Hxm+OGyJKG2WAAAAAElFTkSuQmCC\n",
	"text/plain": [
	"<matplotlib.figure.Figure at 0x114fac908>"
	]
	},
	"metadata": {},
	"output_type": "display_data"
	}
	],
	"source": [
	"sns.kdeplot(samp_aucs, shade=True)"
	]
	},
	{
	"cell_type": "markdown",
	"metadata": {},
	"source": [
	"Cautionary Note: Obviously, this is a special case where the positive & negative scores are balanced and drawn from unit-scale normal distributions with different means"
	]
	}
	],
	"metadata": {
	"kernelspec": {
	"display_name": "Python [ipykernel_py3]",
	"language": "python",
	"name": "Python [ipykernel_py3]"
	},
	"language_info": {
	"codemirror_mode": {
	"name": "ipython",
	"version": 3
	},
	"file_extension": ".py",
	"mimetype": "text/x-python",
	"name": "python",
	"nbconvert_exporter": "python",
	"pygments_lexer": "ipython3",
	"version": "3.6.0"
	}
	},
	"nbformat": 4,
	"nbformat_minor": 2
	}