play_with_selection_bias.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import mean, std\n",
    "import numpy as np\n",
    "from math import sqrt\n",
    "import numpy.random as npr\n",
    "import scipy.stats as stats\n",
    "from tqdm import tqdm\n",
    "import hvplot.pandas\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 1000\n",
    "n = 20\n",
    "a = 0.05\n",
    "μ_required = 1.\n",
    "Z = abs(stats.t.ppf(a, n-1))\n",
    "\n",
    "def make_a_feature():\n",
    "    μμ = 0.5\n",
    "    σμ = 0.5\n",
    "    μ = npr.normal(μμ, σμ)\n",
    "    σ = npr.exponential(scale=σμ)\n",
    "    return μ, σ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:00<00:00, 7965.15it/s]\n"
     ]
    }
   ],
   "source": [
    "bias_features = []\n",
    "bias_estimates = []\n",
    "unbias_features = []\n",
    "unbias_estimates = []\n",
    "\n",
    "for _ in tqdm(range(N)):\n",
    "    μ, σ = make_a_feature()\n",
    "    samples = [npr.normal(μ, σ) for _ in range(n)]\n",
    "    s = std(samples)\n",
    "    m = mean(samples)\n",
    "    reject_h0 = (sqrt(n) * (m - μ_required) / s) > Z  # could use this feature\n",
    "    if reject_h0:\n",
    "        if μ < s * Z / sqrt(n) + μ_required:\n",
    "            bias_features.append((μ, σ))\n",
    "            bias_estimates.append((m, s))\n",
    "        else:\n",
    "            unbias_features.append((μ, σ))\n",
    "            unbias_estimates.append((m, s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:18<00:00,  1.09it/s]\n"
     ]
    }
   ],
   "source": [
    "num_unbias_plot_data = []\n",
    "for n in tqdm(range(20, 601, 30)):\n",
    "    bias = 0\n",
    "    unbias = 0\n",
    "    for _ in range(N):\n",
    "        μ, σ = make_a_feature()\n",
    "        samples = [npr.normal(μ, σ) for _ in range(n)]\n",
    "        s = std(samples)\n",
    "        m = mean(samples)\n",
    "        reject_h0 = (sqrt(n) * (m - μ_required) / s) > Z  # could use this feature\n",
    "        if reject_h0:\n",
    "            if μ < s * Z / sqrt(n) + μ_required:\n",
    "                bias += 1\n",
    "            else:\n",
    "                unbias += 1\n",
    "    num_unbias_plot_data.append((n, bias, unbias))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(num_unbias_plot_data, columns=['Sample Num', 'Num of Bias Feat', 'other'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.holoviews_exec.v0+json": "",
      "text/html": [
       "<div id='4310'>\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"8c88ca93-bfd1-400c-8197-391cabe7a1fd\" data-root-id=\"4310\"></div>\n",
       "</div>\n",
       "<script type=\"application/javascript\">(function(root) {\n",
       "  function embed_document(root) {\n",
       "    var docs_json = {\"714e86a7-6971-43da-9141-fbd48a241e96\":{\"roots\":{\"references\":[{\"attributes\":{},\"id\":\"4322\",\"type\":\"LinearScale\"},{\"attributes\":{\"below\":[{\"id\":\"4324\"}],\"center\":[{\"id\":\"4327\"},{\"id\":\"4331\"}],\"left\":[{\"id\":\"4328\"}],\"margin\":[5,5,5,5],\"min_border_bottom\":10,\"min_border_left\":10,\"min_border_right\":10,\"min_border_top\":10,\"plot_height\":300,\"plot_width\":700,\"renderers\":[{\"id\":\"4351\"}],\"sizing_mode\":\"fixed\",\"title\":{\"id\":\"4316\"},\"toolbar\":{\"id\":\"4338\"},\"x_range\":{\"id\":\"4312\"},\"x_scale\":{\"id\":\"4320\"},\"y_range\":{\"id\":\"4313\"},\"y_scale\":{\"id\":\"4322\"}},\"id\":\"4315\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"4363\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"4320\",\"type\":\"LinearScale\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"4314\"},{\"id\":\"4332\"},{\"id\":\"4333\"},{\"id\":\"4334\"},{\"id\":\"4335\"},{\"id\":\"4336\"}]},\"id\":\"4338\",\"type\":\"Toolbar\"},{\"attributes\":{\"children\":[{\"id\":\"4311\"},{\"id\":\"4315\"},{\"id\":\"4373\"}],\"margin\":[0,0,0,0],\"name\":\"Row09284\",\"tags\":[\"embedded\"]},\"id\":\"4310\",\"type\":\"Row\"},{\"attributes\":{},\"id\":\"4336\",\"type\":\"ResetTool\"},{\"attributes\":{\"axis_label\":\"Sample Num\",\"bounds\":\"auto\",\"formatter\":{\"id\":\"4353\"},\"major_label_orientation\":\"horizontal\",\"ticker\":{\"id\":\"4325\"}},\"id\":\"4324\",\"type\":\"LinearAxis\"},{\"attributes\":{\"margin\":[5,5,5,5],\"name\":\"HSpacer09288\",\"sizing_mode\":\"stretch_width\"},\"id\":\"4311\",\"type\":\"Spacer\"},{\"attributes\":{},\"id\":\"4334\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"margin\":[5,5,5,5],\"name\":\"HSpacer09289\",\"sizing_mode\":\"stretch_width\"},\"id\":\"4373\",\"type\":\"Spacer\"},{\"attributes\":{},\"id\":\"4333\",\"type\":\"PanTool\"},{\"attributes\":{\"end\":22.9,\"reset_end\":22.9,\"reset_start\":0.09999999999999987,\"start\":0.09999999999999987,\"tags\":[[[\"Num of Bias Feat\",\"Num of Bias Feat\",null]]]},\"id\":\"4313\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"4332\",\"type\":\"SaveTool\"},{\"attributes\":{\"source\":{\"id\":\"4345\"}},\"id\":\"4352\",\"type\":\"CDSView\"},{\"attributes\":{\"overlay\":{\"id\":\"4337\"}},\"id\":\"4335\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"4337\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"line_color\":\"#1f77b3\",\"line_width\":2,\"x\":{\"field\":\"Sample Num\"},\"y\":{\"field\":\"Num of Bias Feat\"}},\"id\":\"4348\",\"type\":\"Line\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b3\",\"line_width\":2,\"x\":{\"field\":\"Sample Num\"},\"y\":{\"field\":\"Num of Bias Feat\"}},\"id\":\"4349\",\"type\":\"Line\"},{\"attributes\":{\"axis_label\":\"Num of Bias Feat\",\"bounds\":\"auto\",\"formatter\":{\"id\":\"4355\"},\"major_label_orientation\":\"horizontal\",\"ticker\":{\"id\":\"4329\"}},\"id\":\"4328\",\"type\":\"LinearAxis\"},{\"attributes\":{\"callback\":null,\"renderers\":[{\"id\":\"4351\"}],\"tags\":[\"hv_created\"],\"tooltips\":[[\"Sample Num\",\"@{Sample_Num}\"],[\"Num of Bias Feat\",\"@{Num_of_Bias_Feat}\"]]},\"id\":\"4314\",\"type\":\"HoverTool\"},{\"attributes\":{\"data_source\":{\"id\":\"4345\"},\"glyph\":{\"id\":\"4348\"},\"hover_glyph\":null,\"muted_glyph\":{\"id\":\"4350\"},\"nonselection_glyph\":{\"id\":\"4349\"},\"selection_glyph\":null,\"view\":{\"id\":\"4352\"}},\"id\":\"4351\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"4346\",\"type\":\"Selection\"},{\"attributes\":{\"text\":\"More Samples \\u2192 Less Bias\",\"text_color\":{\"value\":\"black\"},\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"4316\",\"type\":\"Title\"},{\"attributes\":{\"line_alpha\":0.2,\"line_color\":\"#1f77b3\",\"line_width\":2,\"x\":{\"field\":\"Sample Num\"},\"y\":{\"field\":\"Num of Bias Feat\"}},\"id\":\"4350\",\"type\":\"Line\"},{\"attributes\":{\"axis\":{\"id\":\"4324\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"4327\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"4329\",\"type\":\"BasicTicker\"},{\"attributes\":{\"end\":590.0,\"reset_end\":590.0,\"reset_start\":20.0,\"start\":20.0,\"tags\":[[[\"Sample Num\",\"Sample Num\",null]]]},\"id\":\"4312\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"4353\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"data\":{\"Num of Bias Feat\":[21,12,10,6,3,8,4,3,4,5,6,3,2,5,5,6,5,5,4,2],\"Num_of_Bias_Feat\":[21,12,10,6,3,8,4,3,4,5,6,3,2,5,5,6,5,5,4,2],\"Sample Num\":[20,50,80,110,140,170,200,230,260,290,320,350,380,410,440,470,500,530,560,590],\"Sample_Num\":[20,50,80,110,140,170,200,230,260,290,320,350,380,410,440,470,500,530,560,590]},\"selected\":{\"id\":\"4346\"},\"selection_policy\":{\"id\":\"4363\"}},\"id\":\"4345\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"4325\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"4355\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"axis\":{\"id\":\"4328\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"4331\",\"type\":\"Grid\"}],\"root_ids\":[\"4310\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.3\"}};\n",
       "    var render_items = [{\"docid\":\"714e86a7-6971-43da-9141-fbd48a241e96\",\"root_ids\":[\"4310\"],\"roots\":{\"4310\":\"8c88ca93-bfd1-400c-8197-391cabe7a1fd\"}}];\n",
       "    root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "  }\n",
       "  if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
       "        clearInterval(timer);\n",
       "        embed_document(root);\n",
       "      } else if (document.readyState == \"complete\") {\n",
       "        attempts++;\n",
       "        if (attempts > 100) {\n",
       "          clearInterval(timer);\n",
       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        }\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);</script>"
      ],
      "text/plain": [
       ":Curve   [Sample Num]   (Num of Bias Feat)"
      ]
     },
     "execution_count": 149,
     "metadata": {
      "application/vnd.holoviews_exec.v0+json": {
       "id": "4310"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.hvplot.line(x='Sample Num', y='Num of Bias Feat', title='More Samples → Less Bias')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(list(zip(abs(np.array(unbias_estimates)[:, 0] - np.array(unbias_features)[:, 0]),\n",
    "                   abs(np.array(bias_estimates)[:, 0] - np.array(bias_features)[:, 0]))), columns=['Est Err of Unbias Feat', 'Est Err of Bias Feat'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.holoviews_exec.v0+json": "",
      "text/html": [
       "<div id='4415'>\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"587bcd04-8fc3-4847-b985-c731f91834c3\" data-root-id=\"4415\"></div>\n",
       "</div>\n",
       "<script type=\"application/javascript\">(function(root) {\n",
       "  function embed_document(root) {\n",
       "    var docs_json = {\"7d30c3be-f253-4bfe-90a3-58362c67f60c\":{\"roots\":{\"references\":[{\"attributes\":{\"click_policy\":\"mute\",\"items\":[{\"id\":\"4472\"},{\"id\":\"4491\"}],\"location\":[0,0],\"title\":\"Variable\"},\"id\":\"4471\",\"type\":\"Legend\"},{\"attributes\":{},\"id\":\"4439\",\"type\":\"PanTool\"},{\"attributes\":{\"callback\":null,\"renderers\":[{\"id\":\"4461\"},{\"id\":\"4479\"}],\"tags\":[\"hv_created\"],\"tooltips\":[[\"Variable\",\"@{Variable}\"],[\"value\",\"@{value}\"],[\"Density\",\"@{value_density}\"]]},\"id\":\"4419\",\"type\":\"HoverTool\"},{\"attributes\":{\"source\":{\"id\":\"4455\"}},\"id\":\"4462\",\"type\":\"CDSView\"},{\"attributes\":{\"fill_alpha\":0.2,\"fill_color\":\"#1f77b3\",\"line_alpha\":0.2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"4460\",\"type\":\"Patch\"},{\"attributes\":{\"data\":{\"x\":{\"__ndarray__\":\"MsJEoEIRsL//i4MyXwavv5uTfSQ56q2/Npt3FhPOrL/SonEI7bGrv22qa/rGlaq/CLJl7KB5qb+kuV/eel2ovz/BWdBUQae/2shTwi4lpr920E20CAmlvxHYR6bi7KO/rN9BmLzQor9I5zuKlrShv+PuNXxwmKC//uxf3JT4nr80/FPASMCcv2oLSKT8h5q/oho8iLBPmL/YKTBsZBeWvxA5JFAY35O/RkgYNMymkb/4rhgwAN2Ov2jNAPhnbIq/1Ovov8/7hb9ACtGHN4uBv2BRcp8+NXq/OI5CLw5Ucb8gliV+u+VgvwAEPkeslBs/YHaZQgafYj9gfnyRszByP4BBrAHkEXs/UALuOIr5gT/o4wVxImqGP3jFHam62oo/CKc14VJLjz9QxKaM9d2RPxi1sqhBFpQ/4KW+xI1Olj+olsrg2YaYP3SH1vwlv5o/PHjiGHL3nD8Eae40vi+fP+gsfSgFtKA/TCWDNivQoT+wHYlEUeyiPxYWj1J3CKQ/eg6VYJ0kpT/eBptuw0CmP0T/oHzpXKc/qPemig95qD8M8KyYNZWpP3LosqZbsao/1uC4tIHNqz862b7Cp+msP6DRxNDNBa4/BMrK3vMhrz80YWj2DB+wP2Zda/0frbA/mFluBDM7sT/MVXELRsmxP/5RdBJZV7I/ME53GWzlsj9iSnogf3OzP5RGfSeSAbQ/xkKALqWPtD/4PoM1uB21Pyw7hjzLq7U/XjeJQ945tj+QM4xK8ce2P8Ivj1EEVrc/9CuSWBfktz8mKJVfKnK4P1okmGY9ALk/jCCbbVCOuT++HJ50Yxy6P/AYoXt2qro/IhWkgok4uz9UEaeJnMa7P4YNqpCvVLw/ugmtl8LivD/sBbCe1XC9Px4Cs6Xo/r0/UP61rPuMvj+C+rizDhu/P7T2u7ohqb8/dHnfYJobwD+N92Dko2LAP6Z14metqcA/v/Nj67bwwD/YceVuwDfBP/HvZvLJfsE/C27oddPFwT8k7Gn53AzCPz1q63zmU8I/VuhsAPCawj9vZu6D+eHCP4jkbwcDKcM/oWLxigxwwz+hYvGKDHDDP4jkbwcDKcM/b2bug/nhwj9W6GwA8JrCPz1q63zmU8I/JOxp+dwMwj8Lbuh108XBP/HvZvLJfsE/2HHlbsA3wT+/82PrtvDAP6Z14metqcA/jfdg5KNiwD90ed9gmhvAP7T2u7ohqb8/gvq4sw4bvz9Q/rWs+4y+Px4Cs6Xo/r0/7AWwntVwvT+6Ca2XwuK8P4YNqpCvVLw/VBGniZzGuz8iFaSCiTi7P/AYoXt2qro/vhyedGMcuj+MIJttUI65P1okmGY9ALk/JiiVXypyuD/0K5JYF+S3P8Ivj1EEVrc/kDOMSvHHtj9eN4lD3jm2Pyw7hjzLq7U/+D6DNbgdtT/GQoAupY+0P5RGfSeSAbQ/Ykp6IH9zsz8wTncZbOWyP/5RdBJZV7I/zFVxC0bJsT+YWW4EMzuxP2Zda/0frbA/NGFo9gwfsD8Eysre8yGvP6DRxNDNBa4/Otm+wqfprD/W4Li0gc2rP3LosqZbsao/DPCsmDWVqT+o96aKD3moP0T/oHzpXKc/3gabbsNApj96DpVgnSSlPxYWj1J3CKQ/sB2JRFHsoj9MJYM2K9ChP+gsfSgFtKA/BGnuNL4vnz88eOIYcvecP3SH1vwlv5o/qJbK4NmGmD/gpb7EjU6WPxi1sqhBFpQ/UMSmjPXdkT8IpzXhUkuPP3jFHam62oo/6OMFcSJqhj9QAu44ivmBP4BBrAHkEXs/YH58kbMwcj9gdplCBp9iPwAEPkeslBs/IJYlfrvlYL84jkIvDlRxv2BRcp8+NXq/QArRhzeLgb/U6+i/z/uFv2jNAPhnbIq/+K4YMADdjr9GSBg0zKaRvxA5JFAY35O/2CkwbGQXlr+iGjyIsE+Yv2oLSKT8h5q/NPxTwEjAnL/+7F/clPiev+PuNXxwmKC/SOc7ipa0ob+s30GYvNCivxHYR6bi7KO/dtBNtAgJpb/ayFPCLiWmvz/BWdBUQae/pLlf3npdqL8IsmXsoHmpv22qa/rGlaq/0qJxCO2xq782m3cWE86sv5uTfSQ56q2//4uDMl8Gr78ywkSgQhGwvw==\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[200]},\"y\":{\"__ndarray__\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC7Rmc1opuhP3cSIB2dtKc/RdEY9HiYrz+M0s4/fdi0P3SX7DN2O7s/7X29C16cwT9qfjrWIY3GPx6r6bOAl8w/5WrvRjry0T+HYb3WkU7WP2c1jo+edNs/oKNRtUO74D80T//r2THkP3KCn/XXI+g/wC9aYIqU7D+SgFxPIsLwP6aoqEj8d/M/+K1jJXho9j/5sDYaWo75P1cOsd6S4vw/GBhR7jQuAEAOcOxr4vgBQIh98LjFywNAJXSTSxGhBUBDDx/X/HIHQAyZ/cUQPAlA9aK793H3CkAtMlnlKKEMQNgiKHtfNg5A58ydj5G1D0AkzgdqVo8QQEqVtiqPORFACndWfubaEUCUKo7ogHUSQE9fmvEfDBNAnudJTgeiE0DVrC7S3DoUQDkrZsiE2hRAg+nRJPyEFUB9nKeyMT4WQOquhwjfCRdARIM3s2HrF0DMbGnLlOUYQF3qAhmr+hlAx47zBwosG0C2E0EHJnocQO7xmmdh5B1A2v+cX+9oH0BaYjazXoIgQMciN76yWSFAdU0UEZY3IkCW8B5JhhgjQAot+KRq+CNA/PoJA6vSJEA3WeqTT6IlQNufX4goYiZAFmgbSvwMJ0AsyUsjup0nQFKwuK+uDyhAunEYG7deKEBttSkXcIcoQGjQt51dhyhAQGpf/AhdKEACihxIEwgoQNJHFB47iSdAelqhclXiJkBx//wKOhYmQEXAfQWlKCVAKa79dw8eJECRQ96fgfsiQH2nLF1hxiFArXAeuj+EIEDCyCgsUHUeQGRFPHDm3RtAaEekN0BMGUBn4wCRZ8kWQB4kvfRBXRRA4Kfdw28OEkB9rFEgd8QPQIp02gswuQtAuM2tcVUACEDFta5cw5wEQMQrxI4ijwFAeKlxcGCs/T/L8+HAGd74P0+e8locq/Q/KWq/gR8J8T8L5L8OANnrP+m3/LCZkeY/6vss7YUi4j+JSRXL2uTcP4ohTtCk0dY/oIPhkCnd0T8ce0vtQbnLP6+NwQ1pUsU/aLEDqXpAwD+ELuZ0QI24Pw+35xuGYLI/V1k+/mFCqz/DqOCtEAikPw==\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[200]}},\"selected\":{\"id\":\"4474\"},\"selection_policy\":{\"id\":\"4489\"}},\"id\":\"4473\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"margin\":[5,5,5,5],\"name\":\"HSpacer09510\",\"sizing_mode\":\"stretch_width\"},\"id\":\"4582\",\"type\":\"Spacer\"},{\"attributes\":{\"margin\":[5,5,5,5],\"name\":\"HSpacer09509\",\"sizing_mode\":\"stretch_width\"},\"id\":\"4416\",\"type\":\"Spacer\"},{\"attributes\":{\"text\":\"Histogram of Esimation Error\",\"text_color\":{\"value\":\"black\"},\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"4422\",\"type\":\"Title\"},{\"attributes\":{\"fill_alpha\":0.5,\"fill_color\":\"#1f77b3\",\"line_alpha\":1,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"4459\",\"type\":\"Patch\"},{\"attributes\":{\"below\":[{\"id\":\"4430\"}],\"center\":[{\"id\":\"4433\"},{\"id\":\"4437\"}],\"left\":[{\"id\":\"4434\"}],\"margin\":[5,5,5,5],\"min_border_bottom\":10,\"min_border_left\":10,\"min_border_right\":10,\"min_border_top\":10,\"plot_height\":300,\"plot_width\":700,\"renderers\":[{\"id\":\"4461\"},{\"id\":\"4479\"}],\"right\":[{\"id\":\"4471\"}],\"sizing_mode\":\"fixed\",\"title\":{\"id\":\"4422\"},\"toolbar\":{\"id\":\"4444\"},\"x_range\":{\"id\":\"4417\"},\"x_scale\":{\"id\":\"4426\"},\"y_range\":{\"id\":\"4418\"},\"y_scale\":{\"id\":\"4428\"}},\"id\":\"4421\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"4453\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"end\":0.1518569639618095,\"reset_end\":0.1518569639618095,\"reset_start\":-0.06276337063605622,\"start\":-0.06276337063605622,\"tags\":[[[\"value\",\"value\",null]]]},\"id\":\"4417\",\"type\":\"Range1d\"},{\"attributes\":{\"data_source\":{\"id\":\"4455\"},\"glyph\":{\"id\":\"4458\"},\"hover_glyph\":null,\"muted_glyph\":{\"id\":\"4460\"},\"nonselection_glyph\":{\"id\":\"4459\"},\"selection_glyph\":null,\"view\":{\"id\":\"4462\"}},\"id\":\"4461\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"axis_label\":\"value\",\"bounds\":\"auto\",\"formatter\":{\"id\":\"4451\"},\"major_label_orientation\":\"horizontal\",\"ticker\":{\"id\":\"4431\"}},\"id\":\"4430\",\"type\":\"LinearAxis\"},{\"attributes\":{\"label\":{\"value\":\"Est Err of Unbias Feat\"},\"renderers\":[{\"id\":\"4461\"}]},\"id\":\"4472\",\"type\":\"LegendItem\"},{\"attributes\":{\"end\":27.47294481382679,\"reset_end\":27.47294481382679,\"reset_start\":0.0,\"tags\":[[[\"value_density\",\"Density\",null]]]},\"id\":\"4418\",\"type\":\"Range1d\"},{\"attributes\":{\"fill_alpha\":0.5,\"fill_color\":\"#1f77b3\",\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"4458\",\"type\":\"Patch\"},{\"attributes\":{},\"id\":\"4456\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"4451\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"fill_alpha\":0.5,\"fill_color\":\"#ff7e0e\",\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"4476\",\"type\":\"Patch\"},{\"attributes\":{\"axis\":{\"id\":\"4434\"},\"dimension\":1,\"grid_line_color\":null,\"ticker\":null},\"id\":\"4437\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"4426\",\"type\":\"LinearScale\"},{\"attributes\":{\"data_source\":{\"id\":\"4473\"},\"glyph\":{\"id\":\"4476\"},\"hover_glyph\":null,\"muted_glyph\":{\"id\":\"4478\"},\"nonselection_glyph\":{\"id\":\"4477\"},\"selection_glyph\":null,\"view\":{\"id\":\"4480\"}},\"id\":\"4479\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"axis\":{\"id\":\"4430\"},\"grid_line_color\":null,\"ticker\":null},\"id\":\"4433\",\"type\":\"Grid\"},{\"attributes\":{\"fill_alpha\":0.5,\"fill_color\":\"#ff7e0e\",\"line_alpha\":1,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"4477\",\"type\":\"Patch\"},{\"attributes\":{},\"id\":\"4474\",\"type\":\"Selection\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"4443\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"source\":{\"id\":\"4473\"}},\"id\":\"4480\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"4431\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"4428\",\"type\":\"LinearScale\"},{\"attributes\":{\"axis_label\":\"Density\",\"bounds\":\"auto\",\"formatter\":{\"id\":\"4453\"},\"major_label_orientation\":\"horizontal\",\"ticker\":{\"id\":\"4435\"}},\"id\":\"4434\",\"type\":\"LinearAxis\"},{\"attributes\":{\"fill_alpha\":0.2,\"fill_color\":\"#ff7e0e\",\"line_alpha\":0.2,\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"4478\",\"type\":\"Patch\"},{\"attributes\":{},\"id\":\"4435\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"4440\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"label\":{\"value\":\"Est Err of Bias Feat\"},\"renderers\":[{\"id\":\"4479\"}]},\"id\":\"4491\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"4438\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"4469\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"overlay\":{\"id\":\"4443\"}},\"id\":\"4441\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"4442\",\"type\":\"ResetTool\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"4419\"},{\"id\":\"4438\"},{\"id\":\"4439\"},{\"id\":\"4440\"},{\"id\":\"4441\"},{\"id\":\"4442\"}]},\"id\":\"4444\",\"type\":\"Toolbar\"},{\"attributes\":{\"data\":{\"x\":{\"__ndarray__\":\"gmU1Ks4an7+Tj7iBvBiev6S5O9mqFp2/tOO+MJkUnL/FDUKIhxKbv9Y3xd91EJq/52FIN2QOmb/4i8uOUgyYvwi2TuZACpe/GuDRPS8Ilr8qClWVHQaVvzs02OwLBJS/TF5bRPoBk79ciN6b6P+Rv26yYfPW/ZC//LjJlYr3j78eDdBEZ/ONv0Bh1vND74u/YrXcoiDrib+CCeNR/eaHv6Rd6QDa4oW/xrHvr7beg7/oBfZek9qBvxS0+BvgrH+/WFwFepmke7+YBBLYUpx3v9ysHjYMlHO/QKpWKIsXb7/I+m/k/QZnv6CWEkHh7F2/QG+Kco2XS7+AOkF0nqoiP0CGVVZudlI/oHIRb8RLYT8QIviyUVxpP8hob3tvtnA/gMBiHba+dD9AGFa//MZ4PwBwSWFDz3w/3GOeAcVrgD+8D5hS6G+CP5i7kaMLdIQ/eGeL9C54hj9YE4VFUnyIPzS/fpZ1gIo/FGt455iEjD/wFnI4vIiOP2jhtcRvRpA/VrcybYFIkT9Gja8Vk0qSPzZjLL6kTJM/JDmpZrZOlD8UDyYPyFCVPwLlorfZUpY/8rofYOtUlz/ikJwI/VaYP9BmGbEOWZk/wDyWWSBbmj+uEhMCMl2bP57oj6pDX5w/jr4MU1VhnT98lIn7ZmOeP2xqBqR4ZZ8/LaBBJsUzoD8lC4D6zbSgPx12vs7WNaE/FeH8ot+2oT8LTDt36DeiPwO3eUvxuKI/+yG4H/o5oz/zjPbzArujP+v3NMgLPKQ/4WJznBS9pD/ZzbFwHT6lP9E48EQmv6U/yaMuGS9Apj/BDm3tN8GmP7d5q8FAQqc/r+TplUnDpz+nTyhqUkSoP5+6Zj5bxag/lyWlEmRGqT+NkOPmbMepP4X7Ibt1SKo/fWZgj37Jqj910Z5jh0qrP2083TeQy6s/Y6cbDJlMrD9bElrgoc2sP1N9mLSqTq0/S+jWiLPPrT9DUxVdvFCuPzm+UzHF0a4/MSmSBc5Srz8plNDZ1tOvP5B/B9dvKrA/DLUmQfRqsD+I6kWreKuwPwQgZRX967A/gFWEf4EssT+AVYR/gSyxPwQgZRX967A/iOpFq3irsD8MtSZB9GqwP5B/B9dvKrA/KZTQ2dbTrz8xKZIFzlKvPzm+UzHF0a4/Q1MVXbxQrj9L6NaIs8+tP1N9mLSqTq0/WxJa4KHNrD9jpxsMmUysP2083TeQy6s/ddGeY4dKqz99ZmCPfsmqP4X7Ibt1SKo/jZDj5mzHqT+XJaUSZEapP5+6Zj5bxag/p08oalJEqD+v5OmVScOnP7d5q8FAQqc/wQ5t7TfBpj/Joy4ZL0CmP9E48EQmv6U/2c2xcB0+pT/hYnOcFL2kP+v3NMgLPKQ/84z28wK7oz/7Ibgf+jmjPwO3eUvxuKI/C0w7d+g3oj8V4fyi37ahPx12vs7WNaE/JQuA+s20oD8toEEmxTOgP2xqBqR4ZZ8/fJSJ+2Zjnj+OvgxTVWGdP57oj6pDX5w/rhITAjJdmz/APJZZIFuaP9BmGbEOWZk/4pCcCP1WmD/yuh9g61SXPwLlorfZUpY/FA8mD8hQlT8kOalmtk6UPzZjLL6kTJM/Ro2vFZNKkj9WtzJtgUiRP2jhtcRvRpA/8BZyOLyIjj8Ua3jnmISMPzS/fpZ1gIo/WBOFRVJ8iD94Z4v0LniGP5i7kaMLdIQ/vA+YUuhvgj/cY54BxWuAPwBwSWFDz3w/QBhWv/zGeD+AwGIdtr50P8hob3tvtnA/ECL4slFcaT+gchFvxEthP0CGVVZudlI/gDpBdJ6qIj9Ab4pyjZdLv6CWEkHh7F2/yPpv5P0GZ79AqlYoixdvv9ysHjYMlHO/mAQS2FKcd79YXAV6maR7vxS0+BvgrH+/6AX2XpPagb/Gse+vtt6Dv6Rd6QDa4oW/ggnjUf3mh79itdyiIOuJv0Bh1vND74u/Hg3QRGfzjb/8uMmVivePv26yYfPW/ZC/XIjem+j/kb9MXltE+gGTvzs02OwLBJS/KgpVlR0Glb8a4NE9LwiWvwi2TuZACpe/+IvLjlIMmL/nYUg3ZA6Zv9Y3xd91EJq/xQ1CiIcSm7+0474wmRScv6S5O9mqFp2/k4+4gbwYnr+CZTUqzhqfvw==\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[200]},\"y\":{\"__ndarray__\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD1i5MrozezP1HQ4Gmsmrk/wvENqPrnwD9KwSCLGCHGP+iux6Q3tsw/gvcsTJZ20j+IyH5v74nXPxLyHb7Qv90/Q+On7AKj4j9pB6OqUCbnP6dTV6tqguw/z1LByN9n8T88XYhi2hL1P2ZQ/9/TTPk/aAfRhOse/j9I2m5YD8gBQBDqbhNb0gRA+VXWXN0uCEDLdeHrPNwLQPKT2VwR1w9A+FVgL+EMEkDY6sOqPU4UQGBlfU+bqhZAxSemWvEbGUBwKWofUZsbQJVqYWMTIR5AqNlGHIlSIEAO8Gbkdo8hQA0YYO0swyJA+KeEn7jpI0Cdb1vrhf8kQGphGMOJASZADgEmGGrtJkBmDKhiocEnQFVavMWbfShArbQDBcwhKUCCm3a2tq8pQF0LO2byKSpApvtFwBuUKkC5TsFDvfIqQOaYc4QqSytAEVrDi0+jK0D/qiOBdQEsQAjRGFv+ayxAj8Nw7xnpLEB652BCd34tQKtArmf1MC5AWHNDnVcEL0CqFYFyAPsvQEjbrOXbijBAZ+Ykqr8pMUBFIuVEvNgxQCoh9GVvlTJAFjIPjYtcM0ADza4G6yk0QEgVmsGt+DRAfnafHmHDNUBGQPlGMIQ2QFSDcQ0bNTdA2BYu7DDQN0CiLP9dzU84QLU2UKXSrjhAjN1iEeDoOEDL+F7+gPo4QH806CRS4ThA2F7sTRucOEBAzbQf3Co4QP5Ivm/LjjdA6HpPRUnKNkB2ug12xOA1QI6203KV1jRAVNTfa9CwM0B8q1VeEXUyQElUUttEKTFApSywzuCmL0DRpJZg+vIsQCsS8hkPQipACYZWlWqeJ0CNBiKLKBElQA+2EGYEoiJALcap2TtXIEAYPtfzB2scQHjKHbIegBhARUH+z0XxFEBkUjaSbb8RQFtpqth/0g1AqZixIP7WCEABjrJM0oIEQBBrHl83ygBA7iGH28c/+z9w/GLaa+v1PyAFOLmQevE/Y0Cggb+h6z8IMvP+dKblP2l/t1SZ0OA/zjCQK7rj2T9j/WiidcHTP8T7TKuD4s0/UlvhtIlnxj9XKJyV8qXAPw==\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[200]}},\"selected\":{\"id\":\"4456\"},\"selection_policy\":{\"id\":\"4469\"}},\"id\":\"4455\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"4489\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"children\":[{\"id\":\"4416\"},{\"id\":\"4421\"},{\"id\":\"4582\"}],\"margin\":[0,0,0,0],\"name\":\"Row09505\",\"tags\":[\"embedded\"]},\"id\":\"4415\",\"type\":\"Row\"}],\"root_ids\":[\"4415\"]},\"title\":\"Bokeh Application\",\"version\":\"2.2.3\"}};\n",
       "    var render_items = [{\"docid\":\"7d30c3be-f253-4bfe-90a3-58362c67f60c\",\"root_ids\":[\"4415\"],\"roots\":{\"4415\":\"587bcd04-8fc3-4847-b985-c731f91834c3\"}}];\n",
       "    root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "  }\n",
       "  if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
       "        clearInterval(timer);\n",
       "        embed_document(root);\n",
       "      } else if (document.readyState == \"complete\") {\n",
       "        attempts++;\n",
       "        if (attempts > 100) {\n",
       "          clearInterval(timer);\n",
       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        }\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);</script>"
      ],
      "text/plain": [
       ":NdOverlay   [Variable]\n",
       "   :Distribution   [value]   (Density)"
      ]
     },
     "execution_count": 151,
     "metadata": {
      "application/vnd.holoviews_exec.v0+json": {
       "id": "4415"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.hvplot.density(y=['Est Err of Unbias Feat', 'Est Err of Bias Feat'], title='Histogram of Esimation Error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [09:39<00:00,  1.73it/s]\n"
     ]
    }
   ],
   "source": [
    "n = 200000\n",
    "bias = 0\n",
    "unbias = 0\n",
    "for _ in tqdm(range(N)):\n",
    "    μ, σ = make_a_feature()\n",
    "    samples = [npr.normal(μ, σ) for _ in range(n)]\n",
    "    s = std(samples)\n",
    "    m = mean(samples)\n",
    "    reject_h0 = (sqrt(n) * (m - μ_required) / s) > Z  # could use this feature\n",
    "    if reject_h0:\n",
    "        if μ < s * Z / sqrt(n) + μ_required:\n",
    "            bias += 1\n",
    "        else:\n",
    "            unbias += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bias  # Enough Sampling => No Bias!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}