Skip to content

Instantly share code, notes, and snippets.

@maxberggren
Created March 24, 2016 21:17
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save maxberggren/5f4cb51c31f276a8d315 to your computer and use it in GitHub Desktop.
Save maxberggren/5f4cb51c31f276a8d315 to your computer and use it in GitHub Desktop.
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import pymc as pm\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Working data\n",
"bins_A = [207763, 223077, 210613, 181571, 168385, 159171, 146068, 128502,\n",
" 110505, 94379, 79315, 67084, 57527, 48867, 41862]\n",
"bins_B = [219812, 228003, 208490, 182409, 173357, 164470, 151033, 132412,\n",
" 113750, 95835, 81206, 67876, 58057, 49005, 41808]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Not working data\n",
"bins_A = [1750102, 286721, 122232, 53109, 35203, 23628, 16135,\n",
" 18991, 24309, 11363, 9732, 8494, 5911, 4374,\n",
" 3526, 2462, 2186, 1909, 1811, 1684]\n",
"bins_B = [1726921, 279424, 111627, 48393, 29513, 20356, 13086,\n",
" 18364, 23361, 10805, 8752, 10323, 6007, 4252,\n",
" 3039, 2172, 1829, 1670, 1617, 1569]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" [-----------------100%-----------------] 35000 of 35000 complete in 6.7 secProbability B > A: 1.0\n",
"Confidence interval of B:s lift over A:\n",
"0.452996919268\n",
"0.694436918102\n",
"MCMC error: 0.00739678334491\n",
"Plotting percent_better\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAl8AAAFwCAYAAACGrR5jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmYXFWd//H3h4SwhCUgGCSJAhJWERAIKiBBEHEDd8Df\nOKiIDAi4wEhAZ9IZRwQXxAUcREBQFpFNUAOEpRF1EJCdEFkzJghhX2SRLN/fH+dU+qZS3VXdXdW1\nfV7PU0/de+45956b3Ko+de73nqOIwMzMzMxGxgrNroCZmZlZN3Hjy8zMzGwEufFlZmZmNoLc+DIz\nMzMbQW58mZmZmY0gN77MzMzMRlDVxpekvSTNkXS/pKMrbD9K0m35dZekRZLG5W1zJd2Zt93UiBMw\nM6uVpC/k76m7JX0hp60taZak+yRdVfr+ytuOyd99cyTt2byam1kn0UDjfEkaBfwV2AN4BLgZ2D8i\n7u0n//uBL0bEHnn9YWC7iHi63hU3MxsMSW8CzgN2ABYCVwD/BhwMPBkR38o/MNeKiGmStgDOzfkn\nAFcDm0TEkqacgJl1jGo9X1OAByJibkQsBM4H9hkg/ydIX25FGkb9zMzqZTPgzxHxSkQsBq4HPgLs\nDZyV85wFfDAv7wOcFxELI2Iu8ADpO9HMbFiqNb4mAPMK6/Nz2nIkrQq8G7iokBzA1ZJukXTQcCpq\nZjZMdwO75NuMqwLvBSYC4yNiQc6zABifl9cnfeeV9Pv9Z2Y2GKOrbB/M3EMfAP4QEc8W0naKiEcl\nrQvMkjQnIm4YdC3NzIYpIuZIOgG4CngRuB1YXJYnJA30vef52Mxs2Ko1vh4BJhXWJ7HsL8Gi/Si7\n5RgRj+b3JyRdQuqyX6bxVeWLzsw6VESMeEhCRJwBnAEg6Ruk77MFktaLiMckvQ54PGcv//6bmNOW\n4e8ws+4z7O+viOj3RWqcPQhsAIwh/VLcvEK+NYGngFUKaasCq+flscAfgT0rlI2B6tBOL6Cn2XXw\nuXTmeXTguUSTjvva/P564N783fUt4OicPg04Pi9vkb/zxgAb5u9Ctcq5tMt10qz6kHopl/u/8b9P\n+9WpBesTw93HgD1fEbFI0mHAlcAo4PSIuFfSwXn7qTnrB4ErI+LlQvHxwCWSSo24cyLiqoGOZ2bW\nYBdKeg3pacdDI+I5SccDF0g6EJgLfBwgImZLugCYDSzK+Vu6l2vGjBmx6667AvQ0uSpm/ZoxY0YA\nTJ8+vWsfyKt225GImAnMLEs7tWz9LPqeFiqlPQxsU4c6mpnVRUS8o0La06ThdCrlPw44rtH1qpfp\n06erp6enp9n1MBtINze6SjzCfX31NrsCddTb7ArUSW+zK1BHvc2ugLWF3mZXoExvsytQprfZFSjT\n2+wKVNDb7AqU6W12BeptwEFWR6QCUkQTAm/NrHk66XPfSefSSUoPQvj/xuqtHp9593yZmXWIGTNm\nRCmexqxV+Tp1z5eZNUEnfe476Vw6iXu+rFHc82VmZmbWZtz4MjMzMxtBbnyZmXUIx9JYO/B16pgv\nM2uCTvrcd9K5dBLHfFmjOObLzMzMrM248WVmZmY2gtz4MjPrEI6lsXbg69QxX2bWBJ30ue+kc+kk\njvmyRnHMVz8k/lViyP8wktaU9LEBtv9M0pY17uugwvIBklasd33MzMysfVRtfEnaS9IcSfdLOrrC\n9qMk3ZZfd0laJGlcLWUbQWJ1WHIWsPowdrMW8PEBtg+mu/BzheVPAWNqKSSp2HisWB9JHdl4NjMz\n62QD/vGWNAr4EbAXsAWwv6TNi3ki4jsRsW1EbAscA/RGxLO1lK2FpE9JulTSbyX9XtL6hfTfSyv8\nSdJuOa0XNjs5HfKuN0g6T1KvpFl5+/aSrs37OTKn9Ug6O++/V9LKwCHArjlvf3X+kqRZks6XtIKS\nH+YysyRNkHQIsKmk6yQdC2wDzJT0RUnrSLpE0jWSfpH3MVXS5ZIuJjXUSpapj6RbJZ0EnC3pTbne\nf5L0w3xOknRyPs9r87E2knRFrsuJg/1/MLPWV0ssjaRdJb2tSp6fSfpIrceV9AZJ+xfWt5b0nlrL\n97PPDSTd1c+2GZJ2H6DsPsM5tjWWY76AiOj3BbwNuKKwPg2YNkD+c4EDB1M2VWHAOhwA/CIvvxv4\nPrA2MBPibHh+HnBd2r7mzXBtQASctAT4Utm+ZgFr5uXLgNcC04Gv5bTjgQ8AbwB+NUCdzgQ+m5e/\nCXwIeD8wI6ftCPwwL99cKHcdsGpe/g6wW17+CvARYFfg+grHW6Y+wEPARnl55UL6pcDGwN7ADwrp\nAi4ANszrpwDbDfTv7pdfjXxV+9y302skzgUYXcd99QBHVslzJvCRQexzKnB5Yf1Tpe/AoZ4jsAFw\n1xDP8WekOxQN/7/xq/te9biuqt22mgDMK6zPz2nLkbQqqXF00WDL1uDW/H4LMBl4I7AlTP0X2Hsi\nsE7avO32sEPO+lcBvy/bz5uBSyVdB0zKL4Db8vs80i2+Wvwlv9+c67Q58KG87xOANauU3xyYkfN/\nCBhfOMdy5fFrz0TEQ3l5I0m/S71+vAVYH9gMuL6UOdLVsilwRj7eDgz9/8LMhiD3UJ9UCNHYIaeP\nlXSGpD/nXu29c/qnJF0m6RpgVs53pqQ7Jd0h6cM535655/svki6QNDanz809+3/JZTaVtAFwMKnn\n/jZJOw9Q5T0k3Szpr5Lel/c5StK3Jd2U61AKqzge2CXv8yvADGDfvP6xWs+xQh1GSfqJpLslXZnv\nTCzTMyfpeEn35Pp8O/fqfaDw777REP67zBpqdJXtg+kW/ADwh4h4dghlByJg27y8PXA/qefnTuid\nBMwD5e0vvwgrjE3LEx8B3gH8RdIKEbEEuAP4aEQ8X0qT9P4Kx1sIjKpSr7eQGm07ADcBi4ALIuK/\nASSV/m2L/w4L6fs3nwNcEhF/KOTfCVhS4VivltWnmOffgO9GxDWSfp3rfy+wB7khrBQb9lfgqIj4\nW06rdn5mVl8BrBIR20raBTgD2Ar4KnBNRHxGKV72z5KuzmW2BbaKFMpxAumH15sBJI2TtE4uv3tE\nvKwUW/tl4Ov5eE9ExHZKIRBHRcRBkv4HeCEiBgo/EPCGiNhB0sbAdfn9AODZiJgiaSXgD5KuAo7O\n+/9ArtsCUu/6EXn9uFrOsUI9JgP7RcTnJP2SdIfgnHxuIek1wAcjYrN8nDXy9/tl5NCNwg9Vs5ZR\nrfH1CH29Q+Tl+f3k3Q84byhlJfUUVnsjorewHsAYSTOBscD+EfGU9B+R7tLFeNItvC/CyqvmMh+E\nL18Kd54orfIx4EXgXaRbnxfnxsg/JX2ocIzi8R4FVpH0K9Kt0gcrVHu7HOPwJPDV3JB7p6Rr8z5+\nQeq6/6ukC4ETSbc6L8jr3wBOkzQj7+8rFepSUqrPBcCxZXkuB74vaQ7pCzMi4nKlhx1uIDX4Pk76\ncvyf/MtxMfAZlu2ZNGsYSVNJt6a63XkAEXGDpDUkrQnsCXxA0lE5z0rA60mf81mFRsnuwL6lHeUG\n2ftJMbV/ksS66667zaRJkyA1vgAuzu+3Ah8u1KPa0+BBClUgIh6Q9BCpR31PYCtJH8351iCFOiwq\nK6+yY9R6juUejog78/JfSLcii54FXpF0OvCb/LIWV4r3mj59etcOA1Kt8XULMDl3Vf+d9MHfvzxT\n/gJ5B/CJwZYFiIieKvX4fUSc3Hc8RsPXSz1WY4AvSUyBXgE7RnBTamOcD/C2iPQlEBG3knqEikqN\nH4rHAPoNFo2IT/eT/qUKaZ8srP6J9BBCSaWA1uvLE/Jtw2J9phS2zQLeVKHM58uSngTeW6neZo2W\nf1D1ltYlTW9aZVpL6YfUhyPi/uIGSTuSfjguk1xhH7Mi4hMV0gH+md8XU/37vppSXQ/L3zt9lUqN\n60p5i2o9x6J/FpYXA6sUi0fEYklTSA3TjwKH5WVrYd3c6CoZMOYrIhaRLuYrgdnALyPiXkkHSzq4\nkPWDwJUR8XK1skOsZ/kHedWy9ZWB0jhYN+f3V0obJcYN8bhIOk7pCcHS69ih7svMut6+ADnW6tmI\neJ70HXlEKYNUCqNYrqE1C/h8Id844EZgJ0lvzGljJU2uUocXqD4Uj4CPKXkjsBEpVOJK4NBSWIWk\nTZTifZ8v22f5MWo9x0HJ8W3jImIm6Xbr1oXjm7Wsqr+E8kU9syzt1LL1s4CzailbicSoCBb3c/zl\n9ktf4+tWYEPSr6F1gP+MID+KwCoSU0lPGD7NEAeUjQg3tsysXl6RdCvpu/czOe3rwEmS7iR9Tz1E\nemI5WPaH538DJysNv7AY6ImISyV9Cjgvx2BBigFbpoepbF+XAxcqDcdweCnutEL+v5HiWdcADo6I\nVyX9lHTr71ZJAh4n/fi+E1gs6XZSuMXZwDRJtwHHDeIcK9Wjv/UgNfB+ncMpBJTuPpwPHA4p4N5x\nX9ZqWmJ6ofx5WjOC56vnZyfgDwARSOIR0tAOlwI/jOhrBEpsRfpSoHTr0cyaTx00JU+t56L0pPGR\nOfyhIRxL00eeXqhltft1Wo/vr1ZqfG0awX3V8/MPUuB9qfF1P/A+0tN8u0dwbSHvhqRfWG58mbUQ\nN76s0dz4skbptMbXz4FPA5TfgpQ4izTcwiGkp/fI+SQt0w29VQR3F8qNIQdsFhtfEteQ4sTOAy6O\n4O91Pi0zG0A3Nr5aUY5hLZ839oKI+GYz6lNPbnxZo3RS42sxaRyrS4BXIpZ5apJCA+vN5NuIwG4R\n9JY1vlYoxXwVyl5CiklYG/gpKQbsh4Us90Wwad1OyMyqaucGS7lOOpdO4saXNUonNb4+BvwqJ/2T\nFER5Famn636WH0NmaU+WxNGk0ZUr3lqU2IcUDzaQ5RptZtY4ndRgaaVzafdYmnpy46t1tft12jGN\nr4gov334J+DtZVmfoW/qnykRaUgJiR1Jj1v31/gqjUA/kKX7M7PGa6UGy3B10rl0Eje+rFHq8Zkf\n0vALDbIZffFc5Q0vSA2vRyJQWUOpNEDfFpV2Wpa3NIryO0jn/pq8fuSQamxmbUXSMUrzAN4l6VxJ\nK0laW9IsSfdJuiqPn1XMf7+kOZL2bGbdzaxztEzPV1pmY5YfnwbSwHl3AE9E8NplyzOBNG3RqIiK\n8yIWY8bujFg6CF9p2+HAoRFsPrwzMbNaNaO3KM+2cS2weUT8U2muwN8BWwJPRsS3lOZGXCsipkna\nAjiXvonorwY2iTRPbHG/XdXzJWld0ty2QzU/Iu6pV336454va5R6fOaHO91EvRXnGvwuuUcqgjsl\n/gacVl4ggkcktuqv4VWm0vn+BviBxKci+NkQ6mxm7eF5Uu/6qpIWkwZr/jtwDGmiWEiDRfeS5oHd\nBzgvIhYCcyU9QJra68YRrnfNRiiWZntY7RJ48yvVs5Z7dAw88Uvyk+3Wndo95qseWqrnK60TwA0R\nvKN+x1ja83VjBG8r27YiaRiLWRH4toLZCGhWb5Gkz5F+2L1MmhLtk5KeiYi18nYBT0fEWpJ+CNwY\nEefkbT8FZkbERWX77Laer/fAzufBDWsOvvT/ANPOjnj2gLpXrIx7vqxROi3mq2QysFed9/nV/P7W\n8g0RLCQNP7F2nY9pZi0kz1H4RdL0OOsDq0n6l2KePIn9YKa7MTMbtFa77UgEDzRgn8dJfIP0s6uS\nq4FvSJwRsXS+NTPrLNsDf4qIpwAkXQy8DXhM0noR8Zik15HmKwR4BJhUKD8xpy1HUk9htTcieutc\ndzNrEklTgal13Wer3XZs3HEYDzwbkUa8L9s2mvykpachMmu8JgXcbw2cQwqgfwX4GWkYmjcAT0XE\nCZKmAePKAu6n0Bdwv3GUfWm20m3HkYil8W1HG652j/kakYB7SXsBJ5FGoP9pRJxQIc9U4HvAiqSn\nhqbm9LmkINfFwMKImDKcyg5HBAsG2LZI+Z9R4j8j+C8JATcDX4vgipGppZk1SkTcIels4BZgCXAr\n8BPSoM4XSDoQmAt8POefLekCYDZpoOdDyxteraZd/5hZd/F1WqXnS9Io0oTVe5C6228G9o+Iewt5\nxgF/BN4dEfMlrRMRT+ZtDwPbRcTTAxyjZX41lgLz85yRXyYF5ro3zKzOWulzP1yddC61cM+XdbuR\n6PmaAjwQEXPzAc8nPX59byHPJ4CLImI+QKnhVazncCrYDGWj7ZuZmZnVTbWnHSew7Nhb83Na0WRg\nbUnXSbpF0icL2wK4OqcfNPzqNtx+lRKllnwq1MxsGTNmzIhSPI1Zq/J1Wr3nq5Z/nBVJox3vThq0\n8H8l3RgR9wM7R8Tf84jIsyTNiYgbynfQQk8KXQCcX1g/DPgR6Ymn/2tKjcw6QCOeFrLlOZbG2oGv\n0+qNr/JHrSeRer+K5pGC7F8GXpb0e9J0QPdHxN8BIuIJSZeQbmMu1/iKiJ6hVb++Iggte0nsnN/f\nhhtfZkOWf1D1ltYlTW9aZczMmqza7bRbgMmSNpA0BtgXuKwsz6+BnSWNkrQqsCMwW9KqklYHkDQW\n2BO4q77Vb4ixheWLgZnAKk2qi5mZmXWYAXu+ImKRpMOAK0lDTZweEfdKOjhvPzUi5ki6AriT9Pj2\nafkR7Y2Ai9NsHYwGzomIqxp5MvUQwUuAJLaJ4HaJXUkNyjObXDUzswG1+/hJ1h18nXbRIKtDJfH/\ngPdHsH+z62LWKVr9cz8YnXQutfBQE9btOnVux1bzKrCf1HfrUWJ3KQ3EaGZmZjYYbnxVd31+305i\nlMQo4FLgl02sk5mZmbUpN76qiOBx4FFgNdIDA3/Ky8uRuEfityNYvWGReJfEnyVubnZdzGz4PH6S\ntQNfp475qonE9cC2pDngijYEtgJ+A3yQ9HRk20xHVDaS/1hgYUSaYNyskdrhc1+rTjqXWjjmy7qd\nY75Gzoss3/B6EHgXaeiN84Gf5/TlxjFrRRKrliXdC/y5GXUxMzPrJm581eafFdIeBH6Slz9O3/hg\nW0vsJRESK0i8RVr+NqXEaIl3Nai+tdi0bP31wLYSx0ms34wKmZmZdQM3vmrzfNn6usBDFfIdD6wB\njMvri4G/AP9SIe+HgWaOe1bq+bqlLP0YPJq/WVtyLI21A1+njvmqicRfSPNX/gfwjTwN0a6k6VL2\npK8R9SbgRtKgtB8p7qM8DkxiLvAGYM2I5Rp3DSfxXuDwCN4j8RgwvizLqAiWjHS9rDu0w+e+Vp10\nLrVwzJd1O8d8jZybASL47wjyB5rrgVdYtufoAdLclx8p34HExwrLryU1vAB+LvG5BtV7IGvQ16N3\nWoXta41gXczMzLqGG181iODfKj3BGMEqETxTWP8n8JqybD35/YIcB7Y6sENh+97AqRI71rna1ZwH\nvDsvf4/0NOcs4KCc5vkszczMGsCNr/p4D32Nlq/l94eB0yKYUZb3etLQFC+UpX9CGpkhKiRWzotr\nAkTwdAS3R7BnBD8lxbO58WXWZhxLY+3A16ljvupOYgvgHlJQ/tMRLJF4AzC3LOv/Ai8D7yykHQic\nWbq12cA6rgM8AfxnBF+vsD2A+RFMamQ9rHt10ue+k86lFo75sm43IjFfkvaSNEfS/ZKO7ifPVEm3\nSbpbUu9gynaaCGYDb4rgyVLAekTFpwffRrrttzkwJaedDnx/BKq5Un7/7362zwEmjkA9zMzMus7o\ngTZKGgX8CNgDeAS4WdJlEXFvIc844GTg3RExX9I6tZbtVBHcUyH5DZQN4RDBIlJDB/W1od/eyLpl\nKwEPD9DDtifwNwk1uhfOzMys21Tr+ZoCPBARcyNiIWkk933K8nwCuCgi5gNExJODKNs1IvhbXiwF\n1h9XluUd+X27EajOqqRbnhVFMA94inTr1MzahGNprB34Oq3S8wVMAOYV1ufDck/lTQZWlHQdaQqe\n70fEz2ss221WjGCRxNakSbqXiuCGUu+XxLYR3NbAeuzG8k9llpsHTAIeb2A9zKyOpk+f7vgma3m+\nTqv3fNXSMl2RNADpe0kxTP8haXKNZbtKvs1IBHf2czvvpPx+aylB4naJbfrbp8R4icmDrMoPWH5Q\n1XLbANcOcr9mZmZWRbWer0dgmSfeJpF6sIrmAU9GxMvAy5J+D2yd81UrC4CknsJqb0T0Vq15Z/o2\n8MWytK2BX0kcDswEzo6g+KTQ6aRxw6o1pspdV0OeNQa5T7OKJE0Fpja5GmZmLaFa4+sWYLKkDYC/\nA/sC+5fl+TXwoxxgvxLp1uKJwH01lAUgInqGUvlOE8HfC7ceozCw68akhhfAv0p8sTC46/vK95OH\nitgogocrbNskL1a7rbktcJvEZ4DLgDdG8OfBnI9ZSf5B1VtalzS9aZXpYKU4Gt/WsVbm67RK4ysi\nFkk6jDRX4Sjg9Ii4V9LBefupETFH0hXAncAS4LSImA1QqWwDz6VTfBc4EkBa2uAqtwrwjMQuA+zn\nwxLfyctbRjBbYhzw15x2VJV6PJDfTyc9pfkGGJlBYM1saLr5j5m1D1+n1Xu+iIiZsGwjICJOLVv/\nDiz9Qz9gWRtYBEdJqfEF7NVPttXy+5uKiRJvAl6bV18qbLpHYkPSyPWl4wwYkxfBPyTOBD5N3zyU\nZmZmNkyeXqgNFG4/XkS6FVya9Lo0TRAS6wK/A64p35Y9TF/P1VM1HvrI6lnMzMxsMNz4al1zSXFX\nEwppQerRKk2I/RrSALcA+7HsAw4nkuaPLG+EEcE6Ndbh+eKKxAJpuQcCzNqCpE3zTByl13OSjpC0\ntqRZku6TdFUeOLpU5pg8Q8ccSXs2s/618PhJ1g58nXpux5YksRLwavHWYL5t+DTp4YVVI5DE/wB3\nAKf0t6+cr7SfA4ADIth9EHU5HiifGurACM6odR9m5Zr9uZe0Aulp7inA4aQntr+Vp0FbKyKmSdoC\nOJf0NPEE4Gpgk4hYUravrvoO89yO1u1GZG5HG3kR/LM8JiuChyN4jhSDdVFOXgd4srw8MKtC2u4R\nnD2Yhlf2U9Ifp6LTB7kPs1azB2kGjnnA3sBZOf0s4IN5eR/gvIhYGBFzSQ+hTCnfkZnZYLnx1X4W\nAOMlRLrt+CRwfVmec0hPn16R13el8Jj/YETwQAQ/AtYDPjaUfZi1oP2A8/Ly+IhYkJcX0Ddm3vos\nOzbhfJYNAzAzG5KqTztay3kU2JkUjzWXFDw/nb7G1Rtz+jnkWQYi+P1wDxrBAuDCwjhk+0bwy+Hu\n12ykSRoDfIDlb6cTEVG6XdWPittaZaBoj59k7aDdrtNGDBLtxlf7eTa/rwZsSer5+ivwmQjOLORb\nUl6wTt5Cmv7ofHDjy9rSe4C/RMQTeX2BpPUi4jFJr6NvPtPyGT4m5rTltMpA0e3yx8y6W7tdp40Y\nJNq3HdtP8QlEAU/lGLEz+ytQTw2e8NtsJOxP3y1HSDM4lALADwAuLaTvJ2mMpA2BycBNI1ZLM+tY\nbny1n38WVyKWXR8hfwCoNKG3xF8kfiPxtZGvltnAJI0lBdtfXEg+HniXpPuAd+Z18kwdFwCzSYNF\nHxrNfjzczDqCG19tpvAU5NGFwVdHWulpy+8WE/NDAG8hzTf59bxu1jIi4sWIWCciXiikPR0Re0TE\nJhGxZ0Q8W9h2XERsHBGbRcSVzal17Tx+krUDX6eO+WpXG7HsU1gj7db8/oGy9PKxv8bB0gnAzazB\n2i2WxrqTr1P3fLWlPObXwiYef+nTk6VbjxKjgE8Vst0HHCghidUljsp5zMzMupobXzZU38vvpUnW\nF+X3Ewp5vp23P5+X3zEyVTMzM2tdVRtfkvbK85rdn6feKN8+Nc+RVpov7T8K2+ZKujOn+ymhzlKa\n0mg3iVfz8r0RTANWBDbJae8ulLlW4kOF6Y7MrI4cS2PtwNdplbkdJY0ijSG1B2l8m5uB/SPi3kKe\nqcCXI2LvCuUfBraLiKcHOEZXzYvWSSSOAL5fSNoggv/L24oX1snA50iNspKNI3iw8bW0VtRJn/tO\nOpdaeG5H63YjMbfjFNL8Z3MjYiFpYM19KtVlgH34wu9cL5atP1VYvqOwfCfw/8ry7taQGpmZmbW4\nao2vCcC8wnqluc0CeLukOyT9TtIWZduulnSLpIOGX11rMbeUrS9tjEWwDXB5Xj0HKH9M/zUNrJeZ\nmVnLqtb4quWe7K3ApIjYGvghfaNDA+wUEduSpvP4vKRdhlZNa0UR3AHsCfwur5dfL4/l9BcjeB74\ne2Fb+TAVZjZMjqWxduDrtPo4X+Vzm02ibHypssEKZ0o6RdLaeeDCR3P6E5IuId3GvKH8IK0yKa0N\nXgSzJO4gjT1W7miWffrxY8AfgY8Dx45A9axFNGJiWluex0+yduDrtHrj6xZgsqQNSL0W+5LmRVtK\n0njg8YgISVNIQfxPS1oVGBURL+QpPfYEZlQ6SKtMSmtDE8Hj9E1GXEx/hsIgqxH8CZDEWyjEAkqM\njlg6VIV1oEZMTGtm1q4GbHxFxCJJh5HidUYBp0fEvZIOzttPBT4KHCJpEfASsF8uvh5wsaTScc6J\niKsacxrWZp4D1pNYAVic0wb9S0hiD+AQ4GsR3Fstv5mZWSuoOr1QRMwkTSpbTDu1sHwyaSiB8nIP\nAdvUoY7WeZ4DxtPX8Bqqc4F1ge2ADYa5L7O2V4qj8W0da2W+TquM8zUiFeiyMXJs6VRE5bcZx0bw\n0iD3s5j80EgTJxm3Ieikz30nnUstPM6XdbuRGOfLrO4iWAw8VJZ8RK3l83yR36fK9SsxTiIk/JSt\nmZm1DDe+rFk2KVvfaxBlN6KvsXY0sFBiJQCJtSUekngfMCvn+X2FfZiZmTWFG1/WFLn361Hgw8Bl\nwK6DKF4c6Pc00rRFMyTGkEbZ3xD4DbB9KZO0zNRGZh3J4ydZO/B1WkPAvVmjRLA+gMR1wDMSW+eB\nW6t5S37/nwieSQ/UcjTLx5EV/RtpEGCzjtXNAczWPnyduufLWkAEz+bF2yWmSVV/FHwvlzukLP2r\nFfJeA7wC3De8WpqZmdWHG1/War4JyzWqAJD4kMRqpFkSPlvY9JYK2Q8CVolgD+A2YOV6V9TMzGwo\n3PiyVvQDibHFBInXAxcDLwC7APeXtkVwGyxzu3K1CH4awSt5/W3kOUclJkpMbmTly0keBsNGhmNp\nrB34OvV6wOfSAAAgAElEQVQ4X9YiJD4K/KoseUoEN+ft5RdqzeOCSRwFfBtYBXiYNPvCmyK4Z3i1\nrk2u+7sjqDrDg8RBwE+AtfP0TB2pkz73nXQutfA4X9btPM6XdYwILgSOAX5USF5ngPyDGZD1u/n9\nfaRpsgDulnhzvo3ZMBJr5cVjquTbXmIGqeEF8I5G1svMzJrHjS9rGREcDxxZSBoNS0fEH85+S71m\nF1KY3Jl0q/Ijw9l3DbbI71Mlxg2Q7zTgPwvrYxpXJTMzayY3vqylRPAq8GpeLfVKrQM8SZoP8mng\nqGEc4mNl6/82jH3VYivgrry8RaUM+bZkcR7UR4DtJfZtcN2swziWxtqBr1PHfFmLkvgJcEsEP5F4\nM3BuBG8axv6KF/pY4MXSSiPmhZQ4EbibNNDry8CX+ztWhXi2pTp1zspO+tx30rnUwjFf1u1GJOZL\n0l6S5ki6X9LRFbZPlfScpNvy62u1ljUbwAvAFvmW4/uB54a5v3eXFnK82LrkJyalZXqd6uVLwOnA\nwcCdDdi/mZm1qSoTE2sUKQB6L9Itk/0lbV4h6/URsW1+/fcgy5pV8jLwBVJM1uGkwVKHLD9puCup\nQUQET0YsnV/ytkpl8gTep0rDuj2/Aim27Mx+jlFp/LFPF7b/n4eqMDPrLNX+qEwBHoiIuRGxEDgf\n2KdCvkp/HGota1ZJ6bbg50lDQ3xruDuM4PcRywzOWs0awOeAnYd56Lmk+SuR2Kls29RcNwGTgKMi\n+Bkpvg3g9cCAt3ckth1m/axDOJbG2oGv0+pzO04A5hXW5wM7luUJ4O2S7iAFCh8VEbNrLGvWn28B\newDvBIjgHw06zo3AW/vZVkq/XmKNCF6olEnitcACYNMI7it/OjOCZyV+nVfPBt5Y2HxhId988rAY\nETyuvp80u0jsApwcwf8Vjrs+8A/gVomdI/hj1bPtcpLGAT8FtiR9d32adPv5l8AbSA3lj0fEszn/\nMcBngMXAERFRday2ZvKcedYOfJ1W7/mqpWV6KzApIrYmTVx86bBrZV0vgsXkIHXggQYe6rMAEu+Q\nluvhKl7LF+XR8adV2MeN+f2veZLwx/L6iuQnNgvDXWwkEfl4gmVH8u/HZcC/kxoGSKwt8TLpx05p\nDLMv1LAfg+8Dv4uIzYE3A3OAacCsiNiEdHt7GoCkLYB9SWETewGnSPIT4mY2bNV6vh4h3QopmUTq\nwVoqIl4oLM+UdIqktXO+AcuWSOoprPZGRG/Vmls3eCi/bzJgruGZm9+vz+8CkBjPsvNBvovckytx\nYcQyDcINC8tTSwsRLAIW9XPc64GPVqnbKqTYt6Xyk5E/L9StdBv1YxL/GcF/VdlnU0iaSuHfpkl1\nWBPYJSIOAIiIRcBzkvYmxQMCnEUaC24aKUzivBw2MVfSA6RwihvL921mNhjVfsXdAkyWtIGkMaRf\ngZcVM0gaLyn/wdIU0vAVT9dStiQiegqv3uGdknWKCF6IQIVeo0Yc48XiusR6EpvR13tVSU8h//Z5\n8bCyPI9XKPe9svULK+Qp1u0V+nq2ij7ZT5EZA+2vmSKit/g5b1I1NgSekHSmpFslnSZpLDA+Ihbk\nPAvoi7dbn2V/MM4nhVO0LMfSWDvwdVql5ysiFkk6DLiSNC3L6RFxr6SD8/ZTSb/eD5G0CHgJ2G+g\nso07FbMhu4++3rU1SUNblHweOALYtJD2Sp74+xVIc08CPwNOBRbm9SUVjnMkcBxwE4XesipjeZ2Q\ny21AXy+dDc1o4C3AYRFxs6STYNnbyBERpfGh+tHSfzAcS2PtwNdp9duORMRMYGZZ2qmF5ZOBk2st\na9ZqIthUYiKpZ3Y8fbcKX4rglDzg6/XA20m3nA7Mr9I8lItLPWgSpwCHAodUOE4AT0ocAVyek79f\npW5PSIyOYLGW/bq6lNTbVuqZ+QRwbs0n3Z3mA/MjotRgLs0n+pik9SLiMUmvo6/XsjzsYmJOW45D\nJ8w6VyPCJjzCvVlWGGn+bOCuCL5T2DYJ+Bvp6bjSmF3fBLaK4AOFfK8j/YEeNdDtUomtgdsHM4K9\nxPeBtwE7AK/NDbPVSI3FjYFLI9i41v01U7M+95J+D3w2Iu7LDaZV86anIuIESdOAcRExLQfcn0uK\n85oAXA1sHGVfmt32HeYR7q3b1eMzX7Xny6wL7Qj8ppgQwTyJiRE8IvF50rRBx5QXjOBRapg5IoI7\nqDw+3kBlvlBqBEbwRE77B4DEo8AbJcaWx7HZMg4HzslxqA+SGtOjgAskHUgeagIgImZLugCYTWrg\nHlre8Go1pTga39axVubr1D1fZkvloR9KsVrbRnD7AHlLH5wPRiwdw6upSnVqlfkg8+3aiODg5bd1\nzue+k86lFu75sm43InM7mnWLfJuwNF7WgoHyFlR8gtcAOAj4nMQ3ml0RM7NW4saXWUEEP8jDWzxa\nJZ+A1Ro5DMYQfBI4r1kHl3hRYkxeLo7yf2yTqmRm1pLc+DIbohaMrXoOWF1id4nDR/LAOfB/VeBD\nOal4S+rBkaxLN/P4SdYOfJ064N6skzxHGqOsNE7ZD0fioBJvo69363yJTwClMf0OgL6nQa2xujmA\n2dqHr1M3vsw6yZPFFYmPk4afeLXBx/1T2fre+QXwKmmkeDMzy3zb0axzPFm2/kvgY408oMQaZUkv\nlK3PA94uLZ070cys67nxZdY5nsrvxXkmfyHxllp3kIfb6G/bBhIrliUXZwTYHHhtP3XqlVip1nrY\n0DiWxtqBr1PfdjTrGHkKorERvCRxDH2TQL+uWlmJ0cA2wO8kvh6xbLxY3v4w8CngrMKmzUnDctwJ\n3B/BYkB5zLHfRTCnMC3SKwxyYFkbHMfSWDvwdeqeL7OOEsFLeXET+uYiO1Ni9f7KSOxAGv3yZmBd\nYHuJ/5ToLWT7XX7fS+KThfQPA+tH8N7c8Cp5I/AvQz4RM7MO5saXWQeK4KUIrgcuIjWonpf4dg7C\nL3cxaaLwkn8FZkCK05JYE3hX3rYfcLZESGxEeppxeoXjPxTBM3n1iHqck5lZp6ja+JK0l6Q5ku6X\ndPQA+XaQtEjSRwppcyXdKek2STfVq9JmVrMlheWjSEH45Sb2V1jiLmCtfjaXxu96TZU6nFbYXxSm\nZrI6cyyNtQNfp1XmdpQ0CvgrsAfwCOm2xP4RcW+FfLOAl4AzI+KinP4wsF1EPD3AMbpqXjSzkSSx\nKiw3GOwZwGeB/YHzgcXA3RFsJTEW0mTdBdsCtwHfITXgyk2M4JEq9Sj7olHHzLnXbd9hntvRut1I\nzO04BXggIuZGxELSF/U+FfIdDlwIPFGpnsOpoJkNXY4Bm12W/Bngi8A5wCdy2gE5/4uknq4fF/KX\nbhtW6vkeX63hlffr7wEzs6xa42sCaZyekvn0PUEFgKQJpAZZ6cu6+As3gKsl3SLpoGHW1cyGIIIt\n8+IpheQT8/vP8/uDhfzPRnAo8N6c9OmcvoT0A2yXQt7HB1mdB4FJgyxjZtZRqg01Ucs92ZOAaRER\nksSyPV07RcSjktYFZkmaExE3lO9AUk9htTciems4rpnV7jPAhRF8vnALMOj7vD5fXiCCmdJyafsD\n5PTlyvRH0lRSED+kW57WAKU4Gj/Kb63M12n1mK+3Aj0RsVdePwZYEhEnFPI8RN8X+DqkuK+DIuKy\nsn1NB/4REd8tS++qeAmzZpP4AulH032kISn6vS0ocQLwFWCniL5phCR+CyyI4DNDq0PnfO476Vxq\nMfyYry/cAq+eO8TD/zEianp4yzFf1ij1+MxXa3yNJgXc7w78HbiJCgH3hfxnApdHxMWSVgVGRcQL\nksYCVwEzIuKqep+EmQ2exL3AZjBwTJbEa4dwe7HKsTvnc99J51KL4TW+/gj8ciEM8Ien/7IrwG3T\nI+K4WnK3e+NLGnsFrLrK0Eq//EjEPz5RPZ8NRT0+8wPedoyIRZIOA64ERgGnR8S9kg7O208doPh6\nwMXpTiSjgXPKG15m1jwRbC7xYWC5UICyfHVteFk32wnYqXyKqhodsyQ9dNstXtoTztHgh+N8Ejji\nmarZrKmqTi8UETOBmWVpFRtdEfHpwvJDpOlKzKxFRXBxs+tgQyNpBcrm0uzp6Xk0v1edUgpYFBHl\nk7FbS9mbwTe+5jeiInXlmC/P7Whm1q7GAo/CGi+XEnp6TszLazw0cNHFK8CSR0jTQJmNqG5udJW4\n8WVm1rZWWgjPDSEu6B5g5zF1r46Z1cRzO5qZmZmNIDe+zMw6RE/PDHp6ZlTPaNZEntvRtx3NzDpG\nT8/0ZlfBrCrHfLnny8zMzGxEufFlZmZmNoLc+DIz6xCO+bJ24Jgvx3yZmXUMx3xZO3DMl3u+zKyL\nSJor6U5Jt0m6KaetLWmWpPskXSVpXCH/MZLulzRH0p7Nq7mZdRI3vsysmwQwNSK2jYgpOW0aMCsi\nNgGuyetI2gLYF9gC2As4JU/pY2Y2LP4iMbNuU37LY2/grLx8FvDBvLwPcF5ELIyIucADwBRamGO+\nrB045quGmC9JewEnAaOAn0bECf3k2wH4X2DfiLhoMGXNzEZIAFdLWgycGhGnAeMjYkHevgAYn5fX\nB24slJ0PTBixmg6BY76sHTjmq0rPl6RRwI9IXe5bAPtL2ryffCcAVwy2bCeRNLXZdaiXTjmXTjkP\n6KxzaaKdImJb4D3A5yXtUtwYEUFqoPWnq3+tm1l9VOv5mgI8kLvckXQ+qSv+3rJ8hwMXAjsMoWwn\nmQr0NrkO9TKVzjiXqXTGeUBnnUtTRMSj+f0JSZeQvqcWSFovIh6T9Drg8Zz9EWBSofjEnLYcST2F\n1d6I6K133c2sOfIP36n13Ge1xtcEYF5hfT6wY1mlJpAaVe8kNb6i1rJmZiNF0qrAqIh4QdJYYE9g\nBnAZcACp9/4A4NJc5DLgXEknkr7PJgM3Vdp3RPQ0tva1KcV7+fajtbJSvFe73H7MP6Z6S+uShv0B\nq9b4qqWL/SRgWkSEJNEXzOrueTNrKElnk4LiZ9aQfTxwSfqaYjRwTkRcJekW4AJJBwJzgY8DRMRs\nSRcAs4FFwKH5tmTLcqPL2kG7NLoaqVrjq7zbfRKpB6toO+D8/IW2DvAeSQtrLAuApJb+QhuMerSI\nW0WnnEunnAd01rnUyUHAvpJ+CfyJ9GDPi5UyRsTDwDYV0p8G9uinzHHAcfWrrllnkrQusOkwdvFI\n/ox2hWqNr1uAyZI2AP5OGvNm/2KGiNiotCzpTODyiLhM0uhqZXP5rm8Bm9mQvQbYCHiO9KTiGaTv\nGjMbWXvAamfCRq8MvujjK8FTJwNH1b1WLWrAxldELJJ0GHAlabiI0yPiXkkH5+2nDrZs/apuZsaR\nwCkR8SCApHlV8nc0x3xZc+3+Kly6ZrVcy1+nJwDddSuy6jhfOZZiZllaxUZXRHy6WlkzszrqLTS8\n3hcRv212hZrJjS5rB75OmzzCvaS98pxp90s6upl1qUTSGZIWSLqrkDboeeAkbSfprrzt+yN9HrkO\nkyRdJ+keSXdLOqIdz0fSypL+LOl2SbMlfbMdz6NI0qg81+Dleb0tz0V1mjdxkOeya2F5l35zmZm1\nkKY1vtQeg7CeSapf0WDmgSt1o/4YODAiJpPi4Mr3ORIWAl+KiC2Bt5IGmNycNjufiHgF2C0itgHe\nDOwmaed2O48yXyA9UVd68KRdz2W48yYO5VzWlbS7pHfSNzK9mVlLa2bP19JBWCNiIVAahLVlRMQN\nwDNlyYOZB25HpUEbV4+I0vhAZxfKjJiIeCwibs/L/yANdjuBNjyfiHgpL44hxRM+QxueB4CkicB7\ngZ/SN0xLW55LNpx5E4dyLkcAmwCbAV8cfvXbm+d2tHbg67SGmK8GatdBWAc7D9xClh1i4xGaPD9c\nfgJ1W+DPtOH5SFoBuBV4I/DjiLhHUtudR/Y94N+BNQpp7Xou9Zg3cbDn8npgTWAlUg/ifw33JNqZ\nY2msHfg6bW7jq+3H9soDy7bVeUhaDbgI+EIe6XvptnY5n4hYAmwjaU3gSkm7lW1vi/OQ9H7g8Yi4\nTf3M29gu55LtFBGPKo33M0vSnOLGBp3Ll4HvkhptZmZtoZmNr5oHYW0xg5kHbn5On1iWXnF+uEaT\ntCKp4fXziChNodK25xMRz0n6LWmg33Y8j7cDe0t6L7AysIakn9Oe51KPeROHci53R8Td9ToHM7OR\n0MyYr6UDuEoaQwq+vayJ9alVaR44WH4euP0kjZG0IXkeuIh4DHhe0o45oPiThTIjJh/7dGB2RJxU\n2NRW5yNpndITc5JWAd4F3NZu5wEQEcdGxKSI2BDYD7g2Ij7ZjuciaVVJq+fl0ryJd9H4c9lN0uWS\nfiXpV/U/s/biWBprB75OgYho2gt4D/BXUrDtMc2sSz/1O480Ov+rpPi0TwNrA1cD9wFXAeMK+Y/N\n5zIHeHchfTvSH6IHgB806Vx2BpYAt5MaK7eRnjJrq/MBtiLFe90O3An8e05vq/OocF67Ape167kA\nG+b/k9uBu0uf50afC7AasENentik/7to0nFXh5VehYjBv+4OGDdviMd9D+z87NCOO5zXtMXAsYP5\nf2nW/02d/n+XwOIh/DvNCxj7dBPquz/s8/zQ/m+PD1jpu83+Nx/MtTXcfSjvyMys7Ug6DXg1Ij4v\n6ZSIOLQJdYhowjRpqadxpafglRUHX/oeYOf5Ec9Mqpp1+eO+B3Y+D26oOpJ5fR2zBI7/j0jzbVZV\nii9sxv9NPUhaAos1+BtU84HNnon4x9qNqFd/JO0P+5wKl64++NInANNPjHjlyLpXrAHq8ZlvZsyX\nmdlw/YO+4WBebmZFzMpJq5wAY4Y6/lxbNhqtNm58mVk7exLYRdJ3SbfVu5rndmw1ow+Ef39Nmv99\nKDqz/eXr1I0vM2tjEfENSZsBK0TE7GbXp9m6+Y9Z6/oXYINmV6Kl+Dp148vM2pik8/LiKpKIiGaN\n7G9mVjM3vsysbUXE/rB0KJUvNbk6ZmY1cePLzNqWpC1JQwqsCGzZ5Oo0nWNprB34OnXjy8za20fz\n+z+BHzSzIq2gm/+YWfvwderGl5m1t1sKyxMlTYyI3zatNmZmNXDjy8za2WeBP5JuPe5ME6buMjMb\nLDe+zKydzYmI7wBIWjcizmp2hZrJsTTWDnyduvFlZm1O0umknq8Fza5Ls3XzHzNrH75O3fgys/b2\nVWAi8Cwp6N7MrOUNdsZOM7NWchIwPSKeB37Y7MqYmdXCjS8za2dLgP/Ly882syKtoKdnxtJ4GrNW\n5evUtx3NrL39E9hC0uHAWs2uTLM5lsbaga9TN77MrE3lKYUuBNYBBJzS3BqZmdXGtx3NrC1FRAC7\nRcTMiPhdRCyupZykUZJuk3R5Xl9b0ixJ90m6StK4Qt5jJN0vaY6kPRt0KmbWZdz4MrO2JGkfYB9J\n10j6laRf1Vj0C8Bs0vAUANOAWRGxCXBNXkfSFsC+wBbAXsApklr6O9OxNNYOfJ22wG1HSVE9l5l1\nmojQMHexV0TsJOnHEXFILQUkTQTeC3wD+HJO3hvYNS+fBfSSGmD7AOdFxEJgrqQHgCnAjcOsd8M4\nlsbaga/TFmh8QV2+hFuCpJ6I6Gl2PeqhU86lU84DOu5c6vGj6/WS3pff3wsQEb+rUuZ7wL8DaxTS\nxkdEaYDWBcD4vLw+yza05gMThl1rM+t6Ld2FbmY2gF+Rgu0vANbNr35Jej/weETcRgrQX06OIxuo\nYeieejMbtpbo+TIzG6yI+Nkgi7wd2Dv3kq0MrCHp58ACSetFxGOSXgc8nvM/AkwqlJ+Y05Yjqaew\n2hsRvYOsW114zjxrB+12nUqaCkyt5z7d+Kqv3mZXoI56m12BOultdgXqqLfZFWhnEXEscCyApF2B\noyLik5K+BRwAnJDfL81FLgPOlXQi6XbjZOCmfvbd09ja16Zd/phZd2u36zT/mOotrUsa9gm48VVH\nzfq12widci6dch7QWefSIkq3EI8HLpB0IDAX+DhARMyWdAHpychFwKH5tqSZ2bC48WVmXScirgeu\nz8tPA3v0k+844LgRrJqZdYEhB9xLOkPSAkl3DZDnB3mAwjskbTvUY5mZWXUeP8naga/T4fV8nQn8\nEDi70sYc1LpxREyWtCPwY+CtwziemZkNoN1iaaw7+TodRs9XRNwAPDNAlr1JAxYSEX8GxkkaP0B+\nMzMzs47XyHG+JgDzCuvzSY9qm5mZmXWtRg+yWj6QoZ8UMjNrEMfSWDvwddrYpx2HMkDhVKAHmNoq\n4+aY2fA1YpBCW55jaawd+DptbM/XZcC/Akh6K/BsYf60ZUREac66XfNYRkv/Z0oNs+II0oNNG0wZ\nM6u/iOgtfc79w8rMut1whpo4D/gTsKmkeZI+I+lgSQfD0gluH5L0AHAqcOgQDzW97H0oaTWXqUdj\nbzANQDMzM+suw3nacf+IWD8ixkTEpIg4IyJOjYhTC3kOi4iNI2LriLi1PlVuuHo09mpuADaqsddf\nA9DMOpdjaawd+DoFIqKpr1SFZZfrnTbYMsCaA+UDfjbUOgArjsQ5D1CHnuJ7I9P88qu/V/GabPdX\ns84FWB1WehUiBv+6O2DcvCEe9z2w87NDO+5wXtMWA8cO5v+l2dcZrPYkPDzC/07zAsY+3YTrcX/Y\n5/mh1fn4gJW+28z/q0Ge67Cvq0Y/7dgUkoZ7XmtV2R7D2PeYWjJJKn9SdNjyPqfn1emFTQ1Ja0RM\nnnv7zMys3bVF40vSpfn995LWz8ufyu9/lLRbXu7NRa6QtHIpTdKsvLx9YT9H5uWe/P7bQvlDctq1\nkjYfoF6l/a5QaixJuja/T5BU2s91ko7NxWbmtHUkXZKXf5Hfp0q6POf7VOE4pX335vfVC9v+N7/v\nLmmNsnwr5qfMkHRxaZ+STurvnOqs1oZbte1t2QA0MzOrqJW670rL5WnAL/L7u4Hv5+WZ+X0scF1e\nvq6wjyMq7GdWYftlOW16Ie34nPaGgepFmlopCvv9EPD+srQf9nMuq+b37wC75eWv5PddSZP9Vvx3\nKOzjs8C6eXl0fhdwVFm+T+Z9ltdho1r+3UcqrYPr0FN8r5Y2lDL97afVX8V/p3Z/NetcqHDbsaen\nJ3p6espu6VR6+bbjyPwf+bZjpdfy16lvO7aqUrD+LcDkvLxlfv8NsE6FMptVSHszpJ4olh2DrGRe\nhbRaTAY2L+wbUtzYQDYHShGHHyqk31KeUdJq+b03J60PbAgQEYvyewBvrFCv5UTEQ1XqZvUxvey9\nWtpQyiyXNlJP6voWcOvp6ZnuMZSs5fk6bZPbjsC2+X174P68fCdAROxW2F50b2mhEAN2R6HMdgMc\nb+Eg63c/MKewb4DP9JO3NLDtHODYXOZthe1LKpTZM+ebmtcFPAggaXR+XwF4oKzcfbWegHWUujbm\n6rUfN8LMzJJ2aXyVgtS/CnwrL58PS2OsvlOhzGl5ey9wZU6bVijz2wGO92jO9ytJ5b1JS0m6Oi/+\nOiIuL+wb0i2/Ur4LJb09r16Q378BfClvv6aw26hwqBtzvt+U8kTEU3n5j/l9N5Y9Z8j/RmYtort/\n6pqZlbTSvVOW3j1bNg34fC35+ksbSpl6p7kOrV0v16E59eqEV7POBcd8Vf1/afZ15pivyi/HfEVD\n53asp2jWgSUdl9+vy+/HRsRxzaqPmVl/uj2OxtqDr9M2ue0YEac08diluKzd8rsbXmZmZjZkbdH4\nMjMzM+sUbnyZmXUIz5ln7cDXKW0T82VmZlUMLpZmyQqSxg/hMNWmXzMbkGO+3PgyM+tCo4BR68Aa\nDw+t/HjfNTEbBje+zMy6zmbA02OqZjOzhvCvFzOzDuFYGmsHvk7d82Vm1jEcS2PtwNfpMHu+JO0l\naY6k+yUdXWH7OpKukHS7pLslfWo4xzMzMzNrd0NufEkaBfwI2AvYAthf0uZl2Q4DbouIbYCpwHdL\nE0GbmZmZdaPh9HxNAR6IiLkRsZA0ifM+ZXkeBdbIy2sAT0XEomEc08xsSCStLOnPuSd+tqRv5vS1\nJc2SdJ+kqySNK5Q5Jvfsz5G0Z/NqXxvH0lg78HU6vJivCcC8wvp8YMeyPKcB10r6O7A68PFhHM/M\nbMgi4hVJu0XES7kH/g+Sdgb2BmZFxLdy+MQ0YJqkLYB9ST37E4CrJW0SEUuadhJVOJbG2oGv0+H1\nfNUy2fWxwO0RsT6wDXCypNWHcUwzsyGLiJfy4hjSYFfPkBpfZ+X0s4AP5uV9gPMiYmFEzAUeIPX4\nm5kNy3B6vh4BJhXWJ5F6v4reDnwDICIelPQwsClwSzGTpJ7C8tRh1MnMWlDpc138rDepHisAtwJv\nBH4cEfdIGh8RC3KWBUBp1Pf1gRsLxeeTesDMzIZlOD1ftwCTJW0gaQype/6ysjxzgD0A8jQWmwIP\nle8oInoioicv9w6jTmbWgkqf6+JnvUn1WJIfAJoIvEPSbmXbg4F79Wvp8W8ax9JYO/B1Ooyer4hY\nJOkw4EpS9/3pEXGvpIPz9lOB44AzJd1Bauh9JSKerkO9zcyGLCKek/RbYDtggaT1IuIxSa8DHs/Z\nynv3J+a05ZT16PU260ekY2msHbTbdZp77qfWc5/DGvYhImYCM8vSTi0sPwl8YDjHMDOrB0nrAIsi\n4llJqwDvAmaQeuwPAE7I75fmIpcB50o6kXS7cTJwU6V9N7M3r4sIVj5KWvugQRXS2g/D4sXw/Bci\n4reNqpx1rvxjqre0LmnYrUePuWVm3eJ1wFk57msF4OcRcY2k24ALJB0IzCU/lR0RsyVdAMwGFgGH\n5tuS1hSHC/ZdC1irtvzb5vdrN4AjX4Jrayxn1nhufJlZV4iIu4C3VEh/mhybWmHbcaTwibZQiqNp\nt9s6tVk/vwZrG2Bcyw4P0o06+zqtjRtfZmYdopv/mDWKpNcAKw6t9FjVtTIdwtepG19mZmYDWO0K\nWLIVjB5C79noUel5NLNlufFlZmbWr5XGwMUrwTuaXRHrIMMZ58vMzFqIx0+yduDr1D1fZmYdw7E0\n1kd24SIAABKcSURBVA58nbrxZWZm3WFLSe8cfLHVV6t/VazbufFlZmYdbtuVYd4hwCGDLztqNKxZ\n9xpZd3Pjy8ysQ3j8pP58bTR8zS2oFuHr1I0vM7OO0c1/zKx9+Dr1045mZmZmI8qNLzMzM7MR5MbX\n/2/v3mPsOMs7jn9/OCbEgGysVA7YWwVaR42rQsPFSbllS6JkSakdFYlgAoq4FEtgoBeK7aptVqoU\nEUTBqkIiN5jIjUjcNElTU2ESJ83hmpuFkzjYbryAW1+IE1JIuUXY5Okf8x4yOd5dnzNzdubM2d9H\nsnZmzsyZ5x2/M/vMzLMzZmZDws9PsiZwP3XNl5nZ0HAtjTWB+6mvfJmZmZlVqnDyJWlM0l5J+ySt\nnWKeUUk7JT0iqVU4SjMzM7MhUei2o6Q5wFXA+cAh4AFJWyNiT26eBcDngAsj4qCkU/sRsJmZTc7P\nT7ImcD8tXvO1HJiIiP0AkrYAK4E9uXneBdwSEQcBIuKHJeI0M7MTmM2/zKw53E+L33ZcDBzIjR9M\n0/KWAgsl3S1ph6T3FFyXmZmZ2dAoeuUruphnLvBq4DxgHnCPpHsjYl/BdZqZmZk1XtHk6xAwkhsf\nIbv6lXcA+GFE/AL4haSvAa8Cjku+JI3nhkcLxmRmA6q9X+f3des/19JYE7ifFk++dgBLJZ0OHAYu\nAVZ1zPPvwFWpOP9k4GzgM5N9WUSMA0i6PCJakgqGZWaDqL1f5/f1mkMaSrP5l5k1h/tpweQrIo5J\nWgPcDswBNkXEHkmr0+cbI2KvpK8ADwPPANdGxO5+BW5mZmbWRIWfcB8R24BtHdM2dox/Gvh00XWY\nmZmZDRs/4d7MbEj4nXnWBO6nfrejmdnQcC2NNYH7qa98mZmZmVXKyZeZmZlZhZx8mZkNCdfSWBO4\nn7rmy8xsaLiWxprA/dRXvsxslpA0kt41+x1Jj0j6aJq+UNJ2SY9KukPSgtwy6yXtk7RX0gX1RW9m\nw8TJl5nNFkeBP4+I3wXOAT4s6UxgHbA9Is4A7krjSFpG9vaOZcAYcLUkHzPNrDQfSMxsVoiIxyLi\nwTT8U2APsBhYAWxOs20GLk7DK4EbI+JoROwHJoDllQbdI9fSWBO4n7rmy8xmofRe2rOA+4BFEXEk\nfXQEWJSGXwbcm1vsIFmyNrBcS2NN4H7q5MvMZhlJLwJuAT4WET+R9OvPIiIkxTSLT/qZpPHcaCsi\nWn0I1cwGgKRRYLSf3+nky8xmDUlzyRKv6yPitjT5iKTTIuIxSS8FHk/TDwEjucWXpGnHiYjxGQrZ\nzGqWTqZa7XFJpS/duebLzGYFZZe4NgG7I2JD7qOtwGVp+DLgttz0d0p6vqSXA0uB+6uKtwjX0lgT\nuJ/6ypeZzR5vAN4NPCxpZ5q2HvgkcJOk9wP7gXcARMRuSTcBu4FjwIciYrpbkrVzLY01gfupky8z\nmyUi4htMfbX//CmWuQK4YsaCMrNZybcdzczMzCrk5MvMbEi4lsaawP20xG1HSWPABmAO8PmIuHKK\n+V4H3AO8IyJuLbo+MzObnmtprAncTwte+ZI0B7iK7JUby4BV6TUdk813JfAVQJ2fm5mZmc02RW87\nLgcmImJ/RBwFtpC9iqPTR4CbgScKrsfMzMxsqBRNvhYDB3Ljx712Q9JisoTsmjRpoP9E28ys6VxL\nY03gflq85qubRGoDsC69rkNMc9sx/2qO9Bh/Mxsi7f264zU81meupbEmcD8tnnx1vnZjhOzqV95r\ngC3pvWmnAm+VdDQitnZ+WfvVHJIuj4hW/l1rZtZ87f06v6/XHJKZWW2KJl87gKWSTgcOA5cAq/Iz\nRMQr2sOSrgO+NFniZWZmZjabFEq+IuKYpDXA7WSPmtgUEXskrU6fb+xjjGZm1oV2HY1v69ggcz8t\n8ZyviNgGbOuYNmnSFRHvLboeMzPrzmz+ZWbN4X7qJ9ybmZmZVcov1jYzMxsqP3uJ9Lxnii17yrcj\nfvba/sbTjV8tkfTmggtPRMThvoYzw5x8mZkNCdfSWPbIzWNQ6K0y3wIuPqW/8Rzv+H66BPi9CyEu\n7P3b/vsF8KO/BD7Xr/iq4OTLzGxIOOmyLOeaU3DZosv15vh+eilw6fxi3/bBp+HasiFVzjVfZmZm\nZhVy8mVmZmZWISdfZmZDwu/MsyZwP3XNl5nZ0HDNl5X3yxdL+qMCC57V7Yzup06+zMzMDIAFwPKF\n8KsvFlv+7Bn/S8lh4eTLzMzMgGXAXS+sO4rZwDVfZmZDwrU01gTup77yZWY2NFxLY03gfuorX2Zm\nZmaVcvJlZmZmViEnX2ZmQ8K1NNYE7qeu+TIzGxqupbEmcD8teeVL0pikvZL2SVo7yeeXSnpI0sOS\nvinplWXWZ2ZmZtZ0hZMvSXOAq4AxsoeDrJJ0Zsds3wPeHBGvBP4e+Kei6zMzK0PSFyQdkbQrN22h\npO2SHpV0h6QFuc/WpxPLvZIuqCdqMxtGZa58LQcmImJ/RBwFtgAr8zNExD0R8VQavQ9YUmJ9ZmZl\nXEd2spi3DtgeEWcAd6VxJC0DLiE7sRwDrpY08DWyrqWxJnA/LVfztRg4kBs/CJw9zfzvB75cYn1m\nZoVFxNclnd4xeQVwbhreDLTIErCVwI3pxHK/pAmyE857Kwm2INfSWBO4n5ZLvqLbGSX9IfA+4A0l\n1mdm1m+LIuJIGj4CLErDL+O5idZBshNOM7PSyiRfh4CR3PgI2QHqOVKR/bXAWET8aLIvkjSeGx4t\nEZOZDaD2fp3f1wdNRISk6U4quz7hNDObTpnkawewNF3GP0xWH7EqP4Ok3wRuBd4dERNTfVFEjKf5\nL4+IlqQSYZnZoGnv1/l9veaQ2o5IOi0iHpP0UuDxNL3z5HJJmjapjqSyFRGtfgfajXYdjW/r2CBr\nWj9NJ4+j/fzOwslXRByTtAa4HZgDbIqIPZJWp883An8HvAS4JiVURyNiefmwzcz6YitwGXBl+nlb\nbvoNkj5DdrtxKXD/VF/STirr1pRfZja7Na2fppOpVnu8HyePpR6yGhHbgG0d0zbmhj8AfKDMOszM\n+kHSjWTF9adKOkB2cvhJ4CZJ7wf2A+8AiIjdkm4CdgPHgA9FhG87mllf+An3ZjYrRMSqKT46f4r5\nrwCumLmIzGy2Gvjn1piZWXf8/CRrAvdTX/kyMxsaTaulsdnJ/dRXvszMzMwq5eTLzMzMrEJOvszM\nhoRraawJ3E9d82VmNjRcS2NN4H7qK19mZmZmlXLyZWZmZlYhJ19mZkPCtTTWBO6nrvkyMxsarqWx\nJnA/9ZUvMzMzs0o5+TIzMzOrkJMvM7Mh4VoaawL3U9d8mZkNDdfSWBO4n/rKl5mZmVmlSiVfksYk\n7ZW0T9LaKeb5x/T5Q5LOKrM+MzMzs6YrnHxJmgNcBYwBy4BVks7smOci4LcjYinwQeCaErGamdk0\nXEtjTeB+Wq7mazkwERH7ASRtAVYCe3LzrAA2A0TEfZIWSFoUEUdKrNfMzCbhWhprAvfTcrcdFwMH\ncuMH07QTzbOkxDrNzMzMGq1M8hVdzqeCy5mZmZkNnTK3HQ8BI7nxEbIrW9PNsyRNew5pPCVklyO1\nAgKpnaS1h8tM69f3OIbhjcsxzGxcrcj273GffM2gdh2Nb+vYIHM/BUUUOxZKOgn4L+A84DBwP7Aq\nIvbk5rkIWBMRF0k6B9gQEed0fE9EROfVMTMbYsO039fVFkkvhpOfhKfnVr3uZmj/lzjfH24ffBqu\n/XhEfK6qNfZjny985SsijklaA9wOzAE2RcQeSavT5xsj4suSLpI0AfwMeG+ZYM3MzMyartQT7iNi\nG7CtY9rGjvE1ZdZhZmZmNkz8hHszsyHh5ydZE7iflqj56lsAQ1T7YWbdGab93jVfg8o1X7NDM2u+\nfOXLzMzMrEJOvszMzMwq5OTLzGwaksYk7ZW0T9LauuOZjmtprAncT13z1VeSRiOiVXcc/TAsbRmW\ndsDQtaUR+72kOWTPMzyf7AHRD3D88wwHrOarBYxWHc40WtQTz1Q1Xy28fU6kxWDF1GLqeFzzZYPV\nW8sarTuAPhmtO4A+Gq07gFloOTAREfsj4iiwBVhZc0wn0Ko7gA6tugPo0Ko7gA6tugOYRKvuADq0\n6g6g75x8mZlNbTFwIDd+ME0zMyus1ENWzcyG3IA/p+CXJ8FbnmqPjY+fO//uu1t89avPTqvf914A\nX3u6hhXPz350bova4pnCoMUDMx3T+Pi587OfX+2yn04Xz56T+xVXlQai5qvWAMysFg2p+ToHGI+I\nsTS+HngmIq7MzeNjmNksU/b4VXvyZWY2qCSdRFZwfx5wGLifjoJ7M7Ne+bajmdkUIuKYpDXA7cAc\nYJMTLzMry1e+zMzMzCpU6187NunhhXmSRiTdLek7kh6R9NE0faGk7ZIelXSHpAV1x9otSXMk7ZT0\npTTeyLZIWiDpZkl7JO2WdHYT2yJpfepfuyTdIOnkprRD0hckHZG0KzdtythTW/elY8EF9UR9vG6P\nT5JeJ+mYpLf3umyF8eyX9HDax+/vRzzdxCRpVNJTab07Jf1Nr+2Z4Xj+NvdZ37dRN21MMe1Mv0ta\nvSxbcTyVbx9JH8/9X+1K/XpBt22pIabut1FE1PKP7BL+BHA6MBd4EDizrnh6jP004PfT8IvIakLO\nBD4FfCJNXwt8su5Ye2jTXwBfBLam8Ua2BdgMvC8Nn0T2F0+NakvaJ74HnJzG/wW4rCntAN4EnAXs\nyk2bNHZgWdr356Z2TwDPG4A2dHV8SvP9J/AfwNt7WbaqeNL07wMLq95GZM+m21q0PVXFMxPbqMt4\nFgDfAZak8VNr3j6TxlPX9umY/23AnTO1fcrG1Os2qvPKVwMfXpiJiMci4sE0/FNgD9mzf1aQ/fIn\n/by4ngh7I2kJcBHweZ59LHTj2iJpPvCmiPgCZPU6EfEUzWvL/wFHgXnKCr7nkRV7N6IdEfF14Ecd\nk6eKfSVwY0QcjYj9ZAe+5VXEeQLdHp8+AtwMPFFg2ariaev3X5d2G9Nk661zG023Hfq5jbqJ513A\nLRFxECAiftjDslXG01b19umM7caCy1YRU1tX26jO5GsoHl4o6XSys/z7gEURcSR9dARYVFNYvfos\n8FfAM7lpTWzLy4EnJF0n6duSrpX0QhrWloj4X+AfgP8hS7p+HBHbaVg7OkwV+8vI9v22QTkOnPD4\nJGkx2YH5mjSpXUA7E8e2MvG0h++UtEPSn5aMpeuY0npfL+khSV+WtKyHZauMp/1ZP7dRN/EsBRYq\nK2PZIek9PSxbZTxQz/YBQNI84ELgll6XrTAm6GEb1fnXjo2v9Jf0IrIN/7GI+In0bMIbEaEGPP9H\n0tuAxyNip6TRyeZpSlvI+vOrgTUR8YCkDcC6/AxNaIuk3wL+jOzS91PAv0p6d36eJrRjKl3EPgjt\n6iaGDcC61B4x9csE644H4A0R8QNJvwFsl7Q3XaGc6Zi+DYxExM8lvRW4DTij5HpnKp5+b6Nu4plL\ndsw6j+wK9z2S7u1y2criiYh9wBsj4nDF26ftj4FvRMSPCyzbizIxQQ99qM4rX4eAkdz4CM89Ax5o\nkuaSJV7XR8RtafIRSaelz18KPF5XfD14PbBC0vfJLp++RdL1NLMtB4GDEfFAGr+Z7EDyWMPa8lrg\nWxHxZEQcA24F/oDmtSNvqv7UeRxYkqbVrZvj02uALWnfeTtwtaQVXS5bZTxExA/SzyeAf6M/t3ZP\nGFNE/CQifp6GtwFzJS1M81W+jaaJZya2UTf/ZweAOyLiFxHxJPA14FVdLltlPETE4fSzyu3T9k6e\ne3tvpvKHMjH11oeKFKX14x/ZVYrvkp3dP59mFdwL+Gfgsx3TPwWsTcPrGNCC6GnadS7wpSa3hexg\ncUYaHk/taFRbyA52jwCnpL62Gfhwk9qR9uvOgvvjYufZgvvnk902/i7pETg1x9/T8Qm4DviTIstW\nEM884MVp+IXAN4ELqthGZLeX2480Wg7sr3MbTRNP37dRl/H8DnAnWaH3PGBX2ifq2j5TxVPL9knz\nzQeeBE4puj9UFFNP26hUoH1o6FvJ/lJwAlhfZyw9xv1GsvqoB4Gd6d8YsDB13EeBO4AFdcfaY7vO\n5dm/dmxkW8gSlweAh8iuGM1vYluAT5D91dEusuRrblPaQXY2eBj4JdmZ9Hunix3463QM2AtcWHf8\nubiOOz4Bq4HVk8z762RnqmXrigd4RTpWPUiW1PftWHuimMhOGh5J6/4WcE6d22iqeGZqG3XzfwZ8\nPLevf7TuPjRZPDVvn8uAG7pZts6YyE4eu95GfsiqmZmZWYVqfciqmZmZ2Wzj5MvMzMysQk6+zMzM\nzCrk5MvMzMysQk6+zMzMzCrk5MvMzMysQk6+zMzMzCrk5MvMzMysQv8Pl1BA4XACJGQAAAAASUVO\nRK5CYII=\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x10ee541d0>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"clicks = range(1, len(bins_A))\n",
"\n",
"# Start with uniform probability over the bins\n",
"p_A = pm.Dirichlet(\"p_A\", theta=np.ones(len(bins_A)))\n",
"p_B = pm.Dirichlet(\"p_B\", theta=np.ones(len(bins_B)))\n",
"\n",
"# Alternativly giving it a more informative prior (did not help)\n",
"#p_A = pm.Dirichlet(\"p_A\", theta=[2**x for x in range(len(bins_A))][::-1])\n",
"#p_B = pm.Dirichlet(\"p_B\", theta=[2**x for x in range(len(bins_B))][::-1])\n",
"\n",
"# A multimodal dist. using the probabilitys of bins\n",
"obs_A = pm.Multinomial(\"obs_A\", p=p_A, n=sum(bins_A), value=bins_A, observed=True)\n",
"obs_B = pm.Multinomial(\"obs_B\", p=p_B, n=sum(bins_B), value=bins_B, observed=True)\n",
"\n",
"@pm.deterministic\n",
"def percent_better(p_B=p_B, p_A=p_A, clicks=clicks):\n",
" exp_clicks_B = np.dot(p_B.astype(float)/sum(p_B), clicks)\n",
" exp_clicks_A = np.dot(p_A.astype(float)/sum(p_A), clicks)\n",
"\n",
" return ((exp_clicks_B / exp_clicks_A) - 1)*100.0\n",
"\n",
"model = pm.Model([p_A, p_B, \n",
" obs_A, obs_B, \n",
" percent_better])\n",
"\n",
"map_ = pm.MAP(model)\n",
"map_.fit()\n",
"mcmc = pm.MCMC(model)\n",
"mcmc.sample(35000, burn=25000, thin=2)\n",
"\n",
"percent_better_samples = mcmc.trace(\"percent_better\")[:]\n",
"\n",
"print \"Probability B > A: {}\".format((percent_better_samples > 0).mean())\n",
"print \"Confidence interval of B:s lift over A:\"\n",
"print np.percentile(percent_better_samples, 2.5)\n",
"print np.percentile(percent_better_samples, 97.5)\n",
"\n",
"print \"MCMC error: {}\".format(mcmc.stats()['percent_better']['mc error'])\n",
"pm.Matplot.plot(mcmc)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment