tillahoffmann/masked_optimization.ipynb

## masked_optimization.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1006f93c8>,\n",
       " <matplotlib.lines.Line2D at 0x120d87d68>,\n",
       " <matplotlib.lines.Line2D at 0x120d87eb8>]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VOXd//H3NxsJ+xbWAGEXEBENiCioCLLUgr9WkbrR\nilvVaqvlUVofH+vS2lqrrQsUl4q4W2WrqCiCiAgSZAeBsJmwJWxhyZ7cvz9mtAEStpnkzPJ5XVeu\nmTlzZ8735gyfnLnPmXObcw4REYkuMV4XICIi1U/hLyIShRT+IiJRSOEvIhKFFP4iIlFI4S8iEoUU\n/iIiUUjhLyIShRT+IiJRKM7rAirTuHFjl5qa6nUZIiJhZcmSJbudc8knahey4Z+amkp6errXZYiI\nhBUz23oy7YIy7GNmQ8xsnZllmNn9lbQZaWZrzGy1mb0RjPWKiMjpCXjP38xigeeAQUAWsNjMpjvn\n1pRr0xEYB1zgnNtnZk0CXa+IiJy+YOz59wYynHObnHNFwFvAiKPa3Aw855zbB+Ccyw7CekVE5DQF\nI/xbApnlHmf5l5XXCehkZl+a2UIzGxKE9YqIyGkKxgFfq2DZ0ZMExAEdgYuBFOALMzvTObf/iBcy\nuwW4BaB169ZBKE1ERCoSjD3/LKBVuccpwPYK2kxzzhU75zYD6/D9MTiCc26icy7NOZeWnHzCM5VE\nROQ0BSP8FwMdzaytmSUAo4DpR7WZClwCYGaN8Q0DbQrCukVE5DQEPOzjnCsxszuBj4FY4GXn3Goz\nexhId85N9z93mZmtAUqBsc65PYGuu0JFh2H+01Xy0hKAGnWgz+0QG7JfLRGJKkH5n+icmwnMPGrZ\ng+XuO+Ae/0/VKs6HeU9U+WrkVPgPATXqAGcM87YUEQFC+Bu+p61WY3ho/4nbSfUpKYIn2sP6DxX+\nIiFCF3aTqheXAO0HwLqPoKzM62pEBIW/VJfOw+BwNny3wOtKRASFv1SXLj+GpIaw4FmvKxERFP5S\nXRJqwnm3+sb9c9Z7XY1I1FP4S/VJGwMxcbB0steViEQ9hb9Un9rJ0GkILH8LSou9rkYkqin8pXr1\nvM534Hflu15XIhLVFP5SvToOhhY9YfbDvm9ji4gnFP5SvWJi4LLH4OAO3/CPiHhC4S/Vr01faHom\nfDPJ60pEopbCX6qfGZwzGnYsh9mP+C7/ICLVKvKu7SPhoee1sHU+fPFXiE2Ai+/zuiKRqKI9f/FG\nQi0Y+Sp0GQ5f/h0O7vS6IpGoovAXbw18CEryYeHzXlciElUU/uKtRu19e//pr0DhIa+rEYkaCn/x\nXt9fQWEufPw7cM7rakSigsJfvJeSBhf+xnfq57f/8boakaig8JfQMOB/oW4KLNG5/yLVQeEvoSEm\nFnqMgo2z4cAOr6sRiXgKfwkdZ1/jG/Of/zevKxGJeAp/CR2N2kOvm2Dxi5CV7nU1IhFN4S+h5VL/\n2P/b18Erl0Pm115XJBKRFP4SWhLrwajXfef8b/kC0l/2uiKRiKTwl9DT/Cy4fyt0HwkbZkFZqdcV\niUQchb+EpphY6DwE8vbAZ4/CoRyvKxKJKAp/CV3tL4UadX1n/7x9nT4BiASRwl9CV1J9uHcdDH8G\nMhfCF096XZFIxFD4S2hLqAk9r4ezroY5j8HaGV5XJBIRFP4S+sx8e/8tesKMu+Hwbq8rEgl7QQl/\nMxtiZuvMLMPM7j9OuyvNzJlZWjDWK1EkrgZcMR4KDmj4RyQIAg5/M4sFngOGAl2Bn5lZ1wra1QHu\nAhYFuk6JUk26QOehsPJdKC3xuhqRsBaMPf/eQIZzbpNzrgh4CxhRQbtHgL8ABUFYp0SrHqPgcA5s\n/MzrSkTCWjDCvyWQWe5xln/ZD8ysJ9DKOaeLtUtgOgyCWsnw1TNeVyIS1oIR/lbBsh+mYzKzGOAp\n4N4TvpDZLWaWbmbpOTn6Uo9UIC4B+t0Lm+dp718kAMEI/yygVbnHKcD2co/rAGcCc81sC9AHmF7R\nQV/n3ETnXJpzLi05OTkIpUlESrsRajeDRRO9rkQkbAUj/BcDHc2srZklAKOA6d8/6ZzLdc41ds6l\nOudSgYXAcOecrtkrpyeuBnS/EjI+gby9vmXOQXGB5gAWOUkBh79zrgS4E/gYWAu845xbbWYPm9nw\nQF9fpEJnjYSyElj2uu/Sz39oAI81hZm/9boykbBgLkT3lNLS0lx6uj4cSCWcg5eH+C77AHDebZDz\nLWxdAL9ZDbWbeFufiEfMbIlz7oTfpYqrjmJEgs4MrvoXvDgI2l8CQ/8MuzfAs2kw5VaolwL1Wvlm\nBqvZ0OtqRUKOwl/CV90WcNdSiI33PW7cEfr91jcN5LZvoGA/LH8LbvxInwREjqJr+0h4i0vwfQr4\n3qX/65sI5v6t8POZcGA7TL/Lu/pEQpTCXyJX6gVw3q1HnhUkIoDCXyJd1xG+s4LWfeh1JSIhReEv\nka1FT6jXWvMAiBxF4S+RzQw6DoItX0BpsdfViIQMhb9Evrb9oegQbF/mdSUiIUPhL5EvtZ/vdvPn\n3tYhEkIU/hL5ajWCpt1hzTQoK/W6GpGQoPCX6HDB3bBzBcx/yutKREKCwl+iQ/crodtPYM5jsOET\nr6sR8ZzCX6KDGQx/Bpp0g7eugZX/9roiEU8p/CV61KgNo6dDi3Ngxt1QkOt1RSKeUfhLdKnZEIY+\n7jv185vJUHQYDmnKUIk+Cn+JPi16QpsL4PO/wPi+8HwfXftHoo7CX6LTFeOhbnPI3w/5++CzR7yu\nSKRaKfwlOjVoA7d+Ab9eAeeOhqWv+f4IiEQJhb9Er7gESKwHPa+H0qL/XvytIBeylmgyeIlomslL\npEVPaNgevnoeEuvDV89C5iLoeBlc886Rk8WIRAjt+YuYwcX3Q24mvHO9L/g7DYENs3zTQYpEIO35\niwCcNdL3DeCNn0HhAegwEJ7oAKvfh5Rzva5OJOi05y/yvdg46HSZ71IQSfWh/QBYPQWK8ryuTCTo\nFP4ilenzS98E8FN/CSVFXlcjElQKf5HKtL8ELnsE1kyFyVfoD4BEFIW/yPH0/RUMfxa2fuk7C0hD\nQBIhFP4iJ3LO9b7TPmf/AZ5oD+kvw/K3fNcF2r3B6+pETou5EP0iS1pamktPT/e6DBGfAztg5bu+\nn50rfMvikqAkH0bP8M0TLBICzGyJcy7thO0U/iKnoCAXNs2F4gJY/ibkfAt1msFNn0GMPkiL9042\n/IPybjWzIWa2zswyzOz+Cp6/x8zWmNkKM5ttZm2CsV6RapdYD7qOgB5Xww1T4dIHYftSWDPF68pE\nTknA4W9mscBzwFCgK/AzM+t6VLOlQJpz7izg38BfAl2vSEg462poeibMfhgKDnhdjchJC8aef28g\nwzm3yTlXBLwFjCjfwDk3xzn3/WkSC4GUIKxXxHsxsTD4Mdif6Zsb4KvnoKzM66pETigY4d8SyCz3\nOMu/rDJjgA+DsF6R0NDuYrjxI9/Y/8e/810SAqA4X1cGlZAVjPCv6JKHFb7jzew6IA14opLnbzGz\ndDNLz8nR1HoSRlr1hhtnQf02sOQVyPwa/tjS9zPrf32nhYqEkGCEfxbQqtzjFGD70Y3MbCDwe2C4\nc66wohdyzk10zqU559KSk5ODUJpINYqJ8U0Ms+UL3wTxSfWh81BY8Ay8PERTRUpICUb4LwY6mllb\nM0sARgHTyzcws57AP/EFf3YQ1ikSmtJuhCZdIXsNnH8HXPkS/OxN33cDlr7mdXUiPwg4/J1zJcCd\nwMfAWuAd59xqM3vYzIb7mz0B1AbeNbNlZja9kpcTCW9JDeAXM+FHT0Kf233LOg+Fpt1h3UxvaxMp\nJyjX83fOzQRmHrXswXL3BwZjPSJhIakB9LrpyGWdh8IXf4XDu6FWY2/qEilHX0kUqQ5dh4Mrg5cH\nw44VXlcjovAXqRbNuvvmAy7Oh9d+AlNug90ZXlclUUzhL1JdOg2G66f6vg+w6j346D6vK5IopvAX\nqU7JneC2+TDgAcj4FLbM97oiiVKawF3EC71v8V0KYu7jkJIGjTrCvCfgjB/5Dhh3GuwbKhKpIgp/\nES/EJ8F5t/kmiNnyhW9ZbA3fbGEA30yC276ExLre1SgRTeEv4pVeY2Ddh9D9Sigp9O3t71zpuz/9\nTvj0/+Dyp7yuUiKUwl/EK4n14KZPjlyW3Nl3u3MlLJoA59wALXpWf20S8XTAVyQUXTIOajeB927W\nPAFSJRT+IqEosR5c+S/YtxneucE3FCQSRAp/kVCVegH8+B+waQ5M/glkLYH938GO5b65hEUCoDF/\nkVDW81qIjfddIvrFAf9dntoPRs+AFe/ArpXQfSQ0P8u7OiXsKPxFQt1ZI31hvy0d8vf7DgZ//U/4\n9y9g9RSwGFj+Nlx8HxzK8X1iqN8aGqR6XbmEMIW/SDio2xzq/th3v6TQd3no1VOgxzW+M4L+NQQ+\nuNf3/OcA5jt1tPfN0EEX1ZVjKfxFwk1cDbhpNpSVQD3/dNl9bof8fdB/LORmwebPYdmb8PpIuGEa\ntO3nbc0ScsyF6ATTaWlpLj093esyRMJXQS68OBAKD8KvvoGEml5XJNXAzJY459JO1E5n+4hEqsR6\nvrOFDu6ATx/SJaTlCAp/kUjW5nzoNNR3gHh8X8hc7HVFEiIU/iKR7qcvwo0f+w4av32dvjEsgMJf\nJPLVqA2t+8BPX4ZDO2HW72HjHNg8z+vKxEM620ckWqScC2ljIP0l+OZVsFi491vfNYQk6mjPXySa\nDPsr3PwZjHgOXCmset/risQj2vMXiSYxMdDyXN/PogmwaDzs2+J1VXK0+q3g/DuqdBUKf5Fo1ed2\n+GgcLHvD60rkaC3OVviLSBU5+xrfj0QljfmLiEQhhb+ISBRS+IuIRCGFv4hIFFL4i4hEoaCc7WNm\nQ4C/A7HAi865x496vgbwKnAusAe42jm3JRjrltB3uLCEWWt2UlRS5nUpImGhYa0aDOratErXEXD4\nm1ks8BwwCMgCFpvZdOfcmnLNxgD7nHMdzGwU8Gfg6kDXLadvw66D7D1cxOxvs5m+bDv1kuL57eDO\nQX3Dbd+fz4erdjJpwRa+25sXtNcViXRnt6of+uEP9AYynHObAMzsLWAEUD78RwAP+e//G3jWzMyF\n6kwyEe67PXn8ZPwCDhaUADCkWzM25hzi5lfTSWmQRI+U+jx7TU+KSx0Jcac3Mjjn22zGTFpMmYNu\nLery2pjzaJdcK5jdEIlY8bFVPyIfjPBvCWSWe5wFnFdZG+dciZnlAo2A3UFYv5yC0jLHr95aigFP\nXtWD9k1qc3ar+hSVlPHMZxuYsy6bD1buIOv5fPKLSph+54Ws3p7Lhl2HGHZWc+omxp/Uep6dk0HL\nBklM+kVv2iXXrtpOicgpC0b4WwXLjt6jP5k2mNktwC0ArVu3DrwyOcbEeZtYnrmfv486mxFnt/xh\neUJcDPde1pmb+7fj/D/OZnnmfgCu/udXrNiWi3Pw4PTV9O/YmPo1E1i8ZS8XdUrmgR91PebTwZcZ\nu1mydR8PXt5VwS9SiWkZ02iU1IgLW154xHLnHEVlRdSIrVGl6w9G+GcBrco9TgG2V9Imy8zigHrA\n3qNfyDk3EZgIvjl8g1BbVCorc2zPzadJnUQS4mJ4+tP1vL7oO4pLy9ifV8zALk0Z3qNFhb9bNzGe\nccO6sGHXQXYfKmL2t7v4Rd+2DOvejA9X7WTq0m3szy+mb/tGvPrVVnYdKOCCDo3J2pfP7oOFLM/a\nT+befNo0qslVaSnV3HOR0Ha4+DDLspexZNcSXlj5AnExcUwcNJFezXoBkF+Sz4NfPkhhaSFPX/I0\nMVZ1wz8BT+DuD/P1wKXANmAxcI1zbnW5NncA3Z1zt/kP+P7EOTfyeK8bLRO45xeVkpQQe9w2G3MO\nMWnBFlZvP0DrhjWJjTGu79OGHq3qH9FuxvLtfLc3j9cXbmV7bgGXntGEq3u14pbJS7ioUzJN69ag\nc7O6XN+nzUmN5ZeUllFc6o6or6C4lLyiUhrWSuC5ORn8ddY6nPN9cqiVEEtaakOa1U3knkGdaFAr\n4fT+UUQiUMa+DG7+5GZ25/tGu/u17EfWoSxyC3MZdcYo6ibUZdaWWSzNXsqvz/01v+j2C8wqGjQ5\nvpOdwD3g8PevbBjwNL5TPV92zj1mZg8D6c656WaWCEwGeuLb4x/1/QHiykRD+L+TnskDU1fxyi96\n0bd94x+WO+f4fH0On6zZBcBbizOJjzXObFGP7/bmkVdUSklZGf8Y1ZPLujXjYEExK7JyufbFRQB0\nb1mPs1vVZ/LCrZhBxya1+c+v+p32wdvjySsqITe/mGZ1E0/rjSoSDbLzshk5YyQxFsPDFzxMpwad\nSE5KJmN/BqP+M4qisqIf2j7R/wmGtB1y2uuq1vCvCpEe/tkHCxj45OccKCghpUESvx7YiQP5xVze\nozmPz/yW95duIzE+hoLiMn5yTkt+P6wLjWr7xgB3HypkzCuLWbktlws7JrMgYzclZY6W9ZN4//a+\nNKnja/f83I3kF5UyqncrUhrU9LK7IlGppKyE9ze8z2trX2Pn4Z28MewNOjTocESbxTsXU+bK2Jy7\nmaS4JEZ0GBHQOhX+ISo3r5ivNu1m8sKtpG/Zx6NXnMmjH6wlN78YgPhYo6TMcfelHbn94g6UlJVR\nM+HYQzN5RSX85aN1fLJmF+e1bUhBSSnXnteGCzo0PqatiFS/g0UHuXnWzazes5qODTrym3N+Q7+U\nflW+XoV/iEnfspe3F2fy2bfZ7Dns+4j355925+perSksKWVj9mF2HsjnH7MzuPvSjlxyhuZVFQlH\nzjn+b8H/sWTXErYf2s6f+v2JwamDq21Y9GTDX5O5VIMX5m3isZlrqZcUT6/UBozum0psjHF+u0YA\n1IiLpWuLunRtUZcBZ1Ttt/pEpGqt3L2SKRlT6NigI4/3f5zBqYO9LqlCCv8qsH7XQf69JIt6SfFk\nHyhg8sKtDD2zGU+O7FHhEI6IRI4PN39IQkwCk4ZMok5CHa/LqZSSqAr8aeZa5qzLAaBGXAyXdVXw\ni0SD0rJSPtryEf1S+oV08IPCP+gy9+Yxd30Odw3owO2XdCA+NobYGJ0CKRJunHOnPE6/eNdidufv\nZmjboVVUVfDoev5BtDIrlzGTFmPAqN6tSYyPVfCLhKEVOSvo80YfHlrwEAeKDpz07324+UNqxtXk\nopSLqrC64FD4B0leUQm3v7GEA/kljL/uXFrUT/K6JBE5TS+tfAmHY1rGNH72n5/98K3cyjjneHnV\ny3y4+UMubX0piXGJ1VTp6VP4V6K4tOyUJh8ZP3cjWfvyeeaangzu1qwKKxORqpR5IJM5mXO4vuv1\nvDj4RbIOZfH62teP+ztzM+fy1JKn6N64O788+5fVVGlgFP4VOFxYwuX/mM+tk0/uewZFJWW8+fV3\nDOzSlF6pDau4OhGpSh9v/RiH46pOV3Fu03O5KOUi3lv/HgUlBRW2Lyot4ulvnia1bioTBk2gVZ1W\nFbYLNVER/jtzCxj3/krStxxzIdFjZGQf5LbXlrBu10HmrMth7Y7jj/eVlTneWLSV3YeKuOY8XYZa\nJNRtyt3EhOUTKg3z2Vtn071xd5rV8n2Cv7bLtewr3MeIqSNYsH3BEW2dc/xx0R/ZlLuJsb3GEh9z\ncvNdhIKIDv9Zq3eyZvsBLn/mC978+jsenLaa432jOedgIVdN+Iql3+1n7ODOJMbH8M/PNwK+0zdv\nf30JxaX/HQpyznH328t4aMYazmhWh/4dk6u8TyJy6uZlzeOB+Q+wJXcLYz4ew3PLnuOO2XcwY+MM\n7pl7D0t2LeG2T2/jpZUvsWrPKga0HvDD757X/Dyeu/Q5kuKSuO2T2xi/fDz7C/Yzfvl47pt3H+9t\neI+but9E/5T+Hvbw1EXsqZ4Z2Qe5ZfISYgxizLi5X1te+GIzc9flcFGnZBZu3kOHJrVpUsd3YKaw\npJR7313O4aJSZt51IR2a1OFwYQnPz91I03qJvPDFJsoctKj3LQ9c3hWASQu2MGP5dn41oAN3XNJB\nZ/aIhJil2Ut5Y+0bzMmcQ2FpIV9u/5KCkgLu6nkXE5ZP4OudXwMwJ3MOJWUlfLntSxomNjzmVM3+\nKf1Ja5rGowsf5fllzzNh+QTKXBmGcUPXG7ir511edC8gERv+s/yXQ06Ii+HGC9ry64Gd+GTNLn77\n7nKS69Tg250H6Z3akLdv7QPAHa9/w7z1OTz+k+50aOL7csY9gzqxdscB/vn5JpLiYxnUtSkvf7mZ\nTs3qUFhSxl9nreeiTsncM6iTLmcsEkKmbJjCtI3T2Fewjx2Hd9CtUTf2F+5nU+4m7j7nbm7qfhOD\n2gxi/rb5ZOzP4L0N7/Fw34dJa5ZGs1rNKhy+qRlfk8cufIwhbYcwL2seP2r3I85sdCbxseEz1FNe\nxF3YraS0jDnrcnj60/XExhhv3NyHWgmxmBkbcw5x1YSvaFAznnNaN+DdJVk8MqIbhSVlPPrBWh74\nURdu6tfuiNcrLXO8/00WdRLjSUttQP+/zCGvqBSAuBjj49/0p72mKhQJGXnFeQx5bwj7CvcB8NeL\n/srg1MHM3zafyWsm8/QlT5MU999TsQtKCvh659dc2PLCKp05q7pE7YXd9uUVc/Orvj8aYwd3pnaN\n/3axfXJtFtw/gITYGMqcY332If53mm/CsfPbNeLGC9oe83qxMcZVaf89ev+H4d1Yu+MgF3Ro9MNr\nioj3pmZMZfHOxdSIrcG+wn3c1uM2DhUdYlCbQQBc2PLCY+bLBUiMSwy78fpgiLg9/6KSMtbtPMjB\ngmLOadOAxPjKp0gsKS1j7rocypyjf6fk47YVkdA1f9t87ph9B2XOd0LG1Z2v5oE+D3hclTeids8/\nIS6G7in1TqptXGwMA7vqEsoi4aq4rJiPNn/Ew189TMf6HflTvz+RW5hLWrMTZl/Ui7jwF5HosHH/\nRv5n3v+wft96zmh4BhMGTqBRUiOvywobCn8RCTtvfvsmf/76z9RJqMPfLv4bA1oNIDZGw7anQuEv\nImHFOcdLK1/irOSzePqSp2mYqEuqnI7wP69JRKLKmr1r2JW3i592/KmCPwDa8xeRsDE3cy6vrH6F\nGIuJytMzg0nhLyJhYV/BPsZ9MY7DxYcZ0HoADRIbeF1SWFP4i0jIc87xzNJnyCvJY8qIKbSv397r\nksKexvxFJOS9sPIF3l3/Ltd1uU7BHyTa8xeRkPTq6leZv20+7eq3461v32Jo26Hcm3av12VFDIW/\niIScWVtm8UT6E7Su05qvdnxF/Rr1+V3v30XEhddChcJfREJKbmEujy58lDMbncmrw15l8c7F1Imv\nQ/3E+l6XFlEU/iISUsYvH8/+wv1MvGwi8THx9G3R1+uSIlJAn6HMrKGZfWJmG/y3x5x7ZWZnm9lX\nZrbazFaY2dWBrFNEItfsrbN5fe3rjOw8kjManuF1OREt0AG0+4HZzrmOwGz/46PlATc457oBQ4Cn\nzUyf30TkCKVlpTyy8BG6NerG2F5jvS4n4gUa/iOASf77k4Arjm7gnFvvnNvgv78dyAY007mIHOGb\n7G/YU7CHG8+8kRqxNbwuJ+IFGv5NnXM7APy3TY7X2Mx6AwnAxkqev8XM0s0sPScnJ8DSRCSczNoy\ni8TYxApn25LgO+EBXzP7FGhWwVO/P5UVmVlzYDIw2jn/dDtHcc5NBCaCbyavU3l9EQk/zjnMjGXZ\ny5ixaQb9UvpRM76m12VFhROGv3NuYGXPmdkuM2vunNvhD/fsStrVBT4AHnDOLTztakUkYhSWFjJy\nxkia1mzKspxlNE5qzNg0jfVXl0CHfaYDo/33RwPTjm5gZgnAFOBV59y7Aa5PRCLEtIxpbMrdxKKd\ni2heqzmvDHmF5rWbe11W1Aj0PP/HgXfMbAzwHXAVgJmlAbc5524CRgL9gUZm9nP/7/3cObcswHWL\nSJj5fphn26FtTFwxkbMan8Xj/R6nYVJDasXX8rq8qBJQ+Dvn9gCXVrA8HbjJf/814LVA1iMi4e/L\nbV/y4JcPUlBaQF5xHknxSYw7bxyt6rbyurSopG/4ikiVy8nL4c7P7iS1biqXNL2E2vG1uaLDFaTW\nS/W6tKil8BeRKvefTf+hpKyEpy5+SoEfInSJPBGpUs45pmVM4+zksxX8IUThLyJVIjsvm9KyUpbs\nWsLG3I1c0eGYCwCIhzTsIyJB9/GWjxn7+Vi6N+5OXEwc9WrUY1i7YV6XJeUo/EUkqHYc2sG4L8bR\nqUEnsg5lsbdgL2POHENSXJLXpUk5Cn8RCaqpGVMpKSvh7wP+TpOkJqzas4qujbp6XZYcReEvIkFT\nVFrE1Iyp9Gneh5a1WwLQs0lPj6uSiuiAr4gExZ78PYz6YBTbD2/nZ2f8zOty5AS05y8iASspK2Hs\nvLF8d+A7nh3wLBe1usjrkuQEFP4iEpDsvGw+2vwRi3cu5pELHlHwhwmFv4ictikbpvDgggcBuLDl\nhYxoP8LjiuRkKfxF5LSszFnJwwsfpmeTnrSp24bbe9yOmXldlpwkhb+InLK84jzGzR9H46TGPDPg\nGerVqOd1SXKKFP4ickqcczy44EEyD2bywqAXFPxhSqd6isgpmZs5l4+3fMxdPe+id/PeXpcjp0nh\nLyKnZGrGVBolNmJ0t9EnbiwhS+EvIidtb8Fe5mXN48ftf0xcjEaNw5nCX0RO2sxNMylxJQxvP9zr\nUiRACn8ROaH8knzfpCwbp9GtUTc6NujodUkSIH1uE5Hj+mTrJ9wz9x5qxdficPFhxvUe53VJEgQK\nfxGpVHFZMU8teYrUuqmc3+J8Dhcf5sftf+x1WRIECn8RqdTfl/ydzIOZulhbBFL4i8gxSspKeDL9\nSV5b+xqjOo+if0p/r0uSIFP4i8gxnl/2PK+tfY3rulzH2F5jdc2eCKTwF5EjrNu7jn+t+hfD2w/n\nvt73eV2OVBGd6ikiPygtK+UPX/2BOgl1+G3ab70uR6qQwl9EfjBt4zRW7l7Jfb3vo0FiA6/LkSqk\n8BcRwHe1zslrJtOlYReGtR3mdTlSxQIKfzNraGafmNkG/22luwpmVtfMtpnZs4GsU0SqxqKdi8jY\nn8G1Xa6/L1rwAAAGr0lEQVTVAd4oEOie//3AbOdcR2C2/3FlHgE+D3B9IlJFpmyYQt2EugxpO8Tr\nUqQaBBr+I4BJ/vuTgCsqamRm5wJNgVkBrk9EqsDh4sN89t1nDEkdQo3YGl6XI9Ug0PBv6pzbAeC/\nbXJ0AzOLAZ4Exga4LhGpIp9u/ZSC0gJduiGKnPA8fzP7FGhWwVO/P8l13A7MdM5lnmgc0cxuAW4B\naN269Um+vIgEasamGaTUTqFHcg+vS5FqcsLwd84NrOw5M9tlZs2dczvMrDmQXUGz84F+ZnY7UBtI\nMLNDzrljjg845yYCEwHS0tLcyXZCRE7fzsM7+XrH19za41Yd6I0igX7DdzowGnjcfzvt6AbOuWu/\nv29mPwfSKgp+EfHGjI0zcDgub3e516VINQp0zP9xYJCZbQAG+R9jZmlm9mKgxYlI1SosLeT1ta/T\nt0Vf2tRt43U5Uo0C2vN3zu0BLq1geTpwUwXLXwFeCWSdIhI8H2z6gD0Fexhz5hivS5Fqpm/4ikSx\neVnzaFm7Jb2a9fK6FKlmCn+RKFXmykjflU6vZr10oDcKKfxFotSGfRvILcyld7PeXpciHoi46/nn\nFuYy+sPRXpchEvIOFh8EIK1pmseViBciLvxjLIZ29dt5XYZIWGhbry3Nazf3ugzxQMSFf52EOvzt\n4r95XYaISEjTmL+ISBRS+IuIRCGFv4hIFFL4i4hEIYW/iEgUUviLiEQhhb+ISBRS+IuIRCFzLjQn\nzDKzHGBrAC/RGNgdpHK8Fil9iZR+gPoSqtQXaOOcSz5Ro5AN/0CZWbpzLiIuWhIpfYmUfoD6EqrU\nl5OnYR8RkSik8BcRiUKRHP4TvS4giCKlL5HSD1BfQpX6cpIidsxfREQqF8l7/iIiUomIC38zG2Jm\n68wsw8zu97qeU2VmW8xspZktM7N0/7KGZvaJmW3w3zbwus6KmNnLZpZtZqvKLauwdvP5h387rTCz\nc7yr/FiV9OUhM9vm3zbLzGxYuefG+fuyzswGe1N1xcyslZnNMbO1ZrbazO72Lw+rbXOcfoTddjGz\nRDP72syW+/vyB//ytma2yL9N3jazBP/yGv7HGf7nUwMuwjkXMT9ALLARaAckAMuBrl7XdYp92AI0\nPmrZX4D7/ffvB/7sdZ2V1N4fOAdYdaLagWHAh4ABfYBFXtd/En15CPhtBW27+t9rNYC2/vdgrNd9\nKFdfc+Ac//06wHp/zWG1bY7Tj7DbLv5/29r++/HAIv+/9TvAKP/yCcAv/fdvByb4748C3g60hkjb\n8+8NZDjnNjnnioC3gBEe1xQMI4BJ/vuTgCs8rKVSzrl5wN6jFldW+wjgVeezEKhvZiEzn2AlfanM\nCOAt51yhc24zkIHvvRgSnHM7nHPf+O8fBNYCLQmzbXOcflQmZLeL/9/2kP9hvP/HAQOAf/uXH71N\nvt9W/wYuNTMLpIZIC/+WQGa5x1kc/80Rihwwy8yWmNkt/mVNnXM7wPcfAGjiWXWnrrLaw3Vb3ekf\nCnm53PBb2PTFP1zQE9+eZthum6P6AWG4Xcws1syWAdnAJ/g+mex3zpX4m5Sv94e++J/PBRoFsv5I\nC/+K/hKG2+lMFzjnzgGGAneYWX+vC6oi4bitxgPtgbOBHcCT/uVh0Rczqw28B/zaOXfgeE0rWBYy\n/amgH2G5XZxzpc65s4EUfJ9IulTUzH8b9L5EWvhnAa3KPU4BtntUy2lxzm3332YDU/C9KXZ9/7Hb\nf5vtXYWnrLLaw25bOed2+f/DlgEv8N8hhJDvi5nF4wvM151z7/sXh922qagf4bxdAJxz+4G5+Mb8\n65tZnP+p8vX+0Bf/8/U4+WHJCkVa+C8GOvqPmCfgOzAy3eOaTpqZ1TKzOt/fBy4DVuHrw2h/s9HA\nNG8qPC2V1T4duMF/ZkkfIPf7IYhQddS49//Dt23A15dR/jMy2gIdga+ru77K+MeGXwLWOuf+Vu6p\nsNo2lfUjHLeLmSWbWX3//SRgIL5jGHOAK/3Njt4m32+rK4HPnP/o72nz+qh3sH/wnamwHt/42e+9\nrucUa2+H7+yE5cDq7+vHN7Y3G9jgv23oda2V1P8mvo/dxfj2VMZUVju+j7HP+bfTSiDN6/pPoi+T\n/bWu8P9nbF6u/e/9fVkHDPW6/qP6ciG+IYIVwDL/z7Bw2zbH6UfYbRfgLGCpv+ZVwIP+5e3w/YHK\nAN4FaviXJ/ofZ/ifbxdoDfqGr4hIFIq0YR8RETkJCn8RkSik8BcRiUIKfxGRKKTwFxGJQgp/EZEo\npPAXEYlCCn8RkSj0/wGb5Naa2pI2GQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x120d15240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ndim = 3\n",
    "\n",
    "with tf.Graph().as_default() as graph:\n",
    "    # Set up quadratic loss\n",
    "    x = tf.placeholder(tf.float32, (None, ndim))\n",
    "    mean = tf.Variable(np.random.normal(0, 1, ndim).astype(np.float32), name='mean')\n",
    "    loss = tf.reduce_sum((x - mean) ** 2)\n",
    "    mask = tf.placeholder(tf.float32, ndim)\n",
    "    \n",
    "    # Optimize the loss in two steps and apply a mask\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.001)\n",
    "    [(grad, var)] = optimizer.compute_gradients(loss)\n",
    "    # Multiply by the mask before applying gradients (e.g. you could have a dictionary of masks\n",
    "    # keyed by variable which you apply here)\n",
    "    train_op = optimizer.apply_gradients([(grad * mask, var)])\n",
    "    \n",
    "    init_op = tf.global_variables_initializer()\n",
    "    \n",
    "session = tf.Session(graph=graph)\n",
    "session.run(init_op)\n",
    "trace = []\n",
    "\n",
    "for i in range(ndim):\n",
    "    _mask = np.zeros(ndim)\n",
    "    _mask[i] = 1\n",
    "    for _ in range(100):\n",
    "        _, _mean = session.run([train_op, mean], {mask: _mask, x: np.random.normal(0, 1, (10, ndim))})\n",
    "        trace.append(_mean)\n",
    "        \n",
    "plt.plot(trace)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
	{
	"cells": [
	{
	"cell_type": "code",
	"execution_count": 1,
	"metadata": {
	"collapsed": true
	},
	"outputs": [],
	"source": [
	"import tensorflow as tf\n",
	"from matplotlib import pyplot as plt\n",
	"import numpy as np\n",
	"%matplotlib inline"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 2,
	"metadata": {
	"collapsed": false
	},
	"outputs": [
	{
	"data": {
	"text/plain": [
	"[<matplotlib.lines.Line2D at 0x1006f93c8>,\n",
	" <matplotlib.lines.Line2D at 0x120d87d68>,\n",
	" <matplotlib.lines.Line2D at 0x120d87eb8>]"
	]
	},
	"execution_count": 2,
	"metadata": {},
	"output_type": "execute_result"
	},
	{
	"data": {
	"image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VOXd//H3NxsJ+xbWAGEXEBENiCioCLLUgr9WkbrR\nilvVaqvlUVofH+vS2lqrrQsUl4q4W2WrqCiCiAgSZAeBsJmwJWxhyZ7cvz9mtAEStpnkzPJ5XVeu\nmTlzZ8735gyfnLnPmXObcw4REYkuMV4XICIi1U/hLyIShRT+IiJRSOEvIhKFFP4iIlFI4S8iEoUU\n/iIiUUjhLyIShRT+IiJRKM7rAirTuHFjl5qa6nUZIiJhZcmSJbudc8knahey4Z+amkp6errXZYiI\nhBUz23oy7YIy7GNmQ8xsnZllmNn9lbQZaWZrzGy1mb0RjPWKiMjpCXjP38xigeeAQUAWsNjMpjvn\n1pRr0xEYB1zgnNtnZk0CXa+IiJy+YOz59wYynHObnHNFwFvAiKPa3Aw855zbB+Ccyw7CekVE5DQF\nI/xbApnlHmf5l5XXCehkZl+a2UIzGxKE9YqIyGkKxgFfq2DZ0ZMExAEdgYuBFOALMzvTObf/iBcy\nuwW4BaB169ZBKE1ERCoSjD3/LKBVuccpwPYK2kxzzhU75zYD6/D9MTiCc26icy7NOZeWnHzCM5VE\nROQ0BSP8FwMdzaytmSUAo4DpR7WZClwCYGaN8Q0DbQrCukVE5DQEPOzjnCsxszuBj4FY4GXn3Goz\nexhId85N9z93mZmtAUqBsc65PYGuu0JFh2H+01Xy0hKAGnWgz+0QG7JfLRGJKkH5n+icmwnMPGrZ\ng+XuO+Ae/0/VKs6HeU9U+WrkVPgPATXqAGcM87YUEQFC+Bu+p61WY3ho/4nbSfUpKYIn2sP6DxX+\nIiFCF3aTqheXAO0HwLqPoKzM62pEBIW/VJfOw+BwNny3wOtKRASFv1SXLj+GpIaw4FmvKxERFP5S\nXRJqwnm3+sb9c9Z7XY1I1FP4S/VJGwMxcbB0steViEQ9hb9Un9rJ0GkILH8LSou9rkYkqin8pXr1\nvM534Hflu15XIhLVFP5SvToOhhY9YfbDvm9ji4gnFP5SvWJi4LLH4OAO3/CPiHhC4S/Vr01faHom\nfDPJ60pEopbCX6qfGZwzGnYsh9mP+C7/ICLVKvKu7SPhoee1sHU+fPFXiE2Ai+/zuiKRqKI9f/FG\nQi0Y+Sp0GQ5f/h0O7vS6IpGoovAXbw18CEryYeHzXlciElUU/uKtRu19e//pr0DhIa+rEYkaCn/x\nXt9fQWEufPw7cM7rakSigsJfvJeSBhf+xnfq57f/8boakaig8JfQMOB/oW4KLNG5/yLVQeEvoSEm\nFnqMgo2z4cAOr6sRiXgKfwkdZ1/jG/Of/zevKxGJeAp/CR2N2kOvm2Dxi5CV7nU1IhFN4S+h5VL/\n2P/b18Erl0Pm115XJBKRFP4SWhLrwajXfef8b/kC0l/2uiKRiKTwl9DT/Cy4fyt0HwkbZkFZqdcV\niUQchb+EpphY6DwE8vbAZ4/CoRyvKxKJKAp/CV3tL4UadX1n/7x9nT4BiASRwl9CV1J9uHcdDH8G\nMhfCF096XZFIxFD4S2hLqAk9r4ezroY5j8HaGV5XJBIRFP4S+sx8e/8tesKMu+Hwbq8rEgl7QQl/\nMxtiZuvMLMPM7j9OuyvNzJlZWjDWK1EkrgZcMR4KDmj4RyQIAg5/M4sFngOGAl2Bn5lZ1wra1QHu\nAhYFuk6JUk26QOehsPJdKC3xuhqRsBaMPf/eQIZzbpNzrgh4CxhRQbtHgL8ABUFYp0SrHqPgcA5s\n/MzrSkTCWjDCvyWQWe5xln/ZD8ysJ9DKOaeLtUtgOgyCWsnw1TNeVyIS1oIR/lbBsh+mYzKzGOAp\n4N4TvpDZLWaWbmbpOTn6Uo9UIC4B+t0Lm+dp718kAMEI/yygVbnHKcD2co/rAGcCc81sC9AHmF7R\nQV/n3ETnXJpzLi05OTkIpUlESrsRajeDRRO9rkQkbAUj/BcDHc2srZklAKOA6d8/6ZzLdc41ds6l\nOudSgYXAcOecrtkrpyeuBnS/EjI+gby9vmXOQXGB5gAWOUkBh79zrgS4E/gYWAu845xbbWYPm9nw\nQF9fpEJnjYSyElj2uu/Sz39oAI81hZm/9boykbBgLkT3lNLS0lx6uj4cSCWcg5eH+C77AHDebZDz\nLWxdAL9ZDbWbeFufiEfMbIlz7oTfpYqrjmJEgs4MrvoXvDgI2l8CQ/8MuzfAs2kw5VaolwL1Wvlm\nBqvZ0OtqRUKOwl/CV90WcNdSiI33PW7cEfr91jcN5LZvoGA/LH8LbvxInwREjqJr+0h4i0vwfQr4\n3qX/65sI5v6t8POZcGA7TL/Lu/pEQpTCXyJX6gVw3q1HnhUkIoDCXyJd1xG+s4LWfeh1JSIhReEv\nka1FT6jXWvMAiBxF4S+RzQw6DoItX0BpsdfViIQMhb9Evrb9oegQbF/mdSUiIUPhL5EvtZ/vdvPn\n3tYhEkIU/hL5ajWCpt1hzTQoK/W6GpGQoPCX6HDB3bBzBcx/yutKREKCwl+iQ/crodtPYM5jsOET\nr6sR8ZzCX6KDGQx/Bpp0g7eugZX/9roiEU8p/CV61KgNo6dDi3Ngxt1QkOt1RSKeUfhLdKnZEIY+\n7jv185vJUHQYDmnKUIk+Cn+JPi16QpsL4PO/wPi+8HwfXftHoo7CX6LTFeOhbnPI3w/5++CzR7yu\nSKRaKfwlOjVoA7d+Ab9eAeeOhqWv+f4IiEQJhb9Er7gESKwHPa+H0qL/XvytIBeylmgyeIlomslL\npEVPaNgevnoeEuvDV89C5iLoeBlc886Rk8WIRAjt+YuYwcX3Q24mvHO9L/g7DYENs3zTQYpEIO35\niwCcNdL3DeCNn0HhAegwEJ7oAKvfh5Rzva5OJOi05y/yvdg46HSZ71IQSfWh/QBYPQWK8ryuTCTo\nFP4ilenzS98E8FN/CSVFXlcjElQKf5HKtL8ELnsE1kyFyVfoD4BEFIW/yPH0/RUMfxa2fuk7C0hD\nQBIhFP4iJ3LO9b7TPmf/AZ5oD+kvw/K3fNcF2r3B6+pETou5EP0iS1pamktPT/e6DBGfAztg5bu+\nn50rfMvikqAkH0bP8M0TLBICzGyJcy7thO0U/iKnoCAXNs2F4gJY/ibkfAt1msFNn0GMPkiL9042\n/IPybjWzIWa2zswyzOz+Cp6/x8zWmNkKM5ttZm2CsV6RapdYD7qOgB5Xww1T4dIHYftSWDPF68pE\nTknA4W9mscBzwFCgK/AzM+t6VLOlQJpz7izg38BfAl2vSEg462poeibMfhgKDnhdjchJC8aef28g\nwzm3yTlXBLwFjCjfwDk3xzn3/WkSC4GUIKxXxHsxsTD4Mdif6Zsb4KvnoKzM66pETigY4d8SyCz3\nOMu/rDJjgA+DsF6R0NDuYrjxI9/Y/8e/810SAqA4X1cGlZAVjPCv6JKHFb7jzew6IA14opLnbzGz\ndDNLz8nR1HoSRlr1hhtnQf02sOQVyPwa/tjS9zPrf32nhYqEkGCEfxbQqtzjFGD70Y3MbCDwe2C4\nc66wohdyzk10zqU559KSk5ODUJpINYqJ8U0Ms+UL3wTxSfWh81BY8Ay8PERTRUpICUb4LwY6mllb\nM0sARgHTyzcws57AP/EFf3YQ1ikSmtJuhCZdIXsNnH8HXPkS/OxN33cDlr7mdXUiPwg4/J1zJcCd\nwMfAWuAd59xqM3vYzIb7mz0B1AbeNbNlZja9kpcTCW9JDeAXM+FHT0Kf233LOg+Fpt1h3UxvaxMp\nJyjX83fOzQRmHrXswXL3BwZjPSJhIakB9LrpyGWdh8IXf4XDu6FWY2/qEilHX0kUqQ5dh4Mrg5cH\nw44VXlcjovAXqRbNuvvmAy7Oh9d+AlNug90ZXlclUUzhL1JdOg2G66f6vg+w6j346D6vK5IopvAX\nqU7JneC2+TDgAcj4FLbM97oiiVKawF3EC71v8V0KYu7jkJIGjTrCvCfgjB/5Dhh3GuwbKhKpIgp/\nES/EJ8F5t/kmiNnyhW9ZbA3fbGEA30yC276ExLre1SgRTeEv4pVeY2Ddh9D9Sigp9O3t71zpuz/9\nTvj0/+Dyp7yuUiKUwl/EK4n14KZPjlyW3Nl3u3MlLJoA59wALXpWf20S8XTAVyQUXTIOajeB927W\nPAFSJRT+IqEosR5c+S/YtxneucE3FCQSRAp/kVCVegH8+B+waQ5M/glkLYH938GO5b65hEUCoDF/\nkVDW81qIjfddIvrFAf9dntoPRs+AFe/ArpXQfSQ0P8u7OiXsKPxFQt1ZI31hvy0d8vf7DgZ//U/4\n9y9g9RSwGFj+Nlx8HxzK8X1iqN8aGqR6XbmEMIW/SDio2xzq/th3v6TQd3no1VOgxzW+M4L+NQQ+\nuNf3/OcA5jt1tPfN0EEX1ZVjKfxFwk1cDbhpNpSVQD3/dNl9bof8fdB/LORmwebPYdmb8PpIuGEa\ntO3nbc0ScsyF6ATTaWlpLj093esyRMJXQS68OBAKD8KvvoGEml5XJNXAzJY459JO1E5n+4hEqsR6\nvrOFDu6ATx/SJaTlCAp/kUjW5nzoNNR3gHh8X8hc7HVFEiIU/iKR7qcvwo0f+w4av32dvjEsgMJf\nJPLVqA2t+8BPX4ZDO2HW72HjHNg8z+vKxEM620ckWqScC2ljIP0l+OZVsFi491vfNYQk6mjPXySa\nDPsr3PwZjHgOXCmset/risQj2vMXiSYxMdDyXN/PogmwaDzs2+J1VXK0+q3g/DuqdBUKf5Fo1ed2\n+GgcLHvD60rkaC3OVviLSBU5+xrfj0QljfmLiEQhhb+ISBRS+IuIRCGFv4hIFFL4i4hEoaCc7WNm\nQ4C/A7HAi865x496vgbwKnAusAe42jm3JRjrltB3uLCEWWt2UlRS5nUpImGhYa0aDOratErXEXD4\nm1ks8BwwCMgCFpvZdOfcmnLNxgD7nHMdzGwU8Gfg6kDXLadvw66D7D1cxOxvs5m+bDv1kuL57eDO\nQX3Dbd+fz4erdjJpwRa+25sXtNcViXRnt6of+uEP9AYynHObAMzsLWAEUD78RwAP+e//G3jWzMyF\n6kwyEe67PXn8ZPwCDhaUADCkWzM25hzi5lfTSWmQRI+U+jx7TU+KSx0Jcac3Mjjn22zGTFpMmYNu\nLery2pjzaJdcK5jdEIlY8bFVPyIfjPBvCWSWe5wFnFdZG+dciZnlAo2A3UFYv5yC0jLHr95aigFP\nXtWD9k1qc3ar+hSVlPHMZxuYsy6bD1buIOv5fPKLSph+54Ws3p7Lhl2HGHZWc+omxp/Uep6dk0HL\nBklM+kVv2iXXrtpOicgpC0b4WwXLjt6jP5k2mNktwC0ArVu3DrwyOcbEeZtYnrmfv486mxFnt/xh\neUJcDPde1pmb+7fj/D/OZnnmfgCu/udXrNiWi3Pw4PTV9O/YmPo1E1i8ZS8XdUrmgR91PebTwZcZ\nu1mydR8PXt5VwS9SiWkZ02iU1IgLW154xHLnHEVlRdSIrVGl6w9G+GcBrco9TgG2V9Imy8zigHrA\n3qNfyDk3EZgIvjl8g1BbVCorc2zPzadJnUQS4mJ4+tP1vL7oO4pLy9ifV8zALk0Z3qNFhb9bNzGe\nccO6sGHXQXYfKmL2t7v4Rd+2DOvejA9X7WTq0m3szy+mb/tGvPrVVnYdKOCCDo3J2pfP7oOFLM/a\nT+befNo0qslVaSnV3HOR0Ha4+DDLspexZNcSXlj5AnExcUwcNJFezXoBkF+Sz4NfPkhhaSFPX/I0\nMVZ1wz8BT+DuD/P1wKXANmAxcI1zbnW5NncA3Z1zt/kP+P7EOTfyeK8bLRO45xeVkpQQe9w2G3MO\nMWnBFlZvP0DrhjWJjTGu79OGHq3qH9FuxvLtfLc3j9cXbmV7bgGXntGEq3u14pbJS7ioUzJN69ag\nc7O6XN+nzUmN5ZeUllFc6o6or6C4lLyiUhrWSuC5ORn8ddY6nPN9cqiVEEtaakOa1U3knkGdaFAr\n4fT+UUQiUMa+DG7+5GZ25/tGu/u17EfWoSxyC3MZdcYo6ibUZdaWWSzNXsqvz/01v+j2C8wqGjQ5\nvpOdwD3g8PevbBjwNL5TPV92zj1mZg8D6c656WaWCEwGeuLb4x/1/QHiykRD+L+TnskDU1fxyi96\n0bd94x+WO+f4fH0On6zZBcBbizOJjzXObFGP7/bmkVdUSklZGf8Y1ZPLujXjYEExK7JyufbFRQB0\nb1mPs1vVZ/LCrZhBxya1+c+v+p32wdvjySsqITe/mGZ1E0/rjSoSDbLzshk5YyQxFsPDFzxMpwad\nSE5KJmN/BqP+M4qisqIf2j7R/wmGtB1y2uuq1vCvCpEe/tkHCxj45OccKCghpUESvx7YiQP5xVze\nozmPz/yW95duIzE+hoLiMn5yTkt+P6wLjWr7xgB3HypkzCuLWbktlws7JrMgYzclZY6W9ZN4//a+\nNKnja/f83I3kF5UyqncrUhrU9LK7IlGppKyE9ze8z2trX2Pn4Z28MewNOjTocESbxTsXU+bK2Jy7\nmaS4JEZ0GBHQOhX+ISo3r5ivNu1m8sKtpG/Zx6NXnMmjH6wlN78YgPhYo6TMcfelHbn94g6UlJVR\nM+HYQzN5RSX85aN1fLJmF+e1bUhBSSnXnteGCzo0PqatiFS/g0UHuXnWzazes5qODTrym3N+Q7+U\nflW+XoV/iEnfspe3F2fy2bfZ7Dns+4j355925+perSksKWVj9mF2HsjnH7MzuPvSjlxyhuZVFQlH\nzjn+b8H/sWTXErYf2s6f+v2JwamDq21Y9GTDX5O5VIMX5m3isZlrqZcUT6/UBozum0psjHF+u0YA\n1IiLpWuLunRtUZcBZ1Ttt/pEpGqt3L2SKRlT6NigI4/3f5zBqYO9LqlCCv8qsH7XQf69JIt6SfFk\nHyhg8sKtDD2zGU+O7FHhEI6IRI4PN39IQkwCk4ZMok5CHa/LqZSSqAr8aeZa5qzLAaBGXAyXdVXw\ni0SD0rJSPtryEf1S+oV08IPCP+gy9+Yxd30Odw3owO2XdCA+NobYGJ0CKRJunHOnPE6/eNdidufv\nZmjboVVUVfDoev5BtDIrlzGTFmPAqN6tSYyPVfCLhKEVOSvo80YfHlrwEAeKDpz07324+UNqxtXk\nopSLqrC64FD4B0leUQm3v7GEA/kljL/uXFrUT/K6JBE5TS+tfAmHY1rGNH72n5/98K3cyjjneHnV\ny3y4+UMubX0piXGJ1VTp6VP4V6K4tOyUJh8ZP3cjWfvyeeaangzu1qwKKxORqpR5IJM5mXO4vuv1\nvDj4RbIOZfH62teP+ztzM+fy1JKn6N64O788+5fVVGlgFP4VOFxYwuX/mM+tk0/uewZFJWW8+fV3\nDOzSlF6pDau4OhGpSh9v/RiH46pOV3Fu03O5KOUi3lv/HgUlBRW2Lyot4ulvnia1bioTBk2gVZ1W\nFbYLNVER/jtzCxj3/krStxxzIdFjZGQf5LbXlrBu10HmrMth7Y7jj/eVlTneWLSV3YeKuOY8XYZa\nJNRtyt3EhOUTKg3z2Vtn071xd5rV8n2Cv7bLtewr3MeIqSNYsH3BEW2dc/xx0R/ZlLuJsb3GEh9z\ncvNdhIKIDv9Zq3eyZvsBLn/mC978+jsenLaa432jOedgIVdN+Iql3+1n7ODOJMbH8M/PNwK+0zdv\nf30JxaX/HQpyznH328t4aMYazmhWh/4dk6u8TyJy6uZlzeOB+Q+wJXcLYz4ew3PLnuOO2XcwY+MM\n7pl7D0t2LeG2T2/jpZUvsWrPKga0HvDD757X/Dyeu/Q5kuKSuO2T2xi/fDz7C/Yzfvl47pt3H+9t\neI+but9E/5T+Hvbw1EXsqZ4Z2Qe5ZfISYgxizLi5X1te+GIzc9flcFGnZBZu3kOHJrVpUsd3YKaw\npJR7313O4aJSZt51IR2a1OFwYQnPz91I03qJvPDFJsoctKj3LQ9c3hWASQu2MGP5dn41oAN3XNJB\nZ/aIhJil2Ut5Y+0bzMmcQ2FpIV9u/5KCkgLu6nkXE5ZP4OudXwMwJ3MOJWUlfLntSxomNjzmVM3+\nKf1Ja5rGowsf5fllzzNh+QTKXBmGcUPXG7ir511edC8gERv+s/yXQ06Ii+HGC9ry64Gd+GTNLn77\n7nKS69Tg250H6Z3akLdv7QPAHa9/w7z1OTz+k+50aOL7csY9gzqxdscB/vn5JpLiYxnUtSkvf7mZ\nTs3qUFhSxl9nreeiTsncM6iTLmcsEkKmbJjCtI3T2Fewjx2Hd9CtUTf2F+5nU+4m7j7nbm7qfhOD\n2gxi/rb5ZOzP4L0N7/Fw34dJa5ZGs1rNKhy+qRlfk8cufIwhbYcwL2seP2r3I85sdCbxseEz1FNe\nxF3YraS0jDnrcnj60/XExhhv3NyHWgmxmBkbcw5x1YSvaFAznnNaN+DdJVk8MqIbhSVlPPrBWh74\nURdu6tfuiNcrLXO8/00WdRLjSUttQP+/zCGvqBSAuBjj49/0p72mKhQJGXnFeQx5bwj7CvcB8NeL\n/srg1MHM3zafyWsm8/QlT5MU999TsQtKCvh659dc2PLCKp05q7pE7YXd9uUVc/Orvj8aYwd3pnaN\n/3axfXJtFtw/gITYGMqcY332If53mm/CsfPbNeLGC9oe83qxMcZVaf89ev+H4d1Yu+MgF3Ro9MNr\nioj3pmZMZfHOxdSIrcG+wn3c1uM2DhUdYlCbQQBc2PLCY+bLBUiMSwy78fpgiLg9/6KSMtbtPMjB\ngmLOadOAxPjKp0gsKS1j7rocypyjf6fk47YVkdA1f9t87ph9B2XOd0LG1Z2v5oE+D3hclTeids8/\nIS6G7in1TqptXGwMA7vqEsoi4aq4rJiPNn/Ew189TMf6HflTvz+RW5hLWrMTZl/Ui7jwF5HosHH/\nRv5n3v+wft96zmh4BhMGTqBRUiOvywobCn8RCTtvfvsmf/76z9RJqMPfLv4bA1oNIDZGw7anQuEv\nImHFOcdLK1/irOSzePqSp2mYqEuqnI7wP69JRKLKmr1r2JW3i592/KmCPwDa8xeRsDE3cy6vrH6F\nGIuJytMzg0nhLyJhYV/BPsZ9MY7DxYcZ0HoADRIbeF1SWFP4i0jIc87xzNJnyCvJY8qIKbSv397r\nksKexvxFJOS9sPIF3l3/Ltd1uU7BHyTa8xeRkPTq6leZv20+7eq3461v32Jo26Hcm3av12VFDIW/\niIScWVtm8UT6E7Su05qvdnxF/Rr1+V3v30XEhddChcJfREJKbmEujy58lDMbncmrw15l8c7F1Imv\nQ/3E+l6XFlEU/iISUsYvH8/+wv1MvGwi8THx9G3R1+uSIlJAn6HMrKGZfWJmG/y3x5x7ZWZnm9lX\nZrbazFaY2dWBrFNEItfsrbN5fe3rjOw8kjManuF1OREt0AG0+4HZzrmOwGz/46PlATc457oBQ4Cn\nzUyf30TkCKVlpTyy8BG6NerG2F5jvS4n4gUa/iOASf77k4Arjm7gnFvvnNvgv78dyAY007mIHOGb\n7G/YU7CHG8+8kRqxNbwuJ+IFGv5NnXM7APy3TY7X2Mx6AwnAxkqev8XM0s0sPScnJ8DSRCSczNoy\ni8TYxApn25LgO+EBXzP7FGhWwVO/P5UVmVlzYDIw2jn/dDtHcc5NBCaCbyavU3l9EQk/zjnMjGXZ\ny5ixaQb9UvpRM76m12VFhROGv3NuYGXPmdkuM2vunNvhD/fsStrVBT4AHnDOLTztakUkYhSWFjJy\nxkia1mzKspxlNE5qzNg0jfVXl0CHfaYDo/33RwPTjm5gZgnAFOBV59y7Aa5PRCLEtIxpbMrdxKKd\ni2heqzmvDHmF5rWbe11W1Aj0PP/HgXfMbAzwHXAVgJmlAbc5524CRgL9gUZm9nP/7/3cObcswHWL\nSJj5fphn26FtTFwxkbMan8Xj/R6nYVJDasXX8rq8qBJQ+Dvn9gCXVrA8HbjJf/814LVA1iMi4e/L\nbV/y4JcPUlBaQF5xHknxSYw7bxyt6rbyurSopG/4ikiVy8nL4c7P7iS1biqXNL2E2vG1uaLDFaTW\nS/W6tKil8BeRKvefTf+hpKyEpy5+SoEfInSJPBGpUs45pmVM4+zksxX8IUThLyJVIjsvm9KyUpbs\nWsLG3I1c0eGYCwCIhzTsIyJB9/GWjxn7+Vi6N+5OXEwc9WrUY1i7YV6XJeUo/EUkqHYc2sG4L8bR\nqUEnsg5lsbdgL2POHENSXJLXpUk5Cn8RCaqpGVMpKSvh7wP+TpOkJqzas4qujbp6XZYcReEvIkFT\nVFrE1Iyp9Gneh5a1WwLQs0lPj6uSiuiAr4gExZ78PYz6YBTbD2/nZ2f8zOty5AS05y8iASspK2Hs\nvLF8d+A7nh3wLBe1usjrkuQEFP4iEpDsvGw+2vwRi3cu5pELHlHwhwmFv4ictikbpvDgggcBuLDl\nhYxoP8LjiuRkKfxF5LSszFnJwwsfpmeTnrSp24bbe9yOmXldlpwkhb+InLK84jzGzR9H46TGPDPg\nGerVqOd1SXKKFP4ickqcczy44EEyD2bywqAXFPxhSqd6isgpmZs5l4+3fMxdPe+id/PeXpcjp0nh\nLyKnZGrGVBolNmJ0t9EnbiwhS+EvIidtb8Fe5mXN48ftf0xcjEaNw5nCX0RO2sxNMylxJQxvP9zr\nUiRACn8ROaH8knzfpCwbp9GtUTc6NujodUkSIH1uE5Hj+mTrJ9wz9x5qxdficPFhxvUe53VJEgQK\nfxGpVHFZMU8teYrUuqmc3+J8Dhcf5sftf+x1WRIECn8RqdTfl/ydzIOZulhbBFL4i8gxSspKeDL9\nSV5b+xqjOo+if0p/r0uSIFP4i8gxnl/2PK+tfY3rulzH2F5jdc2eCKTwF5EjrNu7jn+t+hfD2w/n\nvt73eV2OVBGd6ikiPygtK+UPX/2BOgl1+G3ab70uR6qQwl9EfjBt4zRW7l7Jfb3vo0FiA6/LkSqk\n8BcRwHe1zslrJtOlYReGtR3mdTlSxQIKfzNraGafmNkG/22luwpmVtfMtpnZs4GsU0SqxqKdi8jY\nn8G1Xa6/L1rwAAAGr0lEQVTVAd4oEOie//3AbOdcR2C2/3FlHgE+D3B9IlJFpmyYQt2EugxpO8Tr\nUqQaBBr+I4BJ/vuTgCsqamRm5wJNgVkBrk9EqsDh4sN89t1nDEkdQo3YGl6XI9Ug0PBv6pzbAeC/\nbXJ0AzOLAZ4Exga4LhGpIp9u/ZSC0gJduiGKnPA8fzP7FGhWwVO/P8l13A7MdM5lnmgc0cxuAW4B\naN269Um+vIgEasamGaTUTqFHcg+vS5FqcsLwd84NrOw5M9tlZs2dczvMrDmQXUGz84F+ZnY7UBtI\nMLNDzrljjg845yYCEwHS0tLcyXZCRE7fzsM7+XrH19za41Yd6I0igX7DdzowGnjcfzvt6AbOuWu/\nv29mPwfSKgp+EfHGjI0zcDgub3e516VINQp0zP9xYJCZbQAG+R9jZmlm9mKgxYlI1SosLeT1ta/T\nt0Vf2tRt43U5Uo0C2vN3zu0BLq1geTpwUwXLXwFeCWSdIhI8H2z6gD0Fexhz5hivS5Fqpm/4ikSx\neVnzaFm7Jb2a9fK6FKlmCn+RKFXmykjflU6vZr10oDcKKfxFotSGfRvILcyld7PeXpciHoi46/nn\nFuYy+sPRXpchEvIOFh8EIK1pmseViBciLvxjLIZ29dt5XYZIWGhbry3Nazf3ugzxQMSFf52EOvzt\n4r95XYaISEjTmL+ISBRS+IuIRCGFv4hIFFL4i4hEIYW/iEgUUviLiEQhhb+ISBRS+IuIRCFzLjQn\nzDKzHGBrAC/RGNgdpHK8Fil9iZR+gPoSqtQXaOOcSz5Ro5AN/0CZWbpzLiIuWhIpfYmUfoD6EqrU\nl5OnYR8RkSik8BcRiUKRHP4TvS4giCKlL5HSD1BfQpX6cpIidsxfREQqF8l7/iIiUomIC38zG2Jm\n68wsw8zu97qeU2VmW8xspZktM7N0/7KGZvaJmW3w3zbwus6KmNnLZpZtZqvKLauwdvP5h387rTCz\nc7yr/FiV9OUhM9vm3zbLzGxYuefG+fuyzswGe1N1xcyslZnNMbO1ZrbazO72Lw+rbXOcfoTddjGz\nRDP72syW+/vyB//ytma2yL9N3jazBP/yGv7HGf7nUwMuwjkXMT9ALLARaAckAMuBrl7XdYp92AI0\nPmrZX4D7/ffvB/7sdZ2V1N4fOAdYdaLagWHAh4ABfYBFXtd/En15CPhtBW27+t9rNYC2/vdgrNd9\nKFdfc+Ac//06wHp/zWG1bY7Tj7DbLv5/29r++/HAIv+/9TvAKP/yCcAv/fdvByb4748C3g60hkjb\n8+8NZDjnNjnnioC3gBEe1xQMI4BJ/vuTgCs8rKVSzrl5wN6jFldW+wjgVeezEKhvZiEzn2AlfanM\nCOAt51yhc24zkIHvvRgSnHM7nHPf+O8fBNYCLQmzbXOcflQmZLeL/9/2kP9hvP/HAQOAf/uXH71N\nvt9W/wYuNTMLpIZIC/+WQGa5x1kc/80Rihwwy8yWmNkt/mVNnXM7wPcfAGjiWXWnrrLaw3Vb3ekf\nCnm53PBb2PTFP1zQE9+eZthum6P6AWG4Xcws1syWAdnAJ/g+mex3zpX4m5Sv94e++J/PBRoFsv5I\nC/+K/hKG2+lMFzjnzgGGAneYWX+vC6oi4bitxgPtgbOBHcCT/uVh0Rczqw28B/zaOXfgeE0rWBYy\n/amgH2G5XZxzpc65s4EUfJ9IulTUzH8b9L5EWvhnAa3KPU4BtntUy2lxzm3332YDU/C9KXZ9/7Hb\nf5vtXYWnrLLaw25bOed2+f/DlgEv8N8hhJDvi5nF4wvM151z7/sXh922qagf4bxdAJxz+4G5+Mb8\n65tZnP+p8vX+0Bf/8/U4+WHJCkVa+C8GOvqPmCfgOzAy3eOaTpqZ1TKzOt/fBy4DVuHrw2h/s9HA\nNG8qPC2V1T4duMF/ZkkfIPf7IYhQddS49//Dt23A15dR/jMy2gIdga+ru77K+MeGXwLWOuf+Vu6p\nsNo2lfUjHLeLmSWbWX3//SRgIL5jGHOAK/3Njt4m32+rK4HPnP/o72nz+qh3sH/wnamwHt/42e+9\nrucUa2+H7+yE5cDq7+vHN7Y3G9jgv23oda2V1P8mvo/dxfj2VMZUVju+j7HP+bfTSiDN6/pPoi+T\n/bWu8P9nbF6u/e/9fVkHDPW6/qP6ciG+IYIVwDL/z7Bw2zbH6UfYbRfgLGCpv+ZVwIP+5e3w/YHK\nAN4FaviXJ/ofZ/ifbxdoDfqGr4hIFIq0YR8RETkJCn8RkSik8BcRiUIKfxGRKKTwFxGJQgp/EZEo\npPAXEYlCCn8RkSj0/wGb5Naa2pI2GQAAAABJRU5ErkJggg==\n",
	"text/plain": [
	"<matplotlib.figure.Figure at 0x120d15240>"
	]
	},
	"metadata": {},
	"output_type": "display_data"
	}
	],
	"source": [
	"ndim = 3\n",
	"\n",
	"with tf.Graph().as_default() as graph:\n",
	" # Set up quadratic loss\n",
	" x = tf.placeholder(tf.float32, (None, ndim))\n",
	" mean = tf.Variable(np.random.normal(0, 1, ndim).astype(np.float32), name='mean')\n",
	" loss = tf.reduce_sum((x - mean) ** 2)\n",
	" mask = tf.placeholder(tf.float32, ndim)\n",
	" \n",
	" # Optimize the loss in two steps and apply a mask\n",
	" optimizer = tf.train.GradientDescentOptimizer(0.001)\n",
	" [(grad, var)] = optimizer.compute_gradients(loss)\n",
	" # Multiply by the mask before applying gradients (e.g. you could have a dictionary of masks\n",
	" # keyed by variable which you apply here)\n",
	" train_op = optimizer.apply_gradients([(grad * mask, var)])\n",
	" \n",
	" init_op = tf.global_variables_initializer()\n",
	" \n",
	"session = tf.Session(graph=graph)\n",
	"session.run(init_op)\n",
	"trace = []\n",
	"\n",
	"for i in range(ndim):\n",
	" _mask = np.zeros(ndim)\n",
	" _mask[i] = 1\n",
	" for _ in range(100):\n",
	" _, _mean = session.run([train_op, mean], {mask: _mask, x: np.random.normal(0, 1, (10, ndim))})\n",
	" trace.append(_mean)\n",
	" \n",
	"plt.plot(trace)"
	]
	}
	],
	"metadata": {
	"kernelspec": {
	"display_name": "Python 3",
	"language": "python",
	"name": "python3"
	},
	"language_info": {
	"codemirror_mode": {
	"name": "ipython",
	"version": 3
	},
	"file_extension": ".py",
	"mimetype": "text/x-python",
	"name": "python",
	"nbconvert_exporter": "python",
	"pygments_lexer": "ipython3",
	"version": "3.5.2"
	}
	},
	"nbformat": 4,
	"nbformat_minor": 2
	}