Trying to optimize hidden layers in a deep convolutional neural network.

Hidden layer optimization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cellView": "both",
    "colab_type": "code",
    "collapsed": false,
    "id": "i9hkSm1IOZNR"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data: 1505280 = 10x3x224x224\n",
      "conv1_1: 32112640 = 10x64x224x224\n",
      "conv1_2: 32112640 = 10x64x224x224\n",
      "pool1: 8028160 = 10x64x112x112\n",
      "conv2_1: 16056320 = 10x128x112x112\n",
      "conv2_2: 16056320 = 10x128x112x112\n",
      "pool2: 4014080 = 10x128x56x56\n",
      "conv3_1: 8028160 = 10x256x56x56\n",
      "conv3_2: 8028160 = 10x256x56x56\n",
      "conv3_3: 8028160 = 10x256x56x56\n",
      "conv3_4: 8028160 = 10x256x56x56\n",
      "pool3: 2007040 = 10x256x28x28\n",
      "conv4_1: 4014080 = 10x512x28x28\n",
      "conv4_2: 4014080 = 10x512x28x28\n",
      "conv4_3: 4014080 = 10x512x28x28\n",
      "conv4_4: 4014080 = 10x512x28x28\n",
      "pool4: 1003520 = 10x512x14x14\n",
      "conv5_1: 1003520 = 10x512x14x14\n",
      "conv5_2: 1003520 = 10x512x14x14\n",
      "conv5_3: 1003520 = 10x512x14x14\n",
      "conv5_4: 1003520 = 10x512x14x14\n",
      "pool5: 250880 = 10x512x7x7\n",
      "fc6: 40960 = 10x4096x1x1\n",
      "fc7: 40960 = 10x4096x1x1\n",
      "relu7: 40960 = 10x4096x1x1\n",
      "drop7: 40960 = 10x4096x1x1\n",
      "fc8: 10000 = 10x1000x1x1\n",
      "prob: 10000 = 10x1000x1x1\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from google.protobuf import text_format\n",
    "import caffe\n",
    "\n",
    "# load network\n",
    "# the prototxt has force_backward: true, and fc7 is separated into multiple blobs\n",
    "model_name = 'vgg_ilsvrc_19'\n",
    "model_path = '../caffe/models/' + model_name + '/'\n",
    "net_fn = model_path + 'deploy-expanded.prototxt'\n",
    "param_fn = model_path + 'net.caffemodel'\n",
    "net = caffe.Classifier(net_fn, param_fn)\n",
    "\n",
    "# print blob names and sizes\n",
    "for end in net.blobs.keys():\n",
    "    cur = net.blobs[end]\n",
    "    print end + ': {} = {}x{}x{}x{}'.format(cur.count, cur.num, cur.channels, cur.width, cur.height)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def print_state(net):\n",
    "    for name in net.blobs.keys():\n",
    "        if name in ['fc6', 'fc7', 'relu7', 'drop7', 'fc8', 'prob']:\n",
    "            print name, 'data:', net.blobs[name].data.flat[0:2], 'diff:', net.blobs[name].diff.flat[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n",
      "in: [ 1.  0.  1.  0.  0.  0.  0.  0.  1.  0.]\n",
      "out: [ 0.99  0.    0.    0.    0.    0.    0.    0.    0.    0.  ]\n"
     ]
    }
   ],
   "source": [
    "def optimize(net,\n",
    "             hot=0,\n",
    "             step_size=.01,\n",
    "             iter_n=100,\n",
    "             mu=.9,\n",
    "             basename='fc7',\n",
    "             start='relu7',\n",
    "             end='prob'):\n",
    "    base = net.blobs[basename]\n",
    "    first = net.blobs[start]\n",
    "    last = net.blobs[end]\n",
    "    base.data[0] = np.random.normal(.5, .1, base.data[0].shape)\n",
    "    base.diff[0] = 0.\n",
    "    velocity = np.zeros_like(base.data[0])\n",
    "    velocity_previous = np.zeros_like(base.data[0])\n",
    "    for i in range(iter_n):\n",
    "        net.forward(start=start, end=end)\n",
    "        target = np.zeros_like(last.data[0])\n",
    "        target.flat[hot] = 1.\n",
    "        error = target - last.data[0]\n",
    "        last.diff[0] = error\n",
    "        net.backward(start=end, end=start)\n",
    "        grad = base.diff[0]\n",
    "        learning_rate = (step_size / np.abs(grad).mean())\n",
    "        velocity_previous = velocity\n",
    "        velocity = mu * velocity + learning_rate * grad\n",
    "        base.data[0] += -mu * velocity_previous + (1 + mu) * velocity\n",
    "        base.data[0] = np.clip(base.data[0], 0, +1)\n",
    "    return base.data[0]\n",
    "np.set_printoptions(suppress=True)\n",
    "np.set_printoptions(precision=2)\n",
    "for i in range(10):\n",
    "    result = optimize(net, hot=0)\n",
    "    print 'in:', result[0:10]\n",
    "    print 'out:', net.blobs['prob'].data[0,0:10]"
   ]
  }
 ],
 "metadata": {
  "colabVersion": "0.3.1",
  "default_view": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  },
  "views": {}
 },
 "nbformat": 4,
 "nbformat_minor": 0
}