Created
January 17, 2018 09:07
-
-
Save Z30G0D/cf1ba895ae41605438b266a4fed01125 to your computer and use it in GitHub Desktop.
Fourth assignment of Udacity Deep learning course
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"colab_type": "text", | |
"id": "4embtkV0pNxM" | |
}, | |
"source": [ | |
"Deep Learning\n", | |
"=============\n", | |
"\n", | |
"Assignment 4\n", | |
"------------\n", | |
"\n", | |
"Previously in `2_fullyconnected.ipynb` and `3_regularization.ipynb`, we trained fully connected networks to classify [notMNIST](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) characters.\n", | |
"\n", | |
"The goal of this assignment is make the neural network convolutional." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": { | |
"cellView": "both", | |
"colab": { | |
"autoexec": { | |
"startup": false, | |
"wait_interval": 0 | |
} | |
}, | |
"colab_type": "code", | |
"id": "tm2CQN_Cpwj0" | |
}, | |
"outputs": [], | |
"source": [ | |
"# These are all the modules we'll be using later. Make sure you can import them\n", | |
"# before proceeding further.\n", | |
"from __future__ import print_function\n", | |
"import numpy as np\n", | |
"import tensorflow as tf\n", | |
"from six.moves import cPickle as pickle\n", | |
"from six.moves import range" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": { | |
"cellView": "both", | |
"colab": { | |
"autoexec": { | |
"startup": false, | |
"wait_interval": 0 | |
}, | |
"output_extras": [ | |
{ | |
"item_id": 1 | |
} | |
] | |
}, | |
"colab_type": "code", | |
"executionInfo": { | |
"elapsed": 11948, | |
"status": "ok", | |
"timestamp": 1446658914837, | |
"user": { | |
"color": "", | |
"displayName": "", | |
"isAnonymous": false, | |
"isMe": true, | |
"permissionId": "", | |
"photoUrl": "", | |
"sessionId": "0", | |
"userId": "" | |
}, | |
"user_tz": 480 | |
}, | |
"id": "y3-cj1bpmuxc", | |
"outputId": "016b1a51-0290-4b08-efdb-8c95ffc3cd01" | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Training set (2000, 28, 28) (2000,)\n", | |
"Validation set (100, 28, 28) (100,)\n", | |
"Test set (100, 28, 28) (100,)\n" | |
] | |
} | |
], | |
"source": [ | |
"pickle_file = 'notMNIST.pickle'\n", | |
"\n", | |
"with open(pickle_file, 'rb') as f:\n", | |
" save = pickle.load(f, encoding='latin1')\n", | |
" train_dataset = save['train_dataset']\n", | |
" train_labels = save['train_labels']\n", | |
" valid_dataset = save['valid_dataset']\n", | |
" valid_labels = save['valid_labels']\n", | |
" test_dataset = save['test_dataset']\n", | |
" test_labels = save['test_labels']\n", | |
" del save # hint to help gc free up memory\n", | |
" print('Training set', train_dataset.shape, train_labels.shape)\n", | |
" print('Validation set', valid_dataset.shape, valid_labels.shape)\n", | |
" print('Test set', test_dataset.shape, test_labels.shape)" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"colab_type": "text", | |
"id": "L7aHrm6nGDMB" | |
}, | |
"source": [ | |
"Reformat into a TensorFlow-friendly shape:\n", | |
"- convolutions need the image data formatted as a cube (width by height by #channels)\n", | |
"- labels as float 1-hot encodings." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": { | |
"cellView": "both", | |
"colab": { | |
"autoexec": { | |
"startup": false, | |
"wait_interval": 0 | |
}, | |
"output_extras": [ | |
{ | |
"item_id": 1 | |
} | |
] | |
}, | |
"colab_type": "code", | |
"executionInfo": { | |
"elapsed": 11952, | |
"status": "ok", | |
"timestamp": 1446658914857, | |
"user": { | |
"color": "", | |
"displayName": "", | |
"isAnonymous": false, | |
"isMe": true, | |
"permissionId": "", | |
"photoUrl": "", | |
"sessionId": "0", | |
"userId": "" | |
}, | |
"user_tz": 480 | |
}, | |
"id": "IRSyYiIIGIzS", | |
"outputId": "650a208c-8359-4852-f4f5-8bf10e80ef6c" | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Training set (2000, 28, 28, 1) (2000, 10)\n", | |
"Validation set (100, 28, 28, 1) (100, 10)\n", | |
"Test set (100, 28, 28, 1) (100, 10)\n" | |
] | |
} | |
], | |
"source": [ | |
"image_size = 28\n", | |
"num_labels = 10\n", | |
"num_channels = 1 # grayscale\n", | |
"\n", | |
"import numpy as np\n", | |
"\n", | |
"def reformat(dataset, labels):\n", | |
" dataset = dataset.reshape((-1, image_size, image_size, num_channels)).astype(np.float32)\n", | |
" labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n", | |
" return dataset, labels\n", | |
"train_dataset, train_labels = reformat(train_dataset, train_labels)\n", | |
"valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n", | |
"test_dataset, test_labels = reformat(test_dataset, test_labels)\n", | |
"print('Training set', train_dataset.shape, train_labels.shape)\n", | |
"print('Validation set', valid_dataset.shape, valid_labels.shape)\n", | |
"print('Test set', test_dataset.shape, test_labels.shape)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": { | |
"cellView": "both", | |
"colab": { | |
"autoexec": { | |
"startup": false, | |
"wait_interval": 0 | |
} | |
}, | |
"colab_type": "code", | |
"id": "AgQDIREv02p1" | |
}, | |
"outputs": [], | |
"source": [ | |
"def accuracy(predictions, labels):\n", | |
" return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n", | |
" / predictions.shape[0])" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"colab_type": "text", | |
"id": "5rhgjmROXu2O" | |
}, | |
"source": [ | |
"Let's build a small network with two convolutional layers, followed by one fully connected layer. Convolutional networks are more expensive computationally, so we'll limit its depth and number of fully connected nodes. " | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"metadata": { | |
"cellView": "both", | |
"colab": { | |
"autoexec": { | |
"startup": false, | |
"wait_interval": 0 | |
} | |
}, | |
"colab_type": "code", | |
"id": "IZYv70SvvOan" | |
}, | |
"outputs": [], | |
"source": [ | |
"batch_size = 16\n", | |
"patch_size = 5\n", | |
"depth = 16\n", | |
"num_hidden = 64\n", | |
"\n", | |
"graph = tf.Graph()\n", | |
"\n", | |
"with graph.as_default():\n", | |
"\n", | |
" # Input data.\n", | |
" tf_train_dataset = tf.placeholder(\n", | |
" tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n", | |
" tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n", | |
" tf_valid_dataset = tf.constant(valid_dataset)\n", | |
" tf_test_dataset = tf.constant(test_dataset)\n", | |
" \n", | |
" # Variables.\n", | |
" global_step = tf.Variable(0, trainable=False)\n", | |
" layer1_weights = tf.Variable(tf.truncated_normal(\n", | |
" [patch_size, patch_size, num_channels, depth], stddev=0.1))\n", | |
" layer1_biases = tf.Variable(tf.zeros([depth]))\n", | |
" layer2_weights = tf.Variable(tf.truncated_normal(\n", | |
" [patch_size, patch_size, depth, depth], stddev=0.1))\n", | |
" layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n", | |
" layer3_weights = tf.Variable(tf.truncated_normal(\n", | |
" [image_size // 4 * image_size // 4 * depth, num_hidden], stddev=0.1))\n", | |
" layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n", | |
" layer4_weights = tf.Variable(tf.truncated_normal(\n", | |
" [num_hidden, num_labels], stddev=0.1))\n", | |
" layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n", | |
" \n", | |
" # Model.\n", | |
" def model(data):\n", | |
" conv = tf.nn.conv2d(data, layer1_weights, [1, 2, 2, 1], padding='SAME')\n", | |
" hidden = tf.nn.relu(conv + layer1_biases)\n", | |
" conv = tf.nn.conv2d(hidden, layer2_weights, [1, 2, 2, 1], padding='SAME')\n", | |
" hidden = tf.nn.relu(conv + layer2_biases)\n", | |
" shape = hidden.get_shape().as_list()\n", | |
" reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n", | |
" hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)\n", | |
" return tf.matmul(hidden, layer4_weights) + layer4_biases\n", | |
" \n", | |
" # Training computation.\n", | |
" logits = model(tf_train_dataset)\n", | |
" loss = tf.reduce_mean(\n", | |
" tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n", | |
" \n", | |
" # Optimizer.\n", | |
" optimizer = tf.train.GradientDescentOptimizer(0.05).minimize(loss)\n", | |
" \n", | |
" # Predictions for the training, validation, and test data.\n", | |
" train_prediction = tf.nn.softmax(logits)\n", | |
" valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n", | |
" test_prediction = tf.nn.softmax(model(tf_test_dataset))" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"metadata": { | |
"cellView": "both", | |
"colab": { | |
"autoexec": { | |
"startup": false, | |
"wait_interval": 0 | |
}, | |
"output_extras": [ | |
{ | |
"item_id": 37 | |
} | |
] | |
}, | |
"colab_type": "code", | |
"executionInfo": { | |
"elapsed": 63292, | |
"status": "ok", | |
"timestamp": 1446658966251, | |
"user": { | |
"color": "", | |
"displayName": "", | |
"isAnonymous": false, | |
"isMe": true, | |
"permissionId": "", | |
"photoUrl": "", | |
"sessionId": "0", | |
"userId": "" | |
}, | |
"user_tz": 480 | |
}, | |
"id": "noKFb2UovVFR", | |
"outputId": "28941338-2ef9-4088-8bd1-44295661e628" | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Initialized\n", | |
"Minibatch loss at step 0: 3.511478\n", | |
"Minibatch accuracy: 12.5%\n", | |
"Validation accuracy: 10.0%\n", | |
"Minibatch loss at step 50: 1.389899\n", | |
"Minibatch accuracy: 56.2%\n", | |
"Validation accuracy: 54.0%\n", | |
"Minibatch loss at step 100: 0.954439\n", | |
"Minibatch accuracy: 62.5%\n", | |
"Validation accuracy: 64.0%\n", | |
"Minibatch loss at step 150: 0.672420\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 68.0%\n", | |
"Minibatch loss at step 200: 0.591480\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 69.0%\n", | |
"Minibatch loss at step 250: 0.485368\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 68.0%\n", | |
"Minibatch loss at step 300: 1.236439\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 71.0%\n", | |
"Minibatch loss at step 350: 0.643501\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 400: 0.500857\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 450: 0.978600\n", | |
"Minibatch accuracy: 68.8%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 500: 0.280913\n", | |
"Minibatch accuracy: 100.0%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 550: 0.477067\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 600: 1.169504\n", | |
"Minibatch accuracy: 68.8%\n", | |
"Validation accuracy: 70.0%\n", | |
"Minibatch loss at step 650: 0.391893\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 71.0%\n", | |
"Minibatch loss at step 700: 0.617436\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 74.0%\n", | |
"Minibatch loss at step 750: 0.412628\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 75.0%\n", | |
"Minibatch loss at step 800: 0.696503\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 850: 0.206645\n", | |
"Minibatch accuracy: 100.0%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 900: 0.177372\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 950: 0.265233\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 75.0%\n", | |
"Minibatch loss at step 1000: 0.046155\n", | |
"Minibatch accuracy: 100.0%\n", | |
"Validation accuracy: 75.0%\n", | |
"Test accuracy: 80.0%\n" | |
] | |
} | |
], | |
"source": [ | |
"num_steps = 1001\n", | |
"\n", | |
"with tf.Session(graph=graph) as session:\n", | |
" tf.global_variables_initializer().run()\n", | |
" print('Initialized')\n", | |
" for step in range(num_steps):\n", | |
" offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n", | |
" batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n", | |
" batch_labels = train_labels[offset:(offset + batch_size), :]\n", | |
" feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n", | |
" _, l, predictions = session.run(\n", | |
" [optimizer, loss, train_prediction], feed_dict=feed_dict)\n", | |
" if (step % 50 == 0):\n", | |
" print('Minibatch loss at step %d: %f' % (step, l))\n", | |
" print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n", | |
" print('Validation accuracy: %.1f%%' % accuracy(\n", | |
" valid_prediction.eval(), valid_labels))\n", | |
" print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"colab_type": "text", | |
"id": "KedKkn4EutIK" | |
}, | |
"source": [ | |
"---\n", | |
"Problem 1\n", | |
"---------\n", | |
"\n", | |
"The convolutional model above uses convolutions with stride 2 to reduce the dimensionality. Replace the strides by a max pooling operation (`nn.max_pool()`) of stride 2 and kernel size 2.\n", | |
"\n", | |
"---" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Ok, we are asked to introduce a max pooling operation, this operation will take the highest reaction in the vicinity of the pixel and implement it in the new feature map, hence subsampling our data." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"batch_size = 16\n", | |
"patch_size = 5\n", | |
"depth = 16\n", | |
"num_hidden = 64\n", | |
"\n", | |
"graph = tf.Graph()\n", | |
"\n", | |
"with graph.as_default():\n", | |
"\n", | |
" # Input data.\n", | |
" tf_train_dataset = tf.placeholder(\n", | |
" tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n", | |
" tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n", | |
" tf_valid_dataset = tf.constant(valid_dataset)\n", | |
" tf_test_dataset = tf.constant(test_dataset)\n", | |
" \n", | |
" # Variables.\n", | |
" layer1_weights = tf.Variable(tf.truncated_normal(\n", | |
" [patch_size, patch_size, num_channels, depth], stddev=0.1))\n", | |
" layer1_biases = tf.Variable(tf.zeros([depth]))\n", | |
" layer2_weights = tf.Variable(tf.truncated_normal(\n", | |
" [patch_size, patch_size, depth, depth], stddev=0.1))\n", | |
" layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n", | |
" layer3_weights = tf.Variable(tf.truncated_normal(\n", | |
" [image_size // 4 * image_size // 4 * depth, num_hidden], stddev=0.1))\n", | |
" layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n", | |
" layer4_weights = tf.Variable(tf.truncated_normal(\n", | |
" [num_hidden, num_labels], stddev=0.1))\n", | |
" layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n", | |
" \n", | |
" # Model.\n", | |
" def model(data):\n", | |
" # reduce convolution to 1 stride\n", | |
" conv = tf.nn.conv2d(data, layer1_weights, [1, 1, 1, 1], padding='SAME')\n", | |
" #add pooling, kernel = 2, stride = 2\n", | |
" pool1 = tf.nn.max_pool(conv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n", | |
" hidden = tf.nn.relu(pool1 + layer1_biases)\n", | |
" conv = tf.nn.conv2d(hidden, layer2_weights, [1, 1, 1, 1], padding='SAME')\n", | |
" pool2 = tf.nn.max_pool(conv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n", | |
" hidden = tf.nn.relu(pool2 + layer2_biases)\n", | |
" shape = hidden.get_shape().as_list()\n", | |
" reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n", | |
" hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)\n", | |
" return tf.matmul(hidden, layer4_weights) + layer4_biases\n", | |
" \n", | |
" # Training computation.\n", | |
" logits = model(tf_train_dataset)\n", | |
" loss = tf.reduce_mean(\n", | |
" tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n", | |
" \n", | |
" # Optimizer.\n", | |
" optimizer = tf.train.GradientDescentOptimizer(0.05).minimize(loss)\n", | |
" \n", | |
" # Predictions for the training, validation, and test data.\n", | |
" train_prediction = tf.nn.softmax(logits)\n", | |
" valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n", | |
" test_prediction = tf.nn.softmax(model(tf_test_dataset))" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Let's run the graph..." | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Initialized\n", | |
"Minibatch loss at step 0: 4.143212\n", | |
"Minibatch accuracy: 6.2%\n", | |
"Validation accuracy: 10.0%\n", | |
"Minibatch loss at step 50: 1.858743\n", | |
"Minibatch accuracy: 43.8%\n", | |
"Validation accuracy: 36.0%\n", | |
"Minibatch loss at step 100: 1.196889\n", | |
"Minibatch accuracy: 50.0%\n", | |
"Validation accuracy: 59.0%\n", | |
"Minibatch loss at step 150: 0.710960\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 69.0%\n", | |
"Minibatch loss at step 200: 0.591121\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 69.0%\n", | |
"Minibatch loss at step 250: 0.364006\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 76.0%\n", | |
"Minibatch loss at step 300: 1.070594\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 350: 0.787328\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 75.0%\n", | |
"Minibatch loss at step 400: 0.453832\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 74.0%\n", | |
"Minibatch loss at step 450: 1.025140\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 76.0%\n", | |
"Minibatch loss at step 500: 0.279667\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 76.0%\n", | |
"Minibatch loss at step 550: 0.498919\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 76.0%\n", | |
"Minibatch loss at step 600: 0.709353\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 79.0%\n", | |
"Minibatch loss at step 650: 0.353762\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 71.0%\n", | |
"Minibatch loss at step 700: 0.706299\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 75.0%\n", | |
"Minibatch loss at step 750: 0.397130\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 75.0%\n", | |
"Minibatch loss at step 800: 0.462259\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 74.0%\n", | |
"Minibatch loss at step 850: 0.137734\n", | |
"Minibatch accuracy: 100.0%\n", | |
"Validation accuracy: 76.0%\n", | |
"Minibatch loss at step 900: 0.162059\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 950: 0.254761\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 77.0%\n", | |
"Minibatch loss at step 1000: 0.079590\n", | |
"Minibatch accuracy: 100.0%\n", | |
"Validation accuracy: 73.0%\n", | |
"Test accuracy: 85.0%\n" | |
] | |
} | |
], | |
"source": [ | |
"num_steps = 1001\n", | |
"\n", | |
"with tf.Session(graph=graph) as session:\n", | |
" tf.global_variables_initializer().run()\n", | |
" print('Initialized')\n", | |
" for step in range(num_steps):\n", | |
" offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n", | |
" batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n", | |
" batch_labels = train_labels[offset:(offset + batch_size), :]\n", | |
" feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n", | |
" _, l, predictions = session.run(\n", | |
" [optimizer, loss, train_prediction], feed_dict=feed_dict)\n", | |
" if (step % 50 == 0):\n", | |
" print('Minibatch loss at step %d: %f' % (step, l))\n", | |
" print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n", | |
" print('Validation accuracy: %.1f%%' % accuracy(\n", | |
" valid_prediction.eval(), valid_labels))\n", | |
" print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"colab_type": "text", | |
"id": "klf21gpbAgb-" | |
}, | |
"source": [ | |
"---\n", | |
"Problem 2\n", | |
"---------\n", | |
"\n", | |
"Try to get the best performance you can using a convolutional net. Look for example at the classic [LeNet5](http://yann.lecun.com/exdb/lenet/) architecture, adding Dropout, and/or adding learning rate decay.\n", | |
"\n", | |
"---" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Ok, since I have problems with processing at the moment and I still need to set up my AWS/ have my own GPU to run it on my personal server, I will introduce the learning rate decay " | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 29, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"batch_size = 16\n", | |
"patch_size = 5\n", | |
"depth = 16\n", | |
"num_hidden = 64\n", | |
"starter_learning_rate = 0.05\n", | |
"\n", | |
"graph = tf.Graph()\n", | |
"\n", | |
"with graph.as_default():\n", | |
"\n", | |
" # Input data.\n", | |
" tf_train_dataset = tf.placeholder(\n", | |
" tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n", | |
" tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n", | |
" tf_valid_dataset = tf.constant(valid_dataset)\n", | |
" tf_test_dataset = tf.constant(test_dataset)\n", | |
" \n", | |
" # Variables.\n", | |
" global_step = tf.Variable(0, trainable=False)\n", | |
" layer1_weights = tf.Variable(tf.truncated_normal(\n", | |
" [patch_size, patch_size, num_channels, depth], stddev=0.1))\n", | |
" layer1_biases = tf.Variable(tf.zeros([depth]))\n", | |
" layer2_weights = tf.Variable(tf.truncated_normal(\n", | |
" [patch_size, patch_size, depth, depth], stddev=0.1))\n", | |
" layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n", | |
" layer3_weights = tf.Variable(tf.truncated_normal(\n", | |
" [image_size // 4 * image_size // 4 * depth, num_hidden], stddev=0.1))\n", | |
" layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n", | |
" layer4_weights = tf.Variable(tf.truncated_normal(\n", | |
" [num_hidden, num_labels], stddev=0.1))\n", | |
" layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n", | |
" \n", | |
" # Model.\n", | |
" def model(data):\n", | |
" # reduce convolution to 1 stride\n", | |
" conv = tf.nn.conv2d(data, layer1_weights, [1, 1, 1, 1], padding='SAME')\n", | |
" #add pooling, kernel = 2, stride = 2\n", | |
" pool1 = tf.nn.max_pool(conv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n", | |
" hidden = tf.nn.relu(pool1 + layer1_biases)\n", | |
" conv = tf.nn.conv2d(hidden, layer2_weights, [1, 1, 1, 1], padding='SAME')\n", | |
" pool2 = tf.nn.max_pool(conv, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n", | |
" hidden = tf.nn.relu(pool2 + layer2_biases)\n", | |
" shape = hidden.get_shape().as_list()\n", | |
" reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n", | |
" hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)\n", | |
" return tf.matmul(hidden, layer4_weights) + layer4_biases\n", | |
" \n", | |
" # Training computation.\n", | |
" logits = model(tf_train_dataset)\n", | |
" loss = tf.reduce_mean(\n", | |
" tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n", | |
" \n", | |
" # Optimizer.\n", | |
" learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 2000, 0.95, staircase=True)\n", | |
" optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss ,global_step=global_step)\n", | |
" \n", | |
" # Predictions for the training, validation, and test data.\n", | |
" train_prediction = tf.nn.softmax(logits)\n", | |
" valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n", | |
" test_prediction = tf.nn.softmax(model(tf_test_dataset))" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 30, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Initialized\n", | |
"Minibatch loss at step 0: 3.491250\n", | |
"Minibatch accuracy: 18.8%\n", | |
"Validation accuracy: 12.0%\n", | |
"Minibatch loss at step 50: 1.075704\n", | |
"Minibatch accuracy: 62.5%\n", | |
"Validation accuracy: 65.0%\n", | |
"Minibatch loss at step 100: 0.905266\n", | |
"Minibatch accuracy: 62.5%\n", | |
"Validation accuracy: 66.0%\n", | |
"Minibatch loss at step 150: 0.670949\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 70.0%\n", | |
"Minibatch loss at step 200: 0.464487\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 70.0%\n", | |
"Minibatch loss at step 250: 0.358510\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 300: 0.868001\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 350: 0.489978\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 75.0%\n", | |
"Minibatch loss at step 400: 0.350634\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 450: 1.272129\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 500: 0.228550\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 550: 0.308218\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 74.0%\n", | |
"Minibatch loss at step 600: 0.819818\n", | |
"Minibatch accuracy: 75.0%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 650: 0.431710\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 68.0%\n", | |
"Minibatch loss at step 700: 0.814631\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 750: 0.527242\n", | |
"Minibatch accuracy: 87.5%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 800: 0.616049\n", | |
"Minibatch accuracy: 81.2%\n", | |
"Validation accuracy: 75.0%\n", | |
"Minibatch loss at step 850: 0.123858\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 73.0%\n", | |
"Minibatch loss at step 900: 0.093429\n", | |
"Minibatch accuracy: 100.0%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 950: 0.201918\n", | |
"Minibatch accuracy: 93.8%\n", | |
"Validation accuracy: 72.0%\n", | |
"Minibatch loss at step 1000: 0.038134\n", | |
"Minibatch accuracy: 100.0%\n", | |
"Validation accuracy: 74.0%\n", | |
"Test accuracy: 88.0%\n" | |
] | |
} | |
], | |
"source": [ | |
"num_steps = 1001\n", | |
"\n", | |
"with tf.Session(graph=graph) as session:\n", | |
" tf.global_variables_initializer().run()\n", | |
" print('Initialized')\n", | |
" for step in range(num_steps):\n", | |
" offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n", | |
" batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n", | |
" batch_labels = train_labels[offset:(offset + batch_size), :]\n", | |
" feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n", | |
" _, l, predictions = session.run(\n", | |
" [optimizer, loss, train_prediction], feed_dict=feed_dict)\n", | |
" if (step % 50 == 0):\n", | |
" print('Minibatch loss at step %d: %f' % (step, l))\n", | |
" print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n", | |
" print('Validation accuracy: %.1f%%' % accuracy(\n", | |
" valid_prediction.eval(), valid_labels))\n", | |
" print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"We can see that the test accuracy went up from 85% to 88% by changing the learning rate to a decaying exponential learning rate" | |
] | |
} | |
], | |
"metadata": { | |
"colab": { | |
"default_view": {}, | |
"name": "4_convolutions.ipynb", | |
"provenance": [], | |
"version": "0.3.2", | |
"views": {} | |
}, | |
"kernelspec": { | |
"display_name": "Python 3", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.5.4" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 1 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment