Deep Learning 4

DL4.0_Stacked.py

""" Auto Encoder Example.

Build a 2 layers auto-encoder with TensorFlow to compress images to a
lower latent space and then reconstruct them.

"""
from __future__ import division, print_function, absolute_import

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("MNIST_data", one_hot=True)


learning_rate = 0.01                # changed for q3
num_steps = 5000                    # changed steps to 5000 ********* Increased for Run 8
batch_size = 512                   # changed for q3, optimized at 512

display_step = 1000
examples_to_show = 10

num_hidden_1 = 256
num_hidden_2 = 256
num_hidden_3 = 128
num_input = 784
X = tf.placeholder("float", [None, num_input])
# ********************    added 3rd layer ****************************************************
weights = {
    'encoder_h1': tf.Variable(tf.random_normal([num_input, num_hidden_1])),
    'encoder_h2': tf.Variable(tf.random_normal([num_hidden_1, num_hidden_2])),
    'encoder_h3': tf.Variable(tf.random_normal([num_hidden_2, num_hidden_3])),
    'decoder_h1': tf.Variable(tf.random_normal([num_hidden_3, num_hidden_2])),
    'decoder_h2': tf.Variable(tf.random_normal([num_hidden_2, num_hidden_1])),
    'decoder_h3': tf.Variable(tf.random_normal([num_hidden_1, num_input])),
}
biases = {
    'encoder_b1': tf.Variable(tf.random_normal([num_hidden_1])),
    'encoder_b2': tf.Variable(tf.random_normal([num_hidden_2])),
    'encoder_b3': tf.Variable(tf.random_normal([num_hidden_3])),
    'decoder_b1': tf.Variable(tf.random_normal([num_hidden_1])),
    'decoder_b2': tf.Variable(tf.random_normal([num_hidden_2])),
    'decoder_b3': tf.Variable(tf.random_normal([num_input])),
}


def encoder(x):

    layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['encoder_h1']),
                                   biases['encoder_b1']))
    layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['encoder_h2']),
                                   biases['encoder_b2']))
    layer_3 = tf.nn.sigmoid(tf.add(tf.matmul(layer_2, weights['encoder_h3']),
                                   biases['encoder_b3']))

    return layer_3


def decoder(x):

    layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['decoder_h1']),
                                   biases['decoder_b1']))
    layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['decoder_h2']),
                                   biases['decoder_b2']))
    layer_3 = tf.nn.sigmoid(tf.add(tf.matmul(layer_2, weights['decoder_h3']),
                                   biases['decoder_b3']))
    return layer_3


# Construct model
encoder_op = encoder(X)
decoder_op = decoder(encoder_op)

# Prediction
y_pred = decoder_op
# Targets (Labels) are the input data.
y_true = X

# Define loss and optimizer, minimize the squared error
loss = tf.reduce_mean(tf.pow(y_true - y_pred, 2))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)  # changed for q4
trainer = optimizer.minimize(loss)

# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()

with tf.Session() as sess:
    # Run the initializer

    sess.run(init)
    tf.summary.scalar('Loss', loss)
    merged_summary_op = tf.summary.merge_all()
    summary_writer = tf.summary.FileWriter('tensorflow_dir', graph=tf.get_default_graph())

    for i in range(1, num_steps + 1):
        # Prepare Data
        # Get the next batch of MNIST data (only images are needed, not labels)
        batch_x, _ = mnist.train.next_batch(batch_size)

        # Run optimization op (backprop) and cost op (to get loss value)
        _, l, summary = sess.run([trainer, loss, merged_summary_op], feed_dict={X: batch_x})

        summary_writer.add_summary(summary, i)
        # Display logs per step
        if i % display_step == 0 or i == 1:
            print('Step %i: Minibatch Loss: %f' % (i, l))

    # Testing
    # Encode and decode images from test set and visualize their reconstruction.
    n = 4
    canvas_orig = np.empty((28 * n, 28 * n))
    canvas_recon = np.empty((28 * n, 28 * n))
    for i in range(n):
        # MNIST test set
        batch_x, _ = mnist.test.next_batch(n)
        # Encode and decode the digit image
        g = sess.run(decoder_op, feed_dict={X: batch_x})

        # Display original images
        for j in range(n):
            # Draw the original digits
            canvas_orig[i * 28:(i + 1) * 28, j * 28:(j + 1) * 28] = \
                batch_x[j].reshape([28, 28])
        # Display reconstructed images
        for j in range(n):
            # Draw the reconstructed digits
            canvas_recon[i * 28:(i + 1) * 28, j * 28:(j + 1) * 28] = \
                g[j].reshape([28, 28])

    print("Original Images")
    plt.figure(figsize=(n, n))
    plt.imshow(canvas_orig, origin="upper", cmap="gray")
    plt.show()

    print("Reconstructed Images")
    plt.figure(figsize=(n, n))
    plt.imshow(canvas_recon, origin="upper", cmap="gray")
    plt.show()

DL4.1_Simple.py

#
#   simple_ae.py
#
#   Autoencoder tutorial code
#
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import matplotlib as mpl

mpl.use('Agg')
import matplotlib.pyplot as plt
import tensorflow as tf

# Import data
from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

# Variables
x = tf.placeholder("float", [None, 784])
y_ = tf.placeholder("float", [None, 10])

w_enc = tf.Variable(tf.random_normal([784, 625], mean=0.0, stddev=0.05))
w_dec = tf.Variable(tf.random_normal([625, 784], mean=0.0, stddev=0.05))

b_enc = tf.Variable(tf.zeros([625]))
b_dec = tf.Variable(tf.zeros([784]))


# Create the model
def model(X, w_e, b_e, w_d, b_d):
    encoded = tf.sigmoid(tf.matmul(X, w_e) + b_e)
    decoded = tf.sigmoid(tf.matmul(encoded, w_d) + b_d)

    return encoded, decoded


encoded, decoded = model(x, w_enc, b_enc, w_dec, b_dec)

# Cost Function basic term
cross_entropy = -1. * x * tf.log(decoded) - (1. - x) * tf.log(1. - decoded)
loss = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)# changed for q4
trainer = optimizer.minimize(loss)

# Trai
init = tf.initialize_all_variables()

with tf.Session() as sess:
    sess.run(init)
    tf.summary.scalar('Loss', loss)
    merged_summary_op = tf.summary.merge_all()              # Tensorboard Graphing *****************************
    summary_writer = tf.summary.FileWriter('tensorflow_dir', graph=tf.get_default_graph())
    print('Training...')
    for i in range(6001):
        batch_xs, batch_ys = mnist.train.next_batch(64)
        trainer.run({x: batch_xs, y_: batch_ys})

        if i % 1000 == 0:
            train_loss = loss.eval({x: batch_xs, y_: batch_ys})
            print('  step, loss = %6d: %6.3f' % (i, train_loss))

    # generate decoded image with test data
    test_fd = {x: mnist.test.images, y_: mnist.test.labels}
    decoded_imgs = decoded.eval(test_fd)
    print('loss (test) = ', loss.eval(test_fd))

x_test = mnist.test.images

n = 10  # how many digits we will display
plt.figure(figsize=(20, 4))
for i in range(n):
    # display original
    ax = plt.subplot(2, n, i + 1)
    plt.imshow(x_test[i].reshape(28, 28))
    plt.gray()
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)

    # display reconstruction
    ax = plt.subplot(2, n, i + 1 + n)
    plt.imshow(decoded_imgs[i].reshape(28, 28))
    plt.gray()
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)

plt.savefig('simple_ae.png')

DL_ICP_4_Results.txt

*************************************    STACKED   ********************************************************

Run # 1 (no changes)
Step 1: Minibatch Loss: 0.449478
Step 1000: Minibatch Loss: 0.144752
Step 2000: Minibatch Loss: 0.130133
Step 3000: Minibatch Loss: 0.120846
Step 4000: Minibatch Loss: 0.113468
Step 5000: Minibatch Loss: 0.107940


Run # 2 (Learning Rate at 0.1)
Step 1: Minibatch Loss: 0.443141
Step 1000: Minibatch Loss: 0.197213
Step 2000: Minibatch Loss: 0.185982
Step 3000: Minibatch Loss: 0.179341
Step 4000: Minibatch Loss: 0.176640
Step 5000: Minibatch Loss: 0.176243
Loss increased, image very distorted

Run # 3 (Learning Rate at 0.001)
Step 1: Minibatch Loss: 0.452690
Step 1000: Minibatch Loss: 0.184412
Step 2000: Minibatch Loss: 0.168929
Step 3000: Minibatch Loss: 0.155669
Step 4000: Minibatch Loss: 0.150678
Step 5000: Minibatch Loss: 0.144990
Loss increased, image has medium distortion

Run # 4 (Batch Size 512)
Step 1: Minibatch Loss: 0.444183
Step 1000: Minibatch Loss: 0.129117
Step 2000: Minibatch Loss: 0.110361
Step 3000: Minibatch Loss: 0.101107
Step 4000: Minibatch Loss: 0.093735
Step 5000: Minibatch Loss: 0.089930
Loss decreased, image distortion lowered from Run #1

Run # 5 (Batch Size 1024)
Step 1: Minibatch Loss: 0.452890
Step 1000: Minibatch Loss: 0.135834
Step 2000: Minibatch Loss: 0.117842
Step 3000: Minibatch Loss: 0.106187
Step 4000: Minibatch Loss: 0.101468
Step 5000: Minibatch Loss: 0.098454
Loss/image distortion increased from Run # 4, runtime significantly increased

Run # 6 (Gradient Descent Optimizer - Learning rate at 0.01)
Step 1: Minibatch Loss: 0.449574
Step 1000: Minibatch Loss: 0.447457
Step 2000: Minibatch Loss: 0.442519
Step 3000: Minibatch Loss: 0.438558
Step 4000: Minibatch Loss: 0.434089
Step 5000: Minibatch Loss: 0.431370
Loss significantly increased, Image unrecognizable

Run # 7 (Adam Optimizer - Learning rate 0.01)
Step 1: Minibatch Loss: 0.455449
Step 1000: Minibatch Loss: 0.059743
Step 2000: Minibatch Loss: 0.055238
Step 3000: Minibatch Loss: 0.050760
Step 4000: Minibatch Loss: 0.049519
Step 5000: Minibatch Loss: 0.047893
Loss/image distortion significant decrease, best results overall

Run # 8 (Step increased to 10000)
Step 1: Minibatch Loss: 0.458655
Step 1000: Minibatch Loss: 0.075279
Step 2000: Minibatch Loss: 0.063079
Step 3000: Minibatch Loss: 0.057277
Step 4000: Minibatch Loss: 0.055248
Step 5000: Minibatch Loss: 0.053867
Step 6000: Minibatch Loss: 0.052110
Step 7000: Minibatch Loss: 0.052557
Step 8000: Minibatch Loss: 0.050941
Step 9000: Minibatch Loss: 0.050368
Step 10000: Minibatch Loss: 0.051364
No significant change from run # 7

Run # 9 (3rd layer of Autoencoder added)
Step 1: Minibatch Loss: 0.459115
Step 1000: Minibatch Loss: 0.079921
Step 2000: Minibatch Loss: 0.070803
Step 3000: Minibatch Loss: 0.065422
Step 4000: Minibatch Loss: 0.063323
Step 5000: Minibatch Loss: 0.061583
Slight decrease from run # 8

****************************************** SIMPLE *************************************************

Run # 1 (no changes)
Training...
  step, loss =      0:  0.734
  step, loss =   1000:  0.262
  step, loss =   2000:  0.250
  step, loss =   3000:  0.233
  step, loss =   4000:  0.222
  step, loss =   5000:  0.214
  step, loss =   6000:  0.211
  step, loss =   7000:  0.203
  step, loss =   8000:  0.197
  step, loss =   9000:  0.190
  step, loss =  10000:  0.176
loss (test) =  0.17976473

Run # 2 (Batch Size increased (256), Range decreased (6001), Optimizer .1 to .01)

Training...
  step, loss =      0:  0.740
  step, loss =   1000:  0.342
  step, loss =   2000:  0.292
  step, loss =   3000:  0.283
  step, loss =   4000:  0.268
  step, loss =   5000:  0.270
  step, loss =   6000:  0.270
  loss (test) =  0.2662039

  Loss increased

Run # 3 (Batch Size decreased (64), Optimizer to .1)
Training...
  step, loss =      0:  0.728
  step, loss =   1000:  0.274
  step, loss =   2000:  0.247
  step, loss =   3000:  0.237
  step, loss =   4000:  0.224
  step, loss =   5000:  0.211
  step, loss =   6000:  0.207
  loss (test) =  0.20531747
    Loss increased from Run # 1

Run # 4 (Batch size decreased (32), step increased 10001)

Training...
  step, loss =      0:  0.738
  step, loss =   1000:  0.264
  step, loss =   2000:  0.254
  step, loss =   3000:  0.256
  step, loss =   4000:  0.227
  step, loss =   5000:  0.214
  step, loss =   6000:  0.193
  step, loss =   7000:  0.195
  step, loss =   8000:  0.188
  step, loss =   9000:  0.191
  step, loss =  10000:  0.169
  loss (test) =  0.17982607
  No change from run #1

Run # 5 (Optimizer changed to AdamOptimizer with 0.01 learning rate, steps at 6001)

Training...
  step, loss =      0:  0.391
  step, loss =   1000:  0.065
  step, loss =   2000:  0.068
  step, loss =   3000:  0.063
  step, loss =   4000:  0.066
  step, loss =   5000:  0.067
  step, loss =   6000:  0.065
  loss (test) =  0.066782735
  Significant decrease in loss from all previous runs....