Skip to content

Instantly share code, notes, and snippets.

@saliksyed
Created November 18, 2015 03:30
Show Gist options
  • Save saliksyed/593c950ba1a3b9dd08d5 to your computer and use it in GitHub Desktop.
Save saliksyed/593c950ba1a3b9dd08d5 to your computer and use it in GitHub Desktop.
Tensorflow Auto-Encoder Implementation
""" Deep Auto-Encoder implementation
An auto-encoder works as follows:
Data of dimension k is reduced to a lower dimension j using a matrix multiplication:
softmax(W*x + b) = x'
where W is matrix from R^k --> R^j
A reconstruction matrix W' maps back from R^j --> R^k
so our reconstruction function is softmax'(W' * x' + b')
Now the point of the auto-encoder is to create a reduction matrix (values for W, b)
that is "good" at reconstructing the original data.
Thus we want to minimize ||softmax'(W' * (softmax(W *x+ b)) + b') - x||
A deep auto-encoder is nothing more than stacking successive layers of these reductions.
"""
import tensorflow as tf
import numpy as np
import math
import random
def create(x, layer_sizes):
# Build the encoding layers
next_layer_input = x
encoding_matrices = []
for dim in layer_sizes:
input_dim = int(next_layer_input.get_shape()[1])
# Initialize W using random values in interval [-1/sqrt(n) , 1/sqrt(n)]
W = tf.Variable(tf.random_uniform([input_dim, dim], -1.0 / math.sqrt(input_dim), 1.0 / math.sqrt(input_dim)))
# Initialize b to zero
b = tf.Variable(tf.zeros([dim]))
# We are going to use tied-weights so store the W matrix for later reference.
encoding_matrices.append(W)
output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)
# the input into the next layer is the output of this layer
next_layer_input = output
# The fully encoded x value is now stored in the next_layer_input
encoded_x = next_layer_input
# build the reconstruction layers by reversing the reductions
layer_sizes.reverse()
encoding_matrices.reverse()
for i, dim in enumerate(layer_sizes[1:] + [ int(x.get_shape()[1])]) :
# we are using tied weights, so just lookup the encoding matrix for this step and transpose it
W = tf.transpose(encoding_matrices[i])
b = tf.Variable(tf.zeros([dim]))
output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)
next_layer_input = output
# the fully encoded and reconstructed value of x is here:
reconstructed_x = next_layer_input
return {
'encoded': encoded_x,
'decoded': reconstructed_x,
'cost' : tf.sqrt(tf.reduce_mean(tf.square(x-reconstructed_x)))
}
def simple_test():
sess = tf.Session()
x = tf.placeholder("float", [None, 4])
autoencoder = create(x, [2])
init = tf.initialize_all_variables()
sess.run(init)
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(autoencoder['cost'])
# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
c1 = np.array([0,0,0.5,0])
c2 = np.array([0.5,0,0,0])
# do 1000 training steps
for i in range(2000):
# make a batch of 100:
batch = []
for j in range(100):
# pick a random centroid
if (random.random() > 0.5):
vec = c1
else:
vec = c2
batch.append(np.random.normal(vec, 0.1))
sess.run(train_step, feed_dict={x: np.array(batch)})
if i % 100 == 0:
print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})
def deep_test():
sess = tf.Session()
start_dim = 5
x = tf.placeholder("float", [None, start_dim])
autoencoder = create(x, [4, 3, 2])
init = tf.initialize_all_variables()
sess.run(init)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost'])
# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
c1 = np.zeros(start_dim)
c1[0] = 1
print c1
c2 = np.zeros(start_dim)
c2[1] = 1
# do 1000 training steps
for i in range(5000):
# make a batch of 100:
batch = []
for j in range(1):
# pick a random centroid
if (random.random() > 0.5):
vec = c1
else:
vec = c2
batch.append(np.random.normal(vec, 0.1))
sess.run(train_step, feed_dict={x: np.array(batch)})
if i % 100 == 0:
print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})
print i, " original", batch[0]
print i, " decoded", sess.run(autoencoder['decoded'], feed_dict={x: batch})
if __name__ == '__main__':
deep_test()
@nikwolfe7
Copy link

Hi saliksyed! Great work with this... However when you construct the decoding layers it appears you need to transpose the input when you multiply it by W^T, and transpose the Output again to get it to work for the next layer... Right?

@prince16
Copy link

Auto encoder code i used back propagation algorithm.But performance only 10%. I tried to change learning rate.even though only beloiw 20%. What change should me made
import tensorflow
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
a_0 = tf.placeholder(tf.float32, [None, 784])
y = tf.placeholder(tf.float32, [None, 784])
middle = 785
w_1 = tf.Variable(tf.truncated_normal([784, middle]))
b_1 = tf.Variable(tf.truncated_normal([1, middle]))
w_2 = tf.Variable(tf.truncated_normal([middle, 784]))
b_2 = tf.Variable(tf.truncated_normal([1, 784]))
def sigma(x):
return tf.div(tf.constant(1.0),
tf.add(tf.constant(1.0), tf.exp(tf.neg(x))))
z_1 = tf.add(tf.matmul(a_0, w_1), b_1)
a_1 = sigma(z_1)
z_2 = tf.add(tf.matmul(a_1, w_2), b_2)
a_2 = sigma(z_2)
diff = tf.sub(a_2, y)
def sigmaprime(x):
return tf.mul(sigma(x), tf.sub(tf.constant(1.0), sigma(x)))
d_z_2 = tf.mul(diff, sigmaprime(z_2))
d_b_2 = d_z_2
d_w_2 = tf.matmul(tf.transpose(a_1), d_z_2)
d_a_1 = tf.matmul(d_z_2, tf.transpose(w_2))
d_z_1 = tf.mul(d_a_1, sigmaprime(z_1))
d_b_1 = d_z_1
d_w_1 = tf.matmul(tf.transpose(a_0), d_z_1)
eta = tf.constant( 0.385)
step = [
tf.assign(w_1,
tf.sub(w_1, tf.mul(eta, d_w_1)))
, tf.assign(b_1,
tf.sub(b_1, tf.mul(eta,
tf.reduce_mean(d_b_1, reduction_indices=[0]))))
, tf.assign(w_2,
tf.sub(w_2, tf.mul(eta, d_w_2)))
, tf.assign(b_2,
tf.sub(b_2, tf.mul(eta,
tf.reduce_mean(d_b_2, reduction_indices=[0]))))
]
acct_mat = tf.equal(tf.argmax(a_2, 1), tf.argmax(y, 1))
acct_res = tf.reduce_sum(tf.cast(acct_mat, tf.float32))
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())
for i in xrange(10000):
batch_xs, batch_ys = mnist.train.next_batch(10)
sess.run(step, feed_dict = {a_0: batch_xs,y : batch_xs})
if i % 1000 == 0:
res = sess.run(acct_res, feed_dict ={a_0: mnist.test.images[:1000],y : mnist.test.images[:1000]})
print i,res

@kartiksareen
Copy link

I have a question, how to implement this in such a way so that we can get the outputs like Recall , Precision, F-1 score and confusion matrix....how to add this....I am new to it,..if any1 can help me/////

@oppsitre
Copy link

oppsitre commented Jun 4, 2017

It seems that you didn't implement pretraining layer by layer.

@GraniteConsultingReviews

I am trying this code but this giving me error in a session part.. session not working

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment