Tensorflow Auto-Encoder Implementation

autoencoder.py

""" Deep Auto-Encoder implementation
	
	An auto-encoder works as follows:

	Data of dimension k is reduced to a lower dimension j using a matrix multiplication:
	softmax(W*x + b)  = x'
	
	where W is matrix from R^k --> R^j

	A reconstruction matrix W' maps back from R^j --> R^k

	so our reconstruction function is softmax'(W' * x' + b') 

	Now the point of the auto-encoder is to create a reduction matrix (values for W, b) 
	that is "good" at reconstructing  the original data. 

	Thus we want to minimize  ||softmax'(W' * (softmax(W *x+ b)) + b')  - x||

	A deep auto-encoder is nothing more than stacking successive layers of these reductions.
"""
import tensorflow as tf
import numpy as np
import math
import random

def create(x, layer_sizes):

	# Build the encoding layers
	next_layer_input = x

	encoding_matrices = []
	for dim in layer_sizes:
		input_dim = int(next_layer_input.get_shape()[1])

		# Initialize W using random values in interval [-1/sqrt(n) , 1/sqrt(n)]
		W = tf.Variable(tf.random_uniform([input_dim, dim], -1.0 / math.sqrt(input_dim), 1.0 / math.sqrt(input_dim)))

		# Initialize b to zero
		b = tf.Variable(tf.zeros([dim]))

		# We are going to use tied-weights so store the W matrix for later reference.
		encoding_matrices.append(W)

		output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)

		# the input into the next layer is the output of this layer
		next_layer_input = output

	# The fully encoded x value is now stored in the next_layer_input
	encoded_x = next_layer_input

	# build the reconstruction layers by reversing the reductions
	layer_sizes.reverse()
	encoding_matrices.reverse()


	for i, dim in enumerate(layer_sizes[1:] + [ int(x.get_shape()[1])]) :
		# we are using tied weights, so just lookup the encoding matrix for this step and transpose it
		W = tf.transpose(encoding_matrices[i])
		b = tf.Variable(tf.zeros([dim]))
		output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)
		next_layer_input = output

	# the fully encoded and reconstructed value of x is here:
	reconstructed_x = next_layer_input

	return {
		'encoded': encoded_x,
		'decoded': reconstructed_x,
		'cost' : tf.sqrt(tf.reduce_mean(tf.square(x-reconstructed_x)))
	}

def simple_test():
	sess = tf.Session()
	x = tf.placeholder("float", [None, 4])
	autoencoder = create(x, [2])
	init = tf.initialize_all_variables()
	sess.run(init)
	train_step = tf.train.GradientDescentOptimizer(0.01).minimize(autoencoder['cost'])


	# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
	c1 = np.array([0,0,0.5,0])
	c2 = np.array([0.5,0,0,0])

	# do 1000 training steps
	for i in range(2000):
		# make a batch of 100:
		batch = []
		for j in range(100):
			# pick a random centroid
			if (random.random() > 0.5):
				vec = c1
			else:
				vec = c2
  			batch.append(np.random.normal(vec, 0.1))
		sess.run(train_step, feed_dict={x: np.array(batch)})
		if i % 100 == 0:
			print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})


def deep_test():
	sess = tf.Session()
	start_dim = 5
	x = tf.placeholder("float", [None, start_dim])
	autoencoder = create(x, [4, 3, 2])
	init = tf.initialize_all_variables()
	sess.run(init)
	train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost'])


	# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
	c1 = np.zeros(start_dim)
	c1[0] = 1

	print c1

	c2 = np.zeros(start_dim)
	c2[1] = 1

	# do 1000 training steps
	for i in range(5000):
		# make a batch of 100:
		batch = []
		for j in range(1):
			# pick a random centroid
			if (random.random() > 0.5):
				vec = c1
			else:
				vec = c2
  			batch.append(np.random.normal(vec, 0.1))
		sess.run(train_step, feed_dict={x: np.array(batch)})
		if i % 100 == 0:
			print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})
			print i, " original", batch[0]
			print i, " decoded", sess.run(autoencoder['decoded'], feed_dict={x: batch})
			
if __name__ == '__main__':
	deep_test()

Hi saliksyed! Great work with this... However when you construct the decoding layers it appears you need to transpose the input when you multiply it by W^T, and transpose the Output again to get it to work for the next layer... Right?

Auto encoder code i used back propagation algorithm.But performance only 10%. I tried to change learning rate.even though only beloiw 20%. What change should me made
import tensorflow
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
a_0 = tf.placeholder(tf.float32, [None, 784])
y = tf.placeholder(tf.float32, [None, 784])
middle = 785
w_1 = tf.Variable(tf.truncated_normal([784, middle]))
b_1 = tf.Variable(tf.truncated_normal([1, middle]))
w_2 = tf.Variable(tf.truncated_normal([middle, 784]))
b_2 = tf.Variable(tf.truncated_normal([1, 784]))
def sigma(x):
return tf.div(tf.constant(1.0),
tf.add(tf.constant(1.0), tf.exp(tf.neg(x))))
z_1 = tf.add(tf.matmul(a_0, w_1), b_1)
a_1 = sigma(z_1)
z_2 = tf.add(tf.matmul(a_1, w_2), b_2)
a_2 = sigma(z_2)
diff = tf.sub(a_2, y)
def sigmaprime(x):
return tf.mul(sigma(x), tf.sub(tf.constant(1.0), sigma(x)))
d_z_2 = tf.mul(diff, sigmaprime(z_2))
d_b_2 = d_z_2
d_w_2 = tf.matmul(tf.transpose(a_1), d_z_2)
d_a_1 = tf.matmul(d_z_2, tf.transpose(w_2))
d_z_1 = tf.mul(d_a_1, sigmaprime(z_1))
d_b_1 = d_z_1
d_w_1 = tf.matmul(tf.transpose(a_0), d_z_1)
eta = tf.constant( 0.385)
step = [
tf.assign(w_1,
tf.sub(w_1, tf.mul(eta, d_w_1)))
, tf.assign(b_1,
tf.sub(b_1, tf.mul(eta,
tf.reduce_mean(d_b_1, reduction_indices=[0]))))
, tf.assign(w_2,
tf.sub(w_2, tf.mul(eta, d_w_2)))
, tf.assign(b_2,
tf.sub(b_2, tf.mul(eta,
tf.reduce_mean(d_b_2, reduction_indices=[0]))))
]
acct_mat = tf.equal(tf.argmax(a_2, 1), tf.argmax(y, 1))
acct_res = tf.reduce_sum(tf.cast(acct_mat, tf.float32))
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())
for i in xrange(10000):
batch_xs, batch_ys = mnist.train.next_batch(10)
sess.run(step, feed_dict = {a_0: batch_xs,y : batch_xs})
if i % 1000 == 0:
res = sess.run(acct_res, feed_dict ={a_0: mnist.test.images[:1000],y : mnist.test.images[:1000]})
print i,res

I have a question, how to implement this in such a way so that we can get the outputs like Recall , Precision, F-1 score and confusion matrix....how to add this....I am new to it,..if any1 can help me/////

It seems that you didn't implement pretraining layer by layer.

I am trying this code but this giving me error in a session part.. session not working