Skip to content

Instantly share code, notes, and snippets.

@saliksyed
Created November 18, 2015 03:30
Show Gist options
  • Save saliksyed/593c950ba1a3b9dd08d5 to your computer and use it in GitHub Desktop.
Save saliksyed/593c950ba1a3b9dd08d5 to your computer and use it in GitHub Desktop.
Tensorflow Auto-Encoder Implementation
""" Deep Auto-Encoder implementation
An auto-encoder works as follows:
Data of dimension k is reduced to a lower dimension j using a matrix multiplication:
softmax(W*x + b) = x'
where W is matrix from R^k --> R^j
A reconstruction matrix W' maps back from R^j --> R^k
so our reconstruction function is softmax'(W' * x' + b')
Now the point of the auto-encoder is to create a reduction matrix (values for W, b)
that is "good" at reconstructing the original data.
Thus we want to minimize ||softmax'(W' * (softmax(W *x+ b)) + b') - x||
A deep auto-encoder is nothing more than stacking successive layers of these reductions.
"""
import tensorflow as tf
import numpy as np
import math
import random
def create(x, layer_sizes):
# Build the encoding layers
next_layer_input = x
encoding_matrices = []
for dim in layer_sizes:
input_dim = int(next_layer_input.get_shape()[1])
# Initialize W using random values in interval [-1/sqrt(n) , 1/sqrt(n)]
W = tf.Variable(tf.random_uniform([input_dim, dim], -1.0 / math.sqrt(input_dim), 1.0 / math.sqrt(input_dim)))
# Initialize b to zero
b = tf.Variable(tf.zeros([dim]))
# We are going to use tied-weights so store the W matrix for later reference.
encoding_matrices.append(W)
output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)
# the input into the next layer is the output of this layer
next_layer_input = output
# The fully encoded x value is now stored in the next_layer_input
encoded_x = next_layer_input
# build the reconstruction layers by reversing the reductions
layer_sizes.reverse()
encoding_matrices.reverse()
for i, dim in enumerate(layer_sizes[1:] + [ int(x.get_shape()[1])]) :
# we are using tied weights, so just lookup the encoding matrix for this step and transpose it
W = tf.transpose(encoding_matrices[i])
b = tf.Variable(tf.zeros([dim]))
output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)
next_layer_input = output
# the fully encoded and reconstructed value of x is here:
reconstructed_x = next_layer_input
return {
'encoded': encoded_x,
'decoded': reconstructed_x,
'cost' : tf.sqrt(tf.reduce_mean(tf.square(x-reconstructed_x)))
}
def simple_test():
sess = tf.Session()
x = tf.placeholder("float", [None, 4])
autoencoder = create(x, [2])
init = tf.initialize_all_variables()
sess.run(init)
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(autoencoder['cost'])
# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
c1 = np.array([0,0,0.5,0])
c2 = np.array([0.5,0,0,0])
# do 1000 training steps
for i in range(2000):
# make a batch of 100:
batch = []
for j in range(100):
# pick a random centroid
if (random.random() > 0.5):
vec = c1
else:
vec = c2
batch.append(np.random.normal(vec, 0.1))
sess.run(train_step, feed_dict={x: np.array(batch)})
if i % 100 == 0:
print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})
def deep_test():
sess = tf.Session()
start_dim = 5
x = tf.placeholder("float", [None, start_dim])
autoencoder = create(x, [4, 3, 2])
init = tf.initialize_all_variables()
sess.run(init)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost'])
# Our dataset consists of two centers with gaussian noise w/ sigma = 0.1
c1 = np.zeros(start_dim)
c1[0] = 1
print c1
c2 = np.zeros(start_dim)
c2[1] = 1
# do 1000 training steps
for i in range(5000):
# make a batch of 100:
batch = []
for j in range(1):
# pick a random centroid
if (random.random() > 0.5):
vec = c1
else:
vec = c2
batch.append(np.random.normal(vec, 0.1))
sess.run(train_step, feed_dict={x: np.array(batch)})
if i % 100 == 0:
print i, " cost", sess.run(autoencoder['cost'], feed_dict={x: batch})
print i, " original", batch[0]
print i, " decoded", sess.run(autoencoder['decoded'], feed_dict={x: batch})
if __name__ == '__main__':
deep_test()
@oppsitre
Copy link

oppsitre commented Jun 4, 2017

It seems that you didn't implement pretraining layer by layer.

@GraniteConsultingReviews

I am trying this code but this giving me error in a session part.. session not working

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment