Example of Variational Autoencoder (Tensorflow)

Simple_VAE.py

import tensorflow as tf
import numpy as np

epochs      = 10000
batch_size  = 1500
d           = 2
enc_1_hidden_units  = 128
enc_2_hidden_units  = 64
mean_hidden_units = std_hidden_units = 2
dec_1_hidden_units  = 64
dec_2_hidden_units  = 128

x = tf.placeholder("float", (None, d))
y = tf.placeholder("float", (None, d))

W_encoder_1     = tf.Variable(tf.random_normal([d, enc_1_hidden_units]))
W_encoder_2     = tf.Variable(tf.random_normal([enc_1_hidden_units, enc_2_hidden_units]))
W_mean          = tf.Variable(tf.random_normal([enc_2_hidden_units, 
				   							    mean_hidden_units]))
W_sd            = tf.Variable(tf.random_normal([enc_2_hidden_units, 
											    std_hidden_units]))
W_decoder_1     = tf.Variable(tf.random_normal([std_hidden_units, 
											    dec_1_hidden_units]))
W_decoder_2     = tf.Variable(tf.random_normal([dec_1_hidden_units, 
											    dec_2_hidden_units]))
W_output        = tf.Variable(tf.random_normal([dec_2_hidden_units, d]))
b_encoder_1     = tf.Variable(tf.random_normal([enc_1_hidden_units]))
b_encoder_2     = tf.Variable(tf.random_normal([enc_2_hidden_units]))
b_mean          = tf.Variable(tf.random_normal([mean_hidden_units]))
b_sd            = tf.Variable(tf.random_normal([std_hidden_units]))
b_decoder_1     = tf.Variable(tf.random_normal([dec_1_hidden_units]))
b_decoder_2     = tf.Variable(tf.random_normal([dec_2_hidden_units]))
b_output        = tf.Variable(tf.random_normal([d]))

layer_encoder   = tf.nn.sigmoid(tf.matmul(x, W_encoder_1) + b_encoder_1)
layer_encoder_2 = tf.nn.sigmoid(tf.matmul(layer_encoder, W_encoder_2) + b_encoder_2)
layer_mean      = tf.matmul(layer_encoder_2, W_mean) + b_mean
layer_sd        = tf.matmul(layer_encoder_2, W_sd) + b_sd
epsilon         = tf.random_normal((batch_size, std_hidden_units)) 
sd_encoder      = tf.exp(0.5 * layer_sd)
encoder_output  = layer_mean + (sd_encoder * epsilon)

layer_decoder_1 = tf.nn.sigmoid(tf.matmul(encoder_output, W_decoder_1) + b_decoder_1)
layer_decoder_2 = tf.nn.sigmoid(tf.matmul(layer_decoder_1, W_decoder_2) + b_decoder_2)

layer_output    = tf.matmul(layer_decoder_2, W_output) + b_output

latent_loss     = -0.5 * tf.reduce_sum(1.0 + layer_sd - tf.pow(layer_mean, 2) - tf.exp(layer_sd))
recons_loss     = tf.reduce_mean(tf.square(layer_output-y))
loss            = tf.reduce_mean(latent_loss + recons_loss)

train_step = tf.train.AdamOptimizer().minimize(loss)
init = tf.global_variables_initializer()

sess = tf.Session()
sess.run(init)

x_train =  np.random.normal(0, 1, (batch_size, d))

for i in range(epochs):
	sess.run(train_step, feed_dict={x:x_train, y:x_train}) 	
	print(sess.run(loss, feed_dict={x:x_train, y:x_train}))