Last active
August 18, 2017 06:32
-
-
Save iolalla/5047b0c94f6a93cd2e872447a0db35f7 to your computer and use it in GitHub Desktop.
Very simple Recursive Neural Network with Memory (Long Short Term Memory LSTM) using Tensorboard to track training.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#Source code with the blog post at http://monik.in/a-noobs-guide-to-implementing-rnn-lstm-using-tensorflow/ | |
#Original gist: https://gist.github.com/AlmostDan/a5f4b57104532a68bda2c274f8bcf16f | |
#Tensorboard inspirational: https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/4_Utils/tensorboard_basic.py | |
import numpy as np | |
#import random | |
from random import shuffle | |
import tensorflow as tf | |
NUM_EXAMPLES = 10000 | |
def variable_summaries(var): | |
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization).""" | |
with tf.name_scope('summaries'): | |
mean = tf.reduce_mean(var) | |
tf.summary.scalar('mean', mean) | |
with tf.name_scope('stddev'): | |
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) | |
tf.summary.scalar('stddev', stddev) | |
tf.summary.scalar('max', tf.reduce_max(var)) | |
tf.summary.scalar('min', tf.reduce_min(var)) | |
tf.summary.histogram('histogram', var) | |
# Generate a list of 1.048.576 elements of format: '11111111111110001010', '11111111111110001011' | |
train_input = ['{0:020b}'.format(i) for i in range(2**20)] | |
shuffle(train_input) | |
train_input = [map(int,i) for i in train_input] | |
ti = [] | |
for i in train_input: | |
temp_list = [] | |
for j in i: | |
temp_list.append([j]) | |
ti.append(np.array(temp_list)) | |
train_input = ti | |
train_output = [] | |
for i in train_input: | |
count = 0 | |
for j in i: | |
if j[0] == 1: | |
count+=1 | |
temp_list = ([0]*21) | |
temp_list[count]=1 | |
train_output.append(temp_list) | |
test_input = train_input[NUM_EXAMPLES:] | |
test_output = train_output[NUM_EXAMPLES:] | |
train_input = train_input[:NUM_EXAMPLES] | |
train_output = train_output[:NUM_EXAMPLES] | |
print("test and training data loaded") | |
data = tf.placeholder(tf.float32, [None, 20,1]) #Number of examples, number of input, dimension of each input | |
target = tf.placeholder(tf.float32, [None, 21]) | |
num_hidden = 24 | |
tf.name_scope('Counting1s') | |
# Define the cell to 'remember' the counted data | |
cell = tf.contrib.rnn.LSTMCell(num_hidden,state_is_tuple=True) | |
val, _ = tf.nn.dynamic_rnn(cell, data, dtype=tf.float32) | |
val = tf.transpose(val, [1, 0, 2]) | |
last = tf.gather(val, int(val.get_shape()[0]) - 1) | |
with tf.name_scope('weights'): | |
weight = tf.Variable(tf.truncated_normal([num_hidden, int(target.get_shape()[1])])) | |
variable_summaries(weight) | |
with tf.name_scope('bias'): | |
bias = tf.Variable(tf.constant(0.1, shape=[target.get_shape()[1]])) | |
variable_summaries(bias) | |
with tf.name_scope('prediction'): | |
# Let's use softmax, instead of relu and sigmoid, you can play around with those and see the impact. | |
prediction = tf.nn.softmax(tf.matmul(last, weight) + bias) | |
variable_summaries(prediction) | |
with tf.name_scope('crossEntropy'): | |
cross_entropy = -tf.reduce_sum(target * tf.log(tf.clip_by_value(prediction,1e-10,1.0))) | |
tf.summary.scalar('cross_entropy', cross_entropy) | |
optimizer = tf.train.AdamOptimizer() | |
minimize = optimizer.minimize(cross_entropy) | |
mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1)) | |
error = tf.reduce_mean(tf.cast(mistakes, tf.float32)) | |
variable_summaries(error) | |
sess = tf.Session() | |
# Merge all the summaries and write them out to logs/ | |
merged = tf.summary.merge_all() | |
train_writer = tf.summary.FileWriter('logs/train', sess.graph) | |
test_writer = tf.summary.FileWriter('logs/test', sess.graph) | |
tf.global_variables_initializer().run(session=sess) | |
batch_size = 1000 | |
no_of_batches = int(len(train_input) / batch_size) | |
epoch = 5000 | |
for i in range(epoch): | |
ptr = 0 | |
for j in range(no_of_batches): | |
inp, out = train_input[ptr:ptr+batch_size], train_output[ptr:ptr+batch_size] | |
ptr += batch_size | |
mini, summaries = sess.run([minimize,merged],feed_dict={data: inp, target: out})# Write logs at every iteration | |
train_writer.add_summary(summaries, epoch * batch_size + i) | |
if not i % 100: | |
print("Epoch ",str(i)) | |
incorrect, summs = sess.run([error, merged],{data: test_input, target: test_output}) | |
test_writer.add_summary(summs) | |
print(sess.run(prediction,{data: [[[1],[0],[0],[1],[1],[0],[1],[1],[1],[0],[1],[0],[0],[1],[1],[0],[1],[1],[1],[0]]]})) | |
print('Epoch {:2d} error {:3.1f}%'.format(i + 1, 100 * incorrect)) | |
sess.close() | |
#Execute | |
# $tensorboard --logdir=logs/ | |
# Open a browser and point to http://localhost:6006/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment