Tensorflow RNN-LSTM implementation to count number of set bits in a binary string

rnn-lstm.py

#Source code with the blog post at http://monik.in/a-noobs-guide-to-implementing-rnn-lstm-using-tensorflow/
#Original gist: https://gist.github.com/AlmostDan/a5f4b57104532a68bda2c274f8bcf16f
#Tensorboard inspirational: https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/4_Utils/tensorboard_basic.py
import numpy as np
#import random
from random import shuffle
import tensorflow as tf

NUM_EXAMPLES = 10000

def variable_summaries(var):
  """Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
  with tf.name_scope('summaries'):
    mean = tf.reduce_mean(var)
    tf.summary.scalar('mean', mean)
    with tf.name_scope('stddev'):
      stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
    tf.summary.scalar('stddev', stddev)
    tf.summary.scalar('max', tf.reduce_max(var))
    tf.summary.scalar('min', tf.reduce_min(var))
    tf.summary.histogram('histogram', var)
# Generate a list of 1.048.576 elements of format: '11111111111110001010', '11111111111110001011'
train_input = ['{0:020b}'.format(i) for i in range(2**20)]
shuffle(train_input)
train_input = [map(int,i) for i in train_input]
ti  = []
for i in train_input:
    temp_list = []
    for j in i:
            temp_list.append([j])
    ti.append(np.array(temp_list))
train_input = ti

train_output = []
for i in train_input:
    count = 0
    for j in i:
        if j[0] == 1:
            count+=1
    temp_list = ([0]*21)
    temp_list[count]=1
    train_output.append(temp_list)

test_input = train_input[NUM_EXAMPLES:]
test_output = train_output[NUM_EXAMPLES:]
train_input = train_input[:NUM_EXAMPLES]
train_output = train_output[:NUM_EXAMPLES]

print("test and training data loaded")

data = tf.placeholder(tf.float32, [None, 20,1]) #Number of examples, number of input, dimension of each input
target = tf.placeholder(tf.float32, [None, 21])
num_hidden = 24

tf.name_scope('Counting1s')
# Define the cell to 'remember' the counted data
cell = tf.contrib.rnn.LSTMCell(num_hidden,state_is_tuple=True)
val, _ = tf.nn.dynamic_rnn(cell, data, dtype=tf.float32)
val = tf.transpose(val, [1, 0, 2])
last = tf.gather(val, int(val.get_shape()[0]) - 1)
with tf.name_scope('weights'):
    weight = tf.Variable(tf.truncated_normal([num_hidden, int(target.get_shape()[1])]))
    variable_summaries(weight)
with tf.name_scope('bias'):
    bias = tf.Variable(tf.constant(0.1, shape=[target.get_shape()[1]]))
    variable_summaries(bias)
with tf.name_scope('prediction'):
    # Let's use softmax, instead of relu and sigmoid, you can play around with those and see the impact.
    prediction = tf.nn.softmax(tf.matmul(last, weight) + bias)
    variable_summaries(prediction)
with tf.name_scope('crossEntropy'):
    cross_entropy = -tf.reduce_sum(target * tf.log(tf.clip_by_value(prediction,1e-10,1.0)))
    tf.summary.scalar('cross_entropy', cross_entropy)
optimizer = tf.train.AdamOptimizer()
minimize = optimizer.minimize(cross_entropy)
mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1))
error = tf.reduce_mean(tf.cast(mistakes, tf.float32))
variable_summaries(error)

sess = tf.Session()

# Merge all the summaries and write them out to logs/
merged = tf.summary.merge_all()

train_writer = tf.summary.FileWriter('logs/train', sess.graph)
test_writer = tf.summary.FileWriter('logs/test', sess.graph)

tf.global_variables_initializer().run(session=sess)

batch_size = 1000
no_of_batches = int(len(train_input) / batch_size)
epoch = 5000

for i in range(epoch):
    ptr = 0
    for j in range(no_of_batches):
        inp, out = train_input[ptr:ptr+batch_size], train_output[ptr:ptr+batch_size]
        ptr += batch_size
        mini, summaries = sess.run([minimize,merged],feed_dict={data: inp, target: out})# Write logs at every iteration
        train_writer.add_summary(summaries, epoch * batch_size + i)
    if not i % 100:
        print("Epoch ",str(i))
incorrect, summs = sess.run([error, merged],{data: test_input, target: test_output})
test_writer.add_summary(summs)
print(sess.run(prediction,{data: [[[1],[0],[0],[1],[1],[0],[1],[1],[1],[0],[1],[0],[0],[1],[1],[0],[1],[1],[1],[0]]]}))
print('Epoch {:2d} error {:3.1f}%'.format(i + 1, 100 * incorrect))
sess.close()
#Execute
# $tensorboard --logdir=logs/
# Open a browser and point to http://localhost:6006/