-
-
Save monikkinom/e97d518fe02a79177b081c028a83ec1c to your computer and use it in GitHub Desktop.
| #Source code with the blog post at http://monik.in/a-noobs-guide-to-implementing-rnn-lstm-using-tensorflow/ | |
| import numpy as np | |
| import random | |
| from random import shuffle | |
| import tensorflow as tf | |
| # from tensorflow.models.rnn import rnn_cell | |
| # from tensorflow.models.rnn import rnn | |
| NUM_EXAMPLES = 10000 | |
| train_input = ['{0:020b}'.format(i) for i in range(2**20)] | |
| shuffle(train_input) | |
| train_input = [map(int,i) for i in train_input] | |
| ti = [] | |
| for i in train_input: | |
| temp_list = [] | |
| for j in i: | |
| temp_list.append([j]) | |
| ti.append(np.array(temp_list)) | |
| train_input = ti | |
| train_output = [] | |
| for i in train_input: | |
| count = 0 | |
| for j in i: | |
| if j[0] == 1: | |
| count+=1 | |
| temp_list = ([0]*21) | |
| temp_list[count]=1 | |
| train_output.append(temp_list) | |
| test_input = train_input[NUM_EXAMPLES:] | |
| test_output = train_output[NUM_EXAMPLES:] | |
| train_input = train_input[:NUM_EXAMPLES] | |
| train_output = train_output[:NUM_EXAMPLES] | |
| print "test and training data loaded" | |
| data = tf.placeholder(tf.float32, [None, 20,1]) #Number of examples, number of input, dimension of each input | |
| target = tf.placeholder(tf.float32, [None, 21]) | |
| num_hidden = 24 | |
| cell = tf.nn.rnn_cell.LSTMCell(num_hidden,state_is_tuple=True) | |
| val, _ = tf.nn.dynamic_rnn(cell, data, dtype=tf.float32) | |
| val = tf.transpose(val, [1, 0, 2]) | |
| last = tf.gather(val, int(val.get_shape()[0]) - 1) | |
| weight = tf.Variable(tf.truncated_normal([num_hidden, int(target.get_shape()[1])])) | |
| bias = tf.Variable(tf.constant(0.1, shape=[target.get_shape()[1]])) | |
| prediction = tf.nn.softmax(tf.matmul(last, weight) + bias) | |
| cross_entropy = -tf.reduce_sum(target * tf.log(tf.clip_by_value(prediction,1e-10,1.0))) | |
| optimizer = tf.train.AdamOptimizer() | |
| minimize = optimizer.minimize(cross_entropy) | |
| mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1)) | |
| error = tf.reduce_mean(tf.cast(mistakes, tf.float32)) | |
| init_op = tf.initialize_all_variables() | |
| sess = tf.Session() | |
| sess.run(init_op) | |
| batch_size = 1000 | |
| no_of_batches = int(len(train_input)) / batch_size | |
| epoch = 5000 | |
| for i in range(epoch): | |
| ptr = 0 | |
| for j in range(no_of_batches): | |
| inp, out = train_input[ptr:ptr+batch_size], train_output[ptr:ptr+batch_size] | |
| ptr+=batch_size | |
| sess.run(minimize,{data: inp, target: out}) | |
| print "Epoch ",str(i) | |
| incorrect = sess.run(error,{data: test_input, target: test_output}) | |
| print sess.run(prediction,{data: [[[1],[0],[0],[1],[1],[0],[1],[1],[1],[0],[1],[0],[0],[1],[1],[0],[1],[1],[1],[0]]]}) | |
| print('Epoch {:2d} error {:3.1f}%'.format(i + 1, 100 * incorrect)) | |
| sess.close() |
The code is not working.
I got the following error: "tensorflow.python.ops.nn' has no attribute 'rnn_cell"
I think It was moved to contrib. Take a look here
This code runs on TensorFlow 0.12
What TensorFlow version are your using guys?
I'm using TensorFlow 1.0.0
on TF 1.0.0, the only thing you need to change is Line 44 which should be this:
cell = tf.contrib.rnn.LSTMCell(num_hidden,state_is_tuple=True)
Also, you probably dont need to run all 5000 epochs, my machine got to 0.1% error in 600.
Working after upgrading to tensorflow 1.0.1 , just change line #62 - no_of_batches = int((len(train_input)) / batch_size)
I modified this code to use a single floating point value as output as ( 0, 1/20, 2/20, ... etc 20/20 )
See here: https://gist.github.com/da-steve101/31693ebfa1b451562810d8644b788900
It trains in 100 epochs and gets an error of 0%
Uses tensorflow 1.1 and python3
Hi!
I ran: print sess.run(prediction,{data: [[[1],[0],[0],[1],[1],[0],[1],[1],[1],[0],[1],[0],[0],[1],[1],[0],[1],[1],[1],[0]]]})
and got the following:
[[ 0.04873509 0.03716513 0.02902525 0.04240027 0.05973569 0.0452175
0.04032514 0.05808202 0.06409416 0.04935085 0.03892809 0.04710475
0.02984658 0.05140518 0.04053102 0.03725993 0.08170271 0.0468277
0.06852488 0.05100909 0.03272888]]
Can you explain what this means?
From what I understand, the last number is the one that's being predicted, and since this is a softmax output, then the last number is a zero. Is that correct?
Thanks!
Also, one more question:
I'm trying this code on my own dataset, which is a list of numbers... In this part:
#unroll the network and pass the data to it and store the output in val
val, state = tf.nn.dynamic_rnn(cell, data, dtype = tf.float32)
#transpose the output to switch batch size with sequence size
val = tf.transpose(val, [1,0,2])
#take the values of outputs only at sequence’s last input
last = tf.gather(val, int(val.get_shape()[0] - 1))
weight = tf.Variable(tf.truncated_normal([num_hidden, int(target.get_shape()[1])]))
bias = tf.Variable(tf.constant(0.1, shape = [target.get_shape()[1]]))
prediction = tf.nn.softmax(tf.matmul(last, weight) + bias)
cross_entropy = -tf.reduce_sum(target * tf.log(tf.clip_by_value(prediction, 1e-10, 1.0)))
optimizer = tf.train.AdamOptimizer()
minimize = optimizer.minimize(cross_entropy)
mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1))
error = tf.reduce_mean(tf.cast(mistakes, tf.float32))
I got an error of:
ValueError Traceback (most recent call last)
in ()
12 cross_entropy = -tf.reduce_sum(target * tf.log(tf.clip_by_value(prediction, 1e-10, 1.0)))
13 optimizer = tf.train.AdamOptimizer()
---> 14 minimize = optimizer.minimize(cross_entropy)
15 mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1))
16 error = tf.reduce_mean(tf.cast(mistakes, tf.float32))
/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/training/optimizer.pyc in minimize(self, loss, global_step, var_list, gate_gradients, aggregation_method, colocate_gradients_with_ops, name, grad_loss)
323
324 return self.apply_gradients(grads_and_vars, global_step=global_step,
--> 325 name=name)
326
327 def compute_gradients(self, loss, var_list=None,/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/training/optimizer.pyc in apply_gradients(self, grads_and_vars, global_step, name)
444 ([str(v) for _, _, v in converted_grads_and_vars],))
445 with ops.control_dependencies(None):
--> 446 self._create_slots([_get_variable_for(v) for v in var_list])
447 update_ops = []
448 with ops.name_scope(name, self._name) as name:/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/training/adam.pyc in _create_slots(self, var_list)
120 # Create slots for the first and second moments.
121 for v in var_list:
--> 122 self._zeros_slot(v, "m", self._name)
123 self._zeros_slot(v, "v", self._name)
124/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/training/optimizer.pyc in _zeros_slot(self, var, slot_name, op_name)
764 named_slots = self._slot_dict(slot_name)
765 if _var_key(var) not in named_slots:
--> 766 named_slots[_var_key(var)] = slot_creator.create_zeros_slot(var, op_name)
767 return named_slots[_var_key(var)]/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.pyc in create_zeros_slot(primary, name, dtype, colocate_with_primary)
172 return create_slot_with_initializer(
173 primary, initializer, slot_shape, dtype, name,
--> 174 colocate_with_primary=colocate_with_primary)
175 else:
176 val = array_ops.zeros(slot_shape, dtype=dtype)/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.pyc in create_slot_with_initializer(primary, initializer, shape, dtype, name, colocate_with_primary)
144 with ops.colocate_with(primary):
145 return _create_slot_var(primary, initializer, "", validate_shape, shape,
--> 146 dtype)
147 else:
148 return _create_slot_var(primary, initializer, "", validate_shape, shape,/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.pyc in _create_slot_var(primary, val, scope, validate_shape, shape, dtype)
64 use_resource=_is_resource(primary),
65 shape=shape, dtype=dtype,
---> 66 validate_shape=validate_shape)
67 variable_scope.get_variable_scope().set_partitioner(current_partitioner)
68/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc in get_variable(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)
1047 collections=collections, caching_device=caching_device,
1048 partitioner=partitioner, validate_shape=validate_shape,
-> 1049 use_resource=use_resource, custom_getter=custom_getter)
1050 get_variable_or_local_docstring = (
1051 """%s/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc in get_variable(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)
946 collections=collections, caching_device=caching_device,
947 partitioner=partitioner, validate_shape=validate_shape,
--> 948 use_resource=use_resource, custom_getter=custom_getter)
949
950 def _get_partitioned_variable(self,/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc in get_variable(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)
354 reuse=reuse, trainable=trainable, collections=collections,
355 caching_device=caching_device, partitioner=partitioner,
--> 356 validate_shape=validate_shape, use_resource=use_resource)
357
358 def _get_partitioned_variable(/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc in _true_getter(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource)
339 trainable=trainable, collections=collections,
340 caching_device=caching_device, validate_shape=validate_shape,
--> 341 use_resource=use_resource)
342
343 if custom_getter is not None:/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc in _get_single_variable(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape, use_resource)
651 " Did you mean to set reuse=True in VarScope? "
652 "Originally defined at:\n\n%s" % (
--> 653 name, "".join(traceback.format_list(tb))))
654 found_var = self._vars[name]
655 if not shape.is_compatible_with(found_var.get_shape()):ValueError: Variable rnn/lstm_cell/weights/Adam/ already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:
File "", line 4, in
minimize = optimizer.minimize(cross_entropy)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2821, in run_ast_nodes
if self.run_code(code, result):
Hope you can shed some light on this. Thanks!
There are small updates in my fork to make it run on TF 0.12 and Python 3.6.
Can you write a Lstm to learn an Sin Wave Please, thanks!
I've added the use of tensorboard and works in TF 1.3r2 and Python 3.6
@franciscogmm The index of the largest value of the output is the guess of how many 1's there are, so if there are three 1's, then the perfect output would be [0,0,0,1 ... ] because there aren't 0 1's, there aren't 1 1's, there aren't 2 1's, but there are 3 1's.
Hi, I tried this with TF1.5, then I got an error for the dimension of the first placeholder
Then I checked the code again and change line62 to no_of_batches = int(len(train_input) / batch_size) and now it works.
@franciscogmm it is the name collision. You need to specify different variable scopes for the LSTM cells.
Hi, i got this error when i run this code .please notify me how can i correct this one.
File "", line 66
print sess.run(prediction,{data: [[[1],[0],[0],[1],[1],[0],[1],[1],[1],[0],[1],[0],[0],[1],[1],[0],[1],[1],[1],[0]]]})
^
SyntaxError: invalid syntax
seriously this code has not been optimized at all. can't even run it