Created
October 26, 2017 08:19
-
-
Save rnd-forests/f28d6d25026449c48c646788e96ac782 to your computer and use it in GitHub Desktop.
Gradient Clipping
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tensorflow as tf | |
from tensorflow.examples.tutorials.mnist import input_data | |
n_inputs = 784 | |
n_hidden1 = 300 | |
n_hidden2 = 50 | |
n_hidden3 = 50 | |
n_hidden4 = 50 | |
n_hidden5 = 50 | |
n_outputs = 10 | |
X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X") | |
y = tf.placeholder(tf.int64, shape=(None), name="y") | |
with tf.name_scope("dnn"): | |
hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name="hidden1") | |
hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name="hidden2") | |
hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name="hidden3") | |
hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name="hidden4") | |
hidden5 = tf.layers.dense(hidden4, n_hidden5, activation=tf.nn.relu, name="hidden5") | |
logits = tf.layers.dense(hidden5, n_outputs, name="outputs") | |
with tf.name_scope("loss"): | |
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits) | |
loss = tf.reduce_mean(xentropy, name="loss") | |
with tf.name_scope("evaluation"): | |
correct = tf.nn.in_top_k(logits, y, 1) | |
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name="accuracy") | |
threshold = 1.0 | |
learning_rate = 0.01 | |
with tf.name_scope("train"): | |
optimizer = tf.train.GradientDescentOptimizer(learning_rate) | |
grads_and_vars = optimizer.compute_gradients(loss) | |
capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var) for grad, var in grads_and_vars] | |
training_op = optimizer.apply_gradients(capped_gvs) | |
init = tf.global_variables_initializer() | |
n_epochs = 20 | |
batch_size = 200 | |
mnist = input_data.read_data_sets("/tmp/data/") | |
with tf.Session() as sess: | |
init.run() | |
for epoch in range(n_epochs): | |
for iteration in range(mnist.train.num_examples // batch_size): | |
X_batch, y_batch = mnist.train.next_batch(batch_size) | |
sess.run(training_op, feed_dict={X: X_batch, y: y_batch}) | |
accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels}) | |
print(epoch, "Test accuracy:", accuracy_val) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment