ways to do gradients clipping and learning rate decay in tensorflow

tf_gradient_clip_lr_decay.py

import tensorflow as tf

#aplly exponential decay on learning rate
global_step = tf.Variable(0, trainable=False)
stater_learning_rate = lr #for start
learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 
                                           decay_steps, decay_rate, staircase=True)

optimizer = tf.train.AdamOptimizer(learning_rate)

#no clipping
train_op = optimizer.minimize(loss, global_step=global_step)

#global norm clipping.
grad_vars = optimizer.compute_gradients(loss)
grad = [x[0] for x in grad_vars]
vars = [x[1] for x in grad_vars]
grad, grad_norm = tf.clip_by_global_norm(grad, max_grad_norm)
train_op = optimizer.apply_gradients(zip(grad, vars), global_step=global_step)

#clip by value
clipped_gvs = [(tf.clip_by_value(grad, min_val, max_val), var) for grad, var in grad_vars]
train_op = optimizer.apply_gradients(clipped_gvs, global_step=global_step)