tensorflow_gpu_nondeterminism.py

# Based on https://github.com/tensorflow/tensorflow/issues/2652#issue-158487397
import tensorflow as tf
tf.reset_default_graph()

def run(on_gpu):
    tf.reset_default_graph()
    tf.set_random_seed(42) # Try to set up determinism
    with tf.device('/gpu:0' if on_gpu else '/cpu:0'):
        a = tf.random_normal([16, 16])
        b = tf.get_variable('b', shape = [], initializer = tf.constant_initializer(value = 0.0))
        c = a*b

        grad = tf.gradients(c, [b], gate_gradients=tf.train.Optimizer.GATE_GRAPH)[0]

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    grad_val = sess.run(grad)
    return grad_val

print("CPU (deterministic)")
for i in range(4):
    print(repr(run(on_gpu=False)))

print("")
print("GPU (nondeterministic)")
for i in range(4):
    print(repr(run(on_gpu=True)))

Outputs the following:

CPU (deterministic)
23.066511
23.066511
23.066511
23.066511

GPU (nondeterministic)
23.066513
23.066511
23.066509
23.066513```