TensorFlow using MNIST. Example of how to implement SGD and fully-connected layers.

mnist.py

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)


def fullyConnected(x, name, input_size, output_size=64):
    with tf.variable_scope(name):
        kernel = tf.get_variable(name="kernel", 
            initializer=tf.random_normal(shape=(input_size, output_size), stddev=0.1))
        bias = tf.get_variable(name="bias", 
            initializer=tf.zeros((output_size)))

    return tf.matmul(x, kernel) + bias

def minimize(loss, learning_rate):
    # get all registered weights
    vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    # here is where the magic happends
    grads = tf.gradients(loss, vars)
    # apply stochastic gradient descent update
    return sgd(vars, grads, learning_rate)

def sgd(vars, grads, learning_rate):
    assert(len(vars) == len(grads), "Variable and gradient array must have same length. ")
    update_ops = []
    for index in range(0, len(vars)):
        v = vars[index]
        g = grads[index]
        # note: we are still building a graph!
        update_op = v.assign_add(-learning_rate * g)
        update_ops.append(update_op)

    return tf.group(update_ops)

# mean squared error
def mse(y, label):
    return 0.5 * tf.reduce_mean( tf.square(label - y) )

def cross_entropy(y, label):
    return -tf.reduce_mean( tf.reduce_sum(label * tf.log(y), 1) )

session = tf.Session()

input = tf.placeholder(tf.float32, shape=(None, 784))
# layer 1
x = fullyConnected(input, "fc1", input_size=784, output_size=128)
x = tf.nn.relu(x)
# layer 2
x = fullyConnected(x, "fc2", input_size=128, output_size=128)
x = tf.nn.relu(x)
# layer 3
x = fullyConnected(x, "fc3", input_size=128, output_size=10)
y = tf.nn.softmax(x)

label = tf.placeholder(tf.float32, (None, 10))
loss = cross_entropy(y, label)
train_op = minimize(loss, 0.1)

session.run(tf.global_variables_initializer())

for _ in range(0, 10000):
    batch_xs, batch_ys = mnist.train.next_batch(64)
    _, current_loss = session.run([train_op, loss], feed_dict={ 
        input: batch_xs,
        label: batch_ys 
    })
    print("Loss: %f" % (current_loss))


batch_xs, batch_ys = mnist.train.next_batch(1)

_y = session.run([y], feed_dict={ input: batch_xs })

print("Model prediction: ")
print(_y)

print("Actual label: ")
print(batch_ys)

session.close()