Use tf.gradient and tf.assign to manually update the weights (variables)

tf.gradient.usage.py

# Author: Costa Huang (costa.huang@outlook.com)
# References include
# https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/linear_regression.py
# https://stackoverflow.com/a/42861919/6611317
import tensorflow as tf
import numpy
import matplotlib.pyplot as plt
tf.reset_default_graph()

# Parameters
learning_rate = 0.01
training_epochs = 1000
display_step = 50
numpy.random.seed(0)

# Training Data
train_X = numpy.asarray([3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167,
                         7.042,10.791,5.313,7.997,5.654,9.27,3.1])
train_Y = numpy.asarray([1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,
                         2.827,3.465,1.65,2.904,2.42,2.94,1.3])
n_samples = train_X.shape[0]

# tf Graph Input
X = tf.placeholder("float")
Y = tf.placeholder("float")

# Set model weights
W = tf.Variable(numpy.random.randn(), name="weight")
b = tf.Variable(numpy.random.randn(), name="bias")

# Construct a linear model
pred = tf.add(tf.multiply(X, W), b)

# Mean squared error
cost = tf.reduce_sum(tf.pow(pred-Y, 2))/(2*n_samples)

# Begin of manual gradient descent. The following code is equivalent to 
# train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
def get_dependent_varialbes(tensor):
    import collections
    op_to_var = {var.op: var for var in tf.trainable_variables()}
    dependent_vars = []
    queue = collections.deque()
    queue.append(tensor.op)
    visited = set([tensor.op])
    while queue:
        op = queue.popleft()
        try:
            dependent_vars.append(op_to_var[op])
        except KeyError:
            # `op` is not a variable, so search its inputs (if any). 
            for op_input in op.inputs:
                if op_input.op not in visited:
                    queue.append(op_input.op)
                    visited.add(op_input.op)
    return dependent_vars

cost_related_vars = get_dependent_varialbes(cost)
grads = tf.gradients(cost, cost_related_vars)
vars_and_grads = list(zip(cost_related_vars, grads))
ops = []
for item in vars_and_grads:
    ops.append(tf.assign(item[0], item[0] - learning_rate * item[1]))
train_op = tf.group(*ops)
# End of manual gradient descent


# Start training
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(training_epochs):
        for (x, y) in zip(train_X, train_Y):
            sess.run(train_op, feed_dict={X: x, Y: y})

    print("Optimization Finished!")
    training_cost = sess.run(cost, feed_dict={X: train_X, Y: train_Y})
    print("Training cost=", training_cost, "W=", sess.run(W), "b=", sess.run(b), '\n')

Try the script by Google Colab here: https://colab.research.google.com/drive/1EkKNH6CqmkvQUsS-VeSgI4Vh8tFmlpvE