relu_pytorch

from IPython.display import clear_output
from torch.autograd import Variable
w = torch.tensor(np.random.rand(10).astype('float32')/10, requires_grad=True)
x = torch.tensor(boston.data[:, -1] / 10, dtype=torch.float32)
y = torch.tensor(boston.target, dtype=torch.float32)

relu = lambda x: torch.max(torch.zeros_like(x), x )

for i in range(1000):

    y_pred = w[0] + \
            w[1] + relu(w[2] * x + w[3]) + \
            w[4] + relu(w[5] * x + w[6]) + \
            w[7] + relu(w[8] * x + w[9])
    loss = torch.mean((y_pred - y)**2)
    loss.backward()

    w.data -= 0.05 * w.grad.data
    b.data -= 0.05 * b.grad.data

    # zero gradients
    w.grad.data.zero_()
    b.grad.data.zero_()

    # the rest of code is just bells and whistles
    if (i+1) % 5 == 0:
        clear_output(True)
        plt.scatter(x.data.numpy(), y.data.numpy())
        plt.scatter(x.data.numpy(), y_pred.data.numpy(),
                    color='orange', linewidth=5)
        plt.show()

        print("loss = ", loss.data.numpy())
        if loss.data.numpy() < 0.5:
            print("Done!")
            break