Neural network implementation with dropout regularization

dropout.py

# WITH dropout
epochs = 400
learning_rate = 0.1
dropout_prob = 0.8
dropout_perf1 = []
for j in range(epochs):
    shuffle(total_train)
    train = total_train[:1000]
    run_costs = []
    predictions = []
    for i in range(len(train)):
        # Feed forward
        x, t = train[i][0], train[i][1]
        z1 = np.dot(Theta1, x) + b1
        a1 = sigmoid(z1)
        mask1 = (np.random.random(len(z1)) < dropout_prob) / dropout_prob
        a1 = a1 * mask1
        
        z2 = np.dot(Theta2, a1) + b2
        a2 = sigmoid(z2)
        mask2 = (np.random.random(len(z2)) < dropout_prob) / dropout_prob
        a2 *= mask2
        
        z3 = np.dot(Theta3, a2) + b3
        y = softmax(z3)
        # No. of misclassifications
        predictions.append(np.argmax(y) != np.argmax(t))

        # Backpropagation
        delta3 = (y - t) * softmax_prime(z3)
        dTheta3 = np.outer(delta3, a2)
        db3 = delta3 
        
        delta2 = np.dot(Theta3.T, delta3) * sigmoid_prime(z2)
        dTheta2 = np.outer(delta2, a1)
        db2 = delta2 
        
        delta1 = np.dot(Theta2.T, delta2) * sigmoid_prime(z1)
        dTheta1 = np.outer(delta1, x)
        db1 = delta1 * 1

        Theta1 -= learning_rate * dTheta1
        b1 -= learning_rate * db1
        Theta2 -= learning_rate * dTheta2
        b2 -= learning_rate * db2
        Theta3 -= learning_rate * dTheta3
        b3 -= learning_rate * db3
    
    test_predictions = []
    shuffle(total_test)
    test = total_test[:1000]
    for i in range(len(test)):
        # Feed forward
        x, t = test[i][0], test[i][1]
        z1 = np.dot(Theta1, x) + b1
        a1 = sigmoid(z1)
        z2 = np.dot(Theta2, a1) + b2
        a2 = sigmoid(z2)
        z3 = np.dot(Theta3, a2) + b3
        y = softmax(z3)
        # No. of misclassifications
        test_predictions.append(np.argmax(y) != np.argmax(t))
    m = len(test_predictions)
    performance = sum(test_predictions)/m
    dropout_perf1.append(performance)