XOR_single_neuron.py

import numpy as np
import matplotlib.pyplot as plt

import tensorflow as tf
from tensorflow.keras.layers import Dense, Activation, Input
from tensorflow.keras import Model
from tensorflow.keras.optimizers import SGD

# Implement activation function
def dCaAP_activation(x):
    return 4*tf.maximum(0.0, tf.cast(sigmoid_derivative(x), 'float32')) * tf.cast((x > 0), 'float32')
    
def sigmoid_derivative(x):
    return tf.sigmoid(x) * (1 - tf.sigmoid(x))

# Create model
def create_model(random_init):
    input_layer = Input(shape=(2,))
    if random_init:
      out = Dense(units=1)(input_layer)
    else:
      out = Dense(units=1, kernel_initializer=tf.keras.initializers.Constant(1))(input_layer)
    out = Activation(dCaAP_activation)(out)
    model = Model(inputs=input_layer, outputs=out)
    model.compile(SGD(lr=0.1), loss='binary_crossentropy', metrics=['accuracy'])
    return model
  
# Plot dCaAP activation function
xplot = np.linspace(-10, 10, 100)
yplot = dCaAP_activation(xplot).numpy()
plt.plot(xplot, yplot)
plt.show()

# Create input output for XOR problem
X = np.array([[0,0],[0,1],[1,0],[1,1]])
y = np.array([[0],[1],[1],[0]])

# Run model for weights initialized to 1, solves XOR roughly 5/10 times
for ii in range(0, 10):
    model = create_model(random_init=False)
    model_weights = model.get_weights()
    print('------------------------------------------------------------------')
    print('Running init {},  w_1 = {:.3f}, w_2 = {:.3f}, b = {:.3f}'.format(ii, model_weights[0][0,0], model_weights[0][1,0], model_weights[1][0]))
    model.fit(X, y, epochs=1000, batch_size=1, verbose=0)
    new_model_weights = model.get_weights()
    model_prediction = model.predict(X)
    print('Trained weights, w_1 = {:.3f}, w_2 = {:.3f}, b = {:.3f}'.format(new_model_weights[0][0,0], new_model_weights[0][1,0], new_model_weights[1][0]))
    print('Model Predictions: [{:.3f},{:.3f},{:.3f},{:.3f}] ---> XOR Problem Solved: {}'.format(model_prediction[0,0], model_prediction[1,0],
                                                                                                model_prediction[2,0], model_prediction[3,0], all(y==(model_prediction > 0.5)))) # np.set_printoptions(precision=3)
 
# Run model for random inits, solves XOR roughly 1/10 times
for ii in range(0, 10):
    model = create_model(random_init=True)
    model_weights = model.get_weights()
    print('------------------------------------------------------------------')
    print('Running init {},  w_1 = {:.3f}, w_2 = {:.3f}, b = {:.3f}'.format(ii, model_weights[0][0,0], model_weights[0][1,0], model_weights[1][0]))
    model.fit(X, y, epochs=1000, batch_size=1, verbose=0)
    new_model_weights = model.get_weights()
    model_prediction = model.predict(X)
    print('Trained weights, w_1 = {:.3f}, w_2 = {:.3f}, b = {:.3f}'.format(new_model_weights[0][0,0], new_model_weights[0][1,0], new_model_weights[1][0]))
    print('Model Predictions: [{:.3f},{:.3f},{:.3f},{:.3f}] ---> XOR Problem Solved: {}'.format(model_prediction[0,0], model_prediction[1,0],
                                                                                                model_prediction[2,0], model_prediction[3,0], all(y==(model_prediction > 0.5)))) # np.set_printoptions(precision=3)

Colab:

Activation Function:

Example Output:

Running init 0,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 4.745, w_2 = 4.745, b = -4.703
Model Predictions: [0.000,1.000,1.000,0.033] ---> XOR Problem Solved: True
------------------------------------------------------------------
Running init 1,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 4.744, w_2 = 4.745, b = -4.703
Model Predictions: [0.000,1.000,1.000,0.033] ---> XOR Problem Solved: True
------------------------------------------------------------------
Running init 2,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 0.046, w_2 = 0.044, b = 1.758
Model Predictions: [0.502,0.486,0.485,0.470] ---> XOR Problem Solved: False
------------------------------------------------------------------
Running init 3,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 4.746, w_2 = 4.745, b = -4.702
Model Predictions: [0.000,1.000,1.000,0.033] ---> XOR Problem Solved: True
------------------------------------------------------------------
Running init 4,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 4.745, w_2 = 4.745, b = -4.702
Model Predictions: [0.000,1.000,1.000,0.033] ---> XOR Problem Solved: True
------------------------------------------------------------------
Running init 5,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 4.746, w_2 = 4.745, b = -4.702
Model Predictions: [0.000,1.000,1.000,0.033] ---> XOR Problem Solved: True
------------------------------------------------------------------
Running init 6,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 0.043, w_2 = 0.040, b = 1.779
Model Predictions: [0.494,0.480,0.479,0.465] ---> XOR Problem Solved: False
------------------------------------------------------------------
Running init 7,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 4.745, w_2 = 4.745, b = -4.703
Model Predictions: [0.000,1.000,1.000,0.033] ---> XOR Problem Solved: True
------------------------------------------------------------------
Running init 8,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 1.055, w_2 = 1.005, b = 41.573
Model Predictions: [0.000,0.000,0.000,0.000] ---> XOR Problem Solved: False
------------------------------------------------------------------
Running init 9,  w_1 = 1.000, w_2 = 1.000, b = 0.000
Trained weights, w_1 = 0.016, w_2 = 0.028, b = 1.747
Model Predictions: [0.506,0.496,0.500,0.490] ---> XOR Problem Solved: False