NeuralNetwork.R

# sigmoid function
sigmoid <- function(x)
  1 / (1 + exp(-x) )

# sigmoid derivative
sigmoid_output_to_derivative <- function(x)
  x*(1-x)

# hidden layer size
hidden_dim = 4

# alpha
alpha = 0.1

# dropout percentage (set to zero for no dropout)
dropout_percent = 0.1


# input data
X = matrix(c(0,0,1,
             0,1,1,
             1,0,1,
             1,1,1), nrow=4, byrow=TRUE)

# output data
y = matrix(c(0,
             1,
             1,
             0),
           nrow=4)

set.seed(1)

# initialize weights randomly with mean 0
synapse_0 = matrix(runif(n = 3*hidden_dim, min=-1, max=1), nrow=3)
synapse_1 = matrix(runif(n =   hidden_dim, min=-1, max=1), ncol=1)

for (j in 1:60000) {
  
  # Feed forward through layers 0, 1, and 2
  layer_0 = X
  layer_1 = sigmoid(layer_0 %*% synapse_0)
  layer_1 = layer_1 * matrix(rbinom(n=4*hidden_dim,size=1,prob=1-dropout_percent), nrow=hidden_dim) * ( 1/(1*dropout_percent) )
  layer_2 = sigmoid(layer_1 %*% synapse_1)
  
  # how much did we miss the target value?
  layer_2_error = layer_2 - y
  
  if (j %% 10000 == 0)
    print(paste("Error:", mean(abs(layer_2_error))))
  
  # in what direction is the target value?
  # were we really sure? if so, don't change too much.
  layer_2_delta = layer_2_error * sigmoid_output_to_derivative(layer_2)
  
  # how much did each layer_1 value contribute to the error (according to the weights)?
  layer_1_error = layer_2_delta %*% t(synapse_1)
  
  # in what direction is the target layer_1?
  # were we really sure? if so, don't change too much.
  layer_1_delta = layer_1_error * sigmoid_output_to_derivative(layer_1)
  
  synapse_1 = synapse_1 - alpha * ( t(layer_1) %*% layer_2_delta )
  synapse_0 = synapse_0 - alpha * ( t(layer_0) %*% layer_1_delta )
  
}