RNN.R

# define some functions

## convert integer to binary
i2b <- function(integer, length=8)
  as.numeric(intToBits(integer))[1:length]

## apply 
int2bin <- function(integer, length=8)
  t(sapply(integer, i2b, length=length))

## sigmoid function
sigmoid <- function(x, k=1, x0=0)
  1 / (1+exp( -k*(x-x0) ))

## derivative
sigmoid_output_to_derivative <- function(x)
  x*(1-x)

# create training numbers
X1 = sample(0:127, 10000, replace=TRUE)
X2 = sample(0:127, 10000, replace=TRUE)

# create training response numbers
Y <- X1 + X2

# convert to binary
X1b <- int2bin(X1, length=8)
X2b <- int2bin(X2, length=8)
Yb  <- int2bin(Y,  length=8)

# input variables
alpha = 0.1
input_dim = 2
hidden_dim = 8
output_dim = 1
binary_dim = 8
largest_number = 2^binary_dim


# initialize neural network weights
synapse_0 = matrix(runif(n = input_dim*hidden_dim, min=-1, max=1), nrow=input_dim)
synapse_1 = matrix(runif(n = hidden_dim*output_dim, min=-1, max=1), nrow=hidden_dim)
synapse_h = matrix(runif(n = hidden_dim*hidden_dim, min=-1, max=1), nrow=hidden_dim)

synapse_0_update = matrix(0, nrow = input_dim, ncol = hidden_dim)
synapse_1_update = matrix(0, nrow = hidden_dim, ncol = output_dim)
synapse_h_update = matrix(0, nrow = hidden_dim, ncol = hidden_dim)

# training logic
for (j in 1:length(X1)) {
  
  # select input variables
  a = X1b[j,]
  b = X2b[j,]
  
  # response variable
  c = Yb[j,]
  
  # where we'll store our best guesss (binary encoded)
  d = matrix(0, nrow = 1, ncol = binary_dim)
  
  overallError = 0
  
  layer_2_deltas = matrix(0)
  layer_1_values = matrix(0, nrow=1, ncol = hidden_dim)
  
  # moving along the positions in the binary encoding
  for (position in 1:binary_dim) {
    
    # generate input and output
    X = cbind(a[position],b[position])
    y = c[position]
    
    # hidden layer (input ~+ prev_hidden)
    layer_1 = sigmoid((X%*%synapse_0) + (layer_1_values[dim(layer_1_values)[1],] %*% synapse_h))
    
    # output layer (new binary representation)
    layer_2 = sigmoid(layer_1 %*% synapse_1)
    
    # did we miss?... if so, by how much?
    layer_2_error = y - layer_2
    layer_2_deltas = rbind(layer_2_deltas, layer_2_error * sigmoid_output_to_derivative(layer_2))
    overallError = overallError + abs(layer_2_error)
    
    # decode estimate so we can print it out
    d[position] = round(layer_2)
    
    # store hidden layer so we can print it out
    layer_1_values = rbind(layer_1_values, layer_1)                                                  }
  
  future_layer_1_delta = matrix(0, nrow = 1, ncol = hidden_dim)
  
  for (position in 1:binary_dim) {
    
    X = cbind(a[binary_dim-(position-1)], b[binary_dim-(position-1)])
    layer_1 = layer_1_values[dim(layer_1_values)[1]-(position-1),]
    prev_layer_1 = layer_1_values[dim(layer_1_values)[1]-position,]
    
    # error at output layer
    layer_2_delta = layer_2_deltas[dim(layer_2_deltas)[1]-(position-1),]
    # error at hidden layer
    layer_1_delta = (future_layer_1_delta %*% t(synapse_h) + layer_2_delta %*% t(synapse_1)) * sigmoid_output_to_derivative(layer_1)
    
    # let's update all our weights so we can try again
    synapse_1_update = synapse_1_update + matrix(layer_1) %*% layer_2_delta
    synapse_h_update = synapse_h_update + matrix(prev_layer_1) %*% layer_1_delta
    synapse_0_update = synapse_0_update + t(X) %*% layer_1_delta
    
    future_layer_1_delta = layer_1_delta                             }
  
  
  synapse_0 = synapse_0 + ( synapse_0_update * alpha )
  synapse_1 = synapse_1 + ( synapse_1_update * alpha )
  synapse_h = synapse_h + ( synapse_h_update * alpha )
  
  synapse_0_update = synapse_0_update * 0
  synapse_1_update = synapse_1_update * 0
  synapse_h_update = synapse_h_update * 0
  
  # print out progress
  if(j %% 1000 ==0) {
    print(paste("Error:", overallError))
    print(paste("Pred:", paste(d, collapse = " ")))
    print(paste("True:", paste(c, collapse = " ")))
    out = 0
    for (x in 1:length(d)) {
      out[x] = rev(d)[x]*2^(x-1) }
    print("----------------")                     
  }             
}