Simple RNN testing stateful mode for forward propagation

SimpleRNN_test.py

from keras.models import Model
from keras.layers import Input
from keras.layers import LSTM, Dense, SimpleRNN
from numpy import array
import numpy as np
import keras


def SimpleRNN_forward(weights, inputs, return_sequence=False):
    kernel = np.array(weights[0])    # kernel weights
    r_kernel = np.array(weights[1])  # recurrent kernel weights
    bias = np.array(weights[2])      # bias weights

    if len(inputs.shape) == 2:  # Only a single input sequence
        seq_len, units = inputs.shape
        h = np.zeros((seq_len + 1, units))
        for t in range(seq_len):
            x = inputs[t]
            h[t] = np.tanh(np.dot(x, kernel) + np.dot(h[t-1], r_kernel) + bias)
        if return_sequence:
            return h[:-1]  # return all hidden layer removing zeros() used for initial h
        else:
            return h[t]   # return last computed h[t]
    else:
        # un-roll batch-size and pass only one sequence at a time
        print("Input shape must be (seq_len, units)")


def Dense_forward(weights, inputs):
    kernel = np.array(weights[0])    # kernel weights
    bias = np.array(weights[1])      # bias weights

    return np.dot(inputs, kernel) + bias


def main():
    np.random.seed(0)
    batch_size = 5
    seq_len = 7
    units = 3
    stateful = True

    inp = Input(batch_shape=(batch_size, seq_len, units))  # batch_size needed for stateful models
    rnn1 = SimpleRNN(units, return_sequences=True, stateful=stateful,
                     bias_initializer='random_uniform',  # default is zero use random to test computation
                     name="rnn1")(inp)
    rnn2 = SimpleRNN(units, return_sequences=False, stateful=stateful,
                     bias_initializer='random_uniform',
                     name="rnn2")(rnn1)
    dense = Dense(units, bias_initializer='random_uniform', name="dense")(rnn2)
    model = Model(inputs=[inp], outputs=[dense])

    # random input data
    shape = (batch_size, seq_len, units)
    in_data = np.random.random_sample(shape)

    # make prediction with Keras
    output = model.predict(in_data)
    print ("\nKeras Ouptut")
    print(output)

    # get weights
    weights = {l.name: l.get_weights() for l in model.layers}

    # Make predictions simulating the forward path with Numpy
    print ("\nNumpy Ouptut")
    for b in range(batch_size):
        inputs = in_data[b]
        output_rnn1 = SimpleRNN_forward(weights["rnn1"], inputs, return_sequence=True)
        output_rnn2 = SimpleRNN_forward(weights["rnn2"], output_rnn1, return_sequence=False)
        output_numpy = Dense_forward(weights["dense"], output_rnn2)
        print (output_numpy)


if __name__ == "__main__":
    main()