lstm_keras.py

import numpy as np
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv1D, Conv2D, MaxPooling2D, MaxPooling1D, AveragePooling2D, GRU
from keras.layers import TimeDistributed, Bidirectional
from keras.layers.recurrent import LSTM
from keras.models import Sequential

def sigmoid(z):
    s = 1.0 / (1.0 + np.exp(-1.0 * z))
    return s

def relu(z):
    z[z < 0] = 0
    return z


def lstm_impl(input, units, W, U, b, activation, recurrent_activation):
    h_tm1 = np.zeros((1, units), dtype=np.float32)  # previous memory state
    c_tm1 = np.zeros((1, units), dtype=np.float32) # previous carry state

    # allocation
    result = np.zeros(input.shape[:-1] + (units,), dtype=np.float32)

    X = np.dot(input, W) + b

    x_i = X[:, :, :units]
    x_f = X[:, :, units: units * 2]
    x_c = X[:, :, units * 2: units * 3]
    x_o = X[:, :, units * 3:]

    for k in range(input.shape[1]):
        ifco = np.dot(h_tm1, U)

        i = recurrent_activation(x_i[:,k,:] + ifco[:,:units])
        f = recurrent_activation(x_f[:,k,:] + ifco[:,units: units * 2])
        c = f * c_tm1 + i * activation(x_c[:,k,:] + ifco[:,units * 2: units * 3])
        o = recurrent_activation(x_o[:,k,:] + ifco[:,units * 3:])

        h = o * activation(c)

        c_tm1 = c[:]
        h_tm1 = h[:]

        result[:,k,:] = h[np.newaxis,:,:]

    return result


if __name__ == "__main__":
    lstm_model = Sequential()

    input_shape = (2,2)

    activation = np.tanh
    recurrent_activation = sigmoid

    # input layer
    np.random.seed(7)
    lstm_model.add(LSTM(units=2, input_shape=input_shape, activation='tanh', recurrent_activation='sigmoid',
                        return_sequences=True))
    # output layer
    lstm_model.compile(optimizer='adam', loss='binary_crossentropy')

    lstm_model.summary()

    input = np.ones((1,2,2))

    result1 = lstm_model.predict(input)

    units = int(int(lstm_model.layers[0].trainable_weights[0].shape[1]) / 4)
    print("Number of units: ", units)

    W = lstm_model.layers[0].get_weights()[0]
    U = lstm_model.layers[0].get_weights()[1]
    b = lstm_model.layers[0].get_weights()[2]

    result2 = lstm_impl(input, units, W, U, b, activation, recurrent_activation)

    print('Result all_close: ' + str(np.allclose(result1, result2, rtol=1e-06)))