test case that demonstrates that metrics are computed incorrectly for masked sequences in sequence-to-sequence model

masking_metrics

import keras.preprocessing.sequence
from keras.models import Model
from keras.layers import Input, Embedding, Recurrent, Masking, GRU, TimeDistributed, Dense
import numpy as np

np.random.seed(0)

# create model
input_layer = Input(shape=(3,), dtype='int32', name='input')
embeddings = Embedding(input_dim=20, output_dim=2, input_length=3, mask_zero=True, name='embeddings')(input_layer)
recurrent = GRU(5, return_sequences=True, name='GRU')(embeddings)
output_layer = TimeDistributed(Dense(1), name='output')(recurrent)
model = Model(input=input_layer, output=output_layer)
model.compile(loss='mse', metrics=['mse'], optimizer='adam', sample_weight_mode='temporal')

# create model inputs
X = [[1], [2, 3]]
X = [[1, 2]]
X_padded = keras.preprocessing.sequence.pad_sequences(X, dtype='float32', maxlen=3)
Y = [[[1]], [[2], [3]]]
Y = [[[1], [2]]]
Y_padded = keras.preprocessing.sequence.pad_sequences(Y, maxlen=3, dtype='float32')

# use model to generate predictions/evaluate
model_predictions = model.predict(X_padded)
model_eval = model.evaluate(X_padded, Y_padded)

# print results
print("Keras computations:")
print('\t'.join(['%s: %s' % (name, val) for (name, val) in zip(model.metrics_names, model_eval)]))

print("\nManually computed by using np.mean:")
print("mse not taking into account padding:", np.mean(np.square(Y_padded - model_predictions)))
print("mse taking into account padding", np.mean(np.square(Y_padded - model_predictions)[0][1:]))