Keras masking example

py

import keras
from keras.layers import *
from keras.models import Model
import theano as T
import tensorflow as tf
print('theano ver.',T.__version__) 
print('tensorflow ver.',tf.__version__) 
print('keras ver.',keras.__version__) 
np.set_printoptions(precision=4)
np.random.seed(1)
# had same results with both tf and T
from keras.models import Sequential

def sequential_non_temporal_example():
    model = Sequential()
    model.add(Masking(mask_value = mask_value,input_shape=(n_timesteps, n_features)))
    model.add(Dense(1,activation='linear',kernel_initializer="one"))
    model.summary()

    output_val = model.predict(data)

    print '--> time'
    print np.linspace(0,n_timesteps-1,n_timesteps)
    for s in xrange(n_samples):
        print '# sample = ',s
        print 'input:'
        print(data[s,:,:].T)
        print 'output_val:'
        print(output_val[s,:,:].T)

def sequential_temporal_example():
    model = Sequential()
    model.add(Masking(mask_value = mask_value,input_shape=(n_timesteps, n_features)))
    model.add(LSTM(2, return_sequences=True, kernel_initializer="one"))
    model.add(Dense(1,activation='linear',kernel_initializer="one"))
    
    model.summary()

    output_val = model.predict(data)

    print '--> time'
    print np.linspace(0,n_timesteps-1,n_timesteps)
    for s in xrange(n_samples):
        print '# sample = ',s
import keras
from keras.layers import *
from keras.models import Model
import theano as T
import tensorflow as tf
print('theano ver.',T.__version__) 
print('tensorflow ver.',tf.__version__) 
print('keras ver.',keras.__version__) 
np.set_printoptions(precision=4)
np.random.seed(1)

from keras.models import Sequential

def sequential_non_temporal_example():
    model = Sequential()
    model.add(Masking(mask_value = mask_value,input_shape=(n_timesteps, n_features)))
    model.add(Dense(1,activation='linear',kernel_initializer="one"))
    model.summary()

    output_val = model.predict(data)

    print '--> time'
    print np.linspace(0,n_timesteps-1,n_timesteps)
    for s in xrange(n_samples):
        print '# sample = ',s
        print 'input:'
        print(data[s,:,:].T)
        print 'output_val:'
        print(output_val[s,:,:].T)

def sequential_temporal_example():
    model = Sequential()
    model.add(Masking(mask_value = mask_value,input_shape=(n_timesteps, n_features)))
    model.add(LSTM(2, return_sequences=True, kernel_initializer="one"))
    model.add(Dense(1,activation='linear',kernel_initializer="one"))
    
    model.summary()

    output_val = model.predict(data)

import keras
from keras.layers import *
from keras.models import Model
import theano as T
import tensorflow as tf
print('theano ver.',T.__version__) 
print('tensorflow ver.',tf.__version__) 
print('keras ver.',keras.__version__) 
np.set_printoptions(precision=4)
np.random.seed(1)

from keras.models import Sequential

def sequential_non_temporal_example():
    model = Sequential()
    model.add(Masking(mask_value = mask_value,input_shape=(n_timesteps, n_features)))
    model.add(Dense(1,activation='linear',kernel_initializer="one"))
    model.summary()

    output_val = model.predict(data)

    print '--> time'
    print np.linspace(0,n_timesteps-1,n_timesteps)
    for s in xrange(n_samples):
        print '# sample = ',s
        print 'input:'
        print(data[s,:,:].T)
        print 'output_val:'
        print(output_val[s,:,:].T)

def sequential_temporal_example():
    model = Sequential()
    model.add(Masking(mask_value = mask_value,input_shape=(n_timesteps, n_features)))
    model.add(LSTM(2, return_sequences=True, kernel_initializer="one"))
    model.add(Dense(1,activation='linear',kernel_initializer="one"))
    
    model.summary()

    output_val = model.predict(data)

    print '--> time'
    print np.linspace(0,n_timesteps-1,n_timesteps)
    for s in xrange(n_samples):
        print '# sample = ',s
        print 'input:'
        print(data[s,:,:].T)
        print 'output_val:'
        print(output_val[s,:,:].T)
    
    
n_samples  = 3
n_timesteps = 7
n_features = 2
mask_value = np.nan # -999999999.0# -1.0 # -1 # 0.0
data = np.ones((n_samples,n_timesteps,n_features))

for s in xrange(n_samples):
    for f in xrange(n_features):
        data[s,:,f] = np.linspace(1,n_timesteps,n_timesteps)

# mask a feature value of one sample and timestep (no effect)
data[1,0,0] = mask_value

# mask all feature values of one sample and timestep (propagates 0.*mask_value at layer of step/sample?)
data[2,3,:] = mask_value

print '####################### sequential_non_temporal_example #######################:'
sequential_non_temporal_example()
print '####################### sequential_temporal_example #######################:'
# As non-temporal but masked timestep state does not propagate through time:
sequential_temporal_example()

# ('theano ver.', '0.8.2')
# ('tensorflow ver.', '1.0.1')
# ('keras ver.', '2.0.1')
# ####################### sequential_non_temporal_example #######################:
# _________________________________________________________________
# Layer (type)                 Output Shape              Param #   
# =================================================================
# masking_1 (Masking)          (None, 7, 2)              0         
# _________________________________________________________________
# dense_1 (Dense)              (None, 7, 1)              3         
# =================================================================
# Total params: 3.0
# Trainable params: 3.0
# Non-trainable params: 0.0
# _________________________________________________________________
# --> time
# [ 0.  1.  2.  3.  4.  5.  6.]
# # sample =  0
# input:
# [[ 1.  2.  3.  4.  5.  6.  7.]
#  [ 1.  2.  3.  4.  5.  6.  7.]]
# output_val:
# [[  2.   4.   6.   8.  10.  12.  14.]]
# # sample =  1
# input:
# [[ nan   2.   3.   4.   5.   6.   7.]
#  [  1.   2.   3.   4.   5.   6.   7.]]
# output_val:
# [[ nan   4.   6.   8.  10.  12.  14.]]
# # sample =  2
# input:
# [[  1.   2.   3.  nan   5.   6.   7.]
#  [  1.   2.   3.  nan   5.   6.   7.]]
# output_val:
# [[  2.   4.   6.  nan  10.  12.  14.]]
# ####################### sequential_temporal_example #######################:
# _________________________________________________________________
# Layer (type)                 Output Shape              Param #   
# =================================================================
# masking_2 (Masking)          (None, 7, 2)              0         
# _________________________________________________________________
# lstm_1 (LSTM)                (None, 7, 2)              40        
# _________________________________________________________________
# dense_2 (Dense)              (None, 7, 1)              3         
# =================================================================
# Total params: 43.0
# Trainable params: 43.0
# Non-trainable params: 0.0
# _________________________________________________________________
# --> time
# [ 0.  1.  2.  3.  4.  5.  6.]
# # sample =  0
# input:
# [[ 1.  2.  3.  4.  5.  6.  7.]
#  [ 1.  2.  3.  4.  5.  6.  7.]]
# output_val:
# [[ 1.2603  1.9066  1.9871  1.9982  1.9998  2.      2.    ]]
# # sample =  1
# input:
# [[ nan   2.   3.   4.   5.   6.   7.]
#  [  1.   2.   3.   4.   5.   6.   7.]]
# output_val:
# [[ nan  nan  nan  nan  nan  nan  nan]]
# # sample =  2
# input:
# [[  1.   2.   3.  nan   5.   6.   7.]
#  [  1.   2.   3.  nan   5.   6.   7.]]
# output_val:
# [[ 1.2603  1.9066  1.9871     nan     nan     nan     nan]]