avito_deep_learning

3d_images_match.py

"""
train on seq_of_iamges vs seq_of_images -> {0,1} sigmoid or contrastive_loss
"""


# model.load_weights("checkpoint_model.h5")
def euclidean_distance(vects):
    x, y = vects
    return K.sqrt(K.sum(K.square(x - y), axis=1, keepdims=True))


def eucl_dist_output_shape(shapes):
    shape1, shape2 = shapes
    return (shape1[0], 1)


def contrastive_loss(y_true, y_pred):
    '''Contrastive loss from Hadsell-et-al.'06
    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
    '''
    margin = 2
    y_pred = K.clip(y_pred, 0, 10000.0)
    return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))

import theano.tensor as T

def model_img_transform():
    model = Sequential()
    model.add(Lambda(lambda x: T.cast(x,"float32") / 256.0, output_shape=lambda x:x,input_shape=input_shape))
    nb_filters = 32
    model.add(Convolution3D(nb_filters, 1,3, 3, activation="relu", input_shape=input_shape))        
#     model.add(Convolution3D(nb_filters*2, 1,3, 3, activation="relu"))           
#     model.add(Convolution3D(nb_filters*4, 1,3, 3, activation="relu"))           
    model.add(MaxPooling3D(pool_size=(5,3, 3)))
    model.add(Dropout(0.5)) 
    model.add(Flatten())
    model.add(Dense(256*2, activation="relu"))
#     model.add(BatchNormalization())
    return model
in1 = Input(shape = input_shape)
in2 = Input(shape = input_shape)
m = model_img_transform()
print "model"
# m.summary()

m1 = m(in1)
m2 = m(in2)

mode = "dist"
if mode == "concat":
    res = merge([m1,m2], mode="concat")
elif mode == "mul":
    res = merge([m1,m2], mode="mul")
elif mode == "cos":
    res = merge([m1, m2], mode='cos', dot_axes=1) # magic dot_axes works here!
    res = Reshape((1,))(res)
elif mode == "dot":
    res = merge([m1, m2], mode='dot', dot_axes=1) # magic dot_axes works here!
    res = Reshape((1,))(res)
elif mode =="dist":
    res = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([m1, m2])
else:
    res = None

# res = Dense(256, activation="relu")(res)

if mode != "dist":    
    res = Dropout(0.5)(res)
    res = Dense(1, activation="sigmoid")(res)
    loss = "binary_crossentropy"
else:
    loss = contrastive_loss
    
model = Model(input=[in1,in2], output=res) #
opt = Adam() #"adam"
# opt= RMSprop()
model.compile(optimizer=opt,
              loss=loss)
# model.summary()

callbacks = [
    EarlyStopping(patience=4, verbose=2),
    ModelCheckpoint("checkpoint_model.h5",verbose=2,save_best_only=True),
#     MyLearningRateScheduler()
]

print "len(y) = ",len(y)
model.fit(X,y,batch_size=112, nb_epoch=30, verbose=2, validation_data=(X_test,y_test), 
          callbacks=callbacks, class_weight={0:1,1:pos_weight}) 

lstm_text_match.py

from keras.models import Model,Sequential
from keras.optimizers import *
#pretrained with gensim
init_weights = word2vec.syn0norm.copy()
print "wrod vectors = ",init_weights.shape
hiddden_dims = init_weights.shape[1]

def build_model():
  
    emb = Embedding(init_weights.shape[0],hiddden_dims, input_length = None, mask_zero=False, input_shape=(seq_len,), 
                    weights=[init_weights], dropout=0.3) 
#     emb.trainable = False
    def max_1d(X):
        return K.max(X, axis=1)
    
    def create_transform():
        m = Sequential()
        nb_filteres = 64*2      
        m.add(Convolution1D(nb_filteres, 3,activation="relu", input_dim=hiddden_dims))
        m.add(Convolution1D(nb_filteres, 3,activation="relu"))
        m.add(MaxPooling1D(pool_length=2))
        
        m.add(Dropout(0.3))
        m.add(Convolution1D(nb_filteres*2, 3,activation="relu"))
        m.add(Convolution1D(nb_filteres*2, 3,activation="relu"))   
        m.add(MaxPooling1D(pool_length=2))
        
        #try global max pooling for match faster results
        m.add(GRU(64*2, consume_less="gpu", return_sequences=False))
        m.summary()
        return m
    
    #GRUx2 slower, but slighlty better results than with convolution
    def create_transform():
        m = Sequential()
        nb_filteres = 256      
        m.add(GRU(nb_filteres, consume_less="gpu", return_sequences=True, input_dim=hiddden_dims))
        m.add(Dropout(0.2))
        m.add(GRU(nb_filteres, consume_less="gpu", return_sequences=False))
        m.summary()
        return m
    
    rnn_descr = create_transform() #GRU(64*4, unroll=False, consume_less="gpu", return_sequences=False)
    rnn_title = create_transform() #GRU(64*4, unroll=False, consume_less="gpu", return_sequences=False)
    def input_pipe(x, rnn):
        x = emb(x)
#         x = Dropout(0.5)(x)
        x = rnn(x)
        x = Dropout(0.5)(x)
        return x

    in1 = Input(shape=(seq_len,),dtype="int32")
    in2 = Input(shape=(seq_len,),dtype="int32")
    in3 = Input(shape=(seq_len,),dtype="int32")
    in4 = Input(shape=(seq_len,),dtype="int32")
    #
    inputs = [input_pipe(in1,rnn_title),input_pipe(in2,rnn_title),input_pipe(in3,rnn_descr),input_pipe(in4,rnn_descr)]
#     res1 = merge(inputs[0:2], mode="mul", dot_axes=1)
#     res2 = merge(inputs[2:4], mode="mul", dot_axes=1)
    

# res = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([processed_a, processed_b])    
    res = merge(inputs, mode="concat")
    

 
    res = Dropout(0.5)(res)
    res = Dense(64,activation="relu")(res)
    res = Dropout(0.5)(res)
    res = Dense(1,activation="sigmoid")(res)

    model = Model(input=[in1,in2,in3,in4], output=res) #
    opt = RMSprop(lr=1e-5, rho=0.9, epsilon=1e-08)
    opt = Adam(lr=1e-4,clipnorm=0.5)
    model.summary()   
    model.compile(optimizer=opt,
                  loss='binary_crossentropy', #'',
                  metrics=['accuracy'])

    return model