end to end implementation of oarriaga/neural_image_captioning

nic.py

from keras.applications.inception_v3 import InceptionV3
from keras.models import Model
from keras.layers import Input, Dropout, TimeDistributed, Masking, Dense, RepeatVector
from keras.layers.merge import Add
from keras.layers.recurrent import LSTM, GRU
from keras.regularizers import l2


def NIC(max_caption_len, vocab_size, h, w, rnn='lstm', num_image_features=2048,
        hidden_size=512, embedding_size=512, regularizer=1e-8, **kwargs):
    # word embedding
    max_caption_len = max_caption_len + 2
    text_input = Input(shape=(max_caption_len, vocab_size), name='text')
    text_mask = Masking(mask_value=0.0, name='text_mask')(text_input)
    text_to_embedding = TimeDistributed(Dense(units=embedding_size,
                                        kernel_regularizer=l2(regularizer),
                                        name='text_embedding'))(text_mask)

    text_dropout = Dropout(.5, name='text_dropout')(text_to_embedding)

    # image embedding
    image_input = Input(shape=(h, w, 3), name='image')
    base_model = InceptionV3(weights='imagenet', input_tensor=image_input)
    model = Model(inputs=image_input,
                  outputs=base_model.get_layer('avg_pool').output)
    model_output = model.output
    # image_input = Input(shape=(max_caption_len, num_image_features),
    #                                                     name='image')
    image_embedding = RepeatVector(max_caption_len)(model_output)
    image_embedding = TimeDistributed(Dense(units=embedding_size,
                                        kernel_regularizer=l2(regularizer),
                                        name='image_embedding'))(image_embedding)
    image_dropout = Dropout(.5,name='image_dropout')(image_embedding)

    # language model
    recurrent_inputs = [text_dropout, image_dropout]
    merged_input = Add()(recurrent_inputs)
    if rnn == 'lstm':
        recurrent_network = LSTM(
            units=hidden_size,
            recurrent_regularizer=l2(regularizer),
            kernel_regularizer=l2(regularizer),
            bias_regularizer=l2(regularizer),
            return_sequences=True,
            name='recurrent_network')(merged_input)

    elif rnn == 'gru':
        recurrent_network = GRU(
            units=hidden_size,
            recurrent_regularizer=l2(regularizer),
            kernel_regularizer=l2(regularizer),
            bias_regularizer=l2(regularizer),
            return_sequences=True,
            name='recurrent_network')(merged_input)
    else:
        raise ValueError('Invalid rnn name')

    inputs = [text_input, image_input]
    output = TimeDistributed(Dense(
        units=vocab_size,
        kernel_regularizer=l2(regularizer),
        activation='softmax'), name='output')(recurrent_network)

    model = Model(inputs=inputs, outputs=output)
    return model