crf_combined_model.py

import traceback

import numpy as np
from keras.layers import Input, Dense, LSTM, Bidirectional, TimeDistributed
from keras.models import Model, Sequential
from keras_contrib.layers import CRF


def make_data(n, sequence_length, embedding_size, output_size):
    X = np.zeros((n, sequence_length, embedding_size))
    y = np.zeros((n, sequence_length, 1))
    for i in range(n):
        X[i] = np.random.normal(size=(sequence_length, embedding_size))
        y[i] = np.random.randint(1, output_size, size=(sequence_length, 1))

    return X, y


def build_representer(input_size, representation_size):
    model = Sequential()
    model.add(Dense(representation_size, input_shape=(None, input_size), activation='tanh'))
    model.add(Dense(representation_size, activation='tanh'))
    return model


def build_classifier(representation_size, lstm_size, output_size, use_crf):
    model = Sequential()
    model.add(LSTM(lstm_size, return_sequences=True, input_shape=(None, representation_size)))
    if use_crf:
        crf = CRF(output_size, sparse_target=True)
        model.add(crf)
        return model, crf.loss_function
    else:
        model.add(TimeDistributed(Dense(output_size, activation='softmax')))
        return model, 'sparse_categorical_crossentropy'


def build_combined_model(input_size, representation_size, lstm_size, output_size, use_crf):
    representer_input = Input(shape=(None, input_size))
    representer = build_representer(input_size, representation_size)
    representation = representer(representer_input)

    classifier, loss = build_classifier(representation_size, lstm_size, output_size, use_crf)
    combined = Model(representer_input, classifier(representation), name='Combined Model')
    combined.compile('adam', loss=loss)
    return combined


def build_single_model(input_size, representation_size, lstm_size, output_size, use_crf):
    model = Sequential(name='Single Model')
    # representation model
    model.add(Dense(representation_size, input_shape=(None, input_size), activation='tanh'))

    # classifier model
    model.add(LSTM(lstm_size, return_sequences=True))
    if use_crf:
        crf = CRF(output_size, sparse_target=True)
        model.add(crf)
        loss = crf.loss_function
    else:
        model.add(TimeDistributed(Dense(output_size, activation='softmax')))
        loss = 'sparse_categorical_crossentropy'
    model.compile('adam', loss=loss)
    return model


def main():
    maxlen = 8
    embedding_size = 16
    representation_size = 5
    lstm_size = 10
    output_size = 5

    X, y = make_data(100, maxlen, embedding_size, output_size)

    for build_model in [build_single_model, build_combined_model]:
        for use_crf in [True, False]:
            # the error happens in build_combined_model with use_crf = True

            print('-', build_model.__name__, 'using', 'crf' if use_crf else 'softmax')
            model = build_model(embedding_size, representation_size, lstm_size, output_size, use_crf)
            try:
                model.fit(X, y)
            except ValueError:
                print(traceback.format_exc())
            print()


if __name__ == '__main__':
    main()