Another toy (binary) mixture of experts model with 4 experts and a gating network.

mixture_of_experts.py

from keras.models import Model
from keras.layers import Input, Dense, concatenate, dot
from numpy.random import randint
import numpy as np


def my_model(n=20):
    inputs = Input(shape=(n,))

    m1 = Dense(1)(inputs)
    m2 = Dense(1)(inputs)
    m3 = Dense(1)(inputs)
    m4 = Dense(1)(inputs)

    gates = Dense(
        4, activation="sigmoid"
    )(inputs)

    models = concatenate([m1, m2, m3, m4])

    output = dot([models, gates], 1)
    model = Model([inputs], output)

    model.compile(
        loss='binary_crossentropy',
        optimizer="sgd"
    )
    return model


def test_data(m=1000, n=20):
    x = np.random.random((m, n))
    y = randint(2, size=(m, 1))
    return y, x


if __name__ == '__main__':
    y_train, x_train = test_data()
    model = my_model()
    model.fit([x_train], y_train)