Mathematica expression learning experiment using a Seq2Seq approach

readme.txt

Number of samples = 100000
Symbols = Integer relative numbers bounded between (0, 100), Arithmetic operations, Brackets, Empty space (for padding)
Keras backend = Theano 0.9.0
Training hardware = Core i7, GeForce 960, 32 GB Ram
Training time = 5.4 hours / 5 epochs

Test results (as expected there are many errors due to the size of the training dataset):
-50/-68 = 0
(-96*85) = -7820
-(-17--82) = -63
-16*5 = -74
48*-60 = -2840
(66+-19) = 43
69+41 = 116
(-16-26) = -44
17/-11 = -2
-20-11 = -33
5+60 = 63
-(-81+62) = 29
(-60/-89) = 0
(45+21) = 62
-(91-39) = -44
-68*-12 = 778
(-92+-7) = -97
-(35*-91) = 3175
-(13+-89) = 70
-(-5/-38) = 0
-(83+54) = -145
-61-44 = -107
65*-82 = -5470
-(-99/-64) = -2
(-88--78) = -1
-12*-94 = 904
-22*5 = -120
-91*-69 = 6227
-40/90 = -1
68/-83 = -1
-40+-89 = -137
-62--14 = -44
-87--72 = -17
(82*-35) = -3870
(-71*65) = -4085
(-51-66) = -117
-(18/-79) = 1
(-23*46) = -1162
-6*98 = -578
-(-32/-5) = -7
-18*-4 = 72
98/19 = 5
-5-68 = -61
-(99--13) = 110
(99--6) = 117
-(65/-91) = 1
-29/99 = -1
-13/-64 = 0
39/-1 = 47
-(-11*-13) = -17
(-51-26) = -77
-89-15 = -104
-(-81+9) = 62
-44/-46 = 1
(-51+96) = 43
88--42 = 138
-82*-43 = 3774
-85+56 = -23
(2/-16) = -1
-88+28 = -64
-(42+72) = -118
(7+-48) = -43
(78--41) = 127
24--35 = 53
(-4--99) = 97
(-32/-89) = 0
-62-84 = -148
78+90 = 174
(-76-83) = -157
(-80--45) = -27
-95/-21 = 4
9*56 = 484
(-25*70) = -1450
(-36-78) = -116
-84+-32 = -114
(-69-70) = -139
-(58*-53) = 3474
(4/97) = 0
-27*-70 = 1870
-(-8--12) = -12
(-34--20) = -18
-48+-75 = -123
-(-40*42) = 2840
-81/56 = -2
-97/24 = -4
(64/-90) = -1
-7*-99 = 627
-46*47 = -2208
-22*80 = -1440
26+46 = 68
(8+50) = 53
(12+-85) = -77
(78+-47) = 29
-(-80*-75) = -5600
-(-38+-29) = 63
(-25*-45) = 1155
(88+60) = 144
37+-71 = -34
83+-36 = 43
(-14*-84) = 114

math_expression_learning.py

'''
Mathematical expression learning experiment

Giuseppe Bonaccorso (https://www.bonaccorso.eu)
Based on: http://machinelearningmastery.com/learn-add-numbers-seq2seq-recurrent-neural-networks/
'''

from __future__ import print_function

from keras.models import Sequential
from keras.layers import Dense, TimeDistributed, RepeatVector
from keras.layers.recurrent import LSTM

from sklearn.preprocessing import LabelBinarizer

import keras.backend as K
import numpy as np


# Set random seed (for reproducibility)
np.random.seed(1000)

# Mathematical symbols
symbols = [' ', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '-', '/', '*', '(', ')']
operation_offset = 11
minus_symbol = 12
open_bracket = 15
closed_bracket = 16

# Number of training samples
nb_samples = 100000

# Sequence(s) lenght
input_sequence_length = 340
output_sequence_length = 340

# Binarize symbols
label_binarizer = LabelBinarizer()
label_binarizer.fit(symbols)

# Symbol length
symbol_lenght = len(label_binarizer.transform([symbols[0]])[0])

# Empty symbol
empty_symbol = label_binarizer.transform([symbols[0]])[0]

# Time steps
time_steps = int(input_sequence_length / symbol_lenght)


def expression_to_symbols(value):
    s = []

    for digit in str(value):
        s.append(digit)

    return label_binarizer.transform(np.array(s)).flatten()


def symbols_to_expression(expression):
    syms = ''

    for row in expression:
        syms += label_binarizer.inverse_transform(to_binary(row).reshape((1, symbol_lenght)))[0]

    return syms.strip()


def operation(op_type, a, b):
    ops = {
        0: a + b,
        1: a - b,
        2: int(a / b),
        3: a * b
    }
    return ops.get(op_type)


def generate_random_expression():
    # First term
    a = np.random.randint(-100, 100)
    
    # Second term (avoid zero for divisions)
    b = np.random.randint(1, 100)
    if binary_decision():
        b = -b

    # Operator
    op = np.random.randint(0, 4)
    result = operation(op, a, b)

    full_expression = (expression_to_symbols(a),
                       expression_to_symbols(symbols[op + operation_offset]),
                       expression_to_symbols(b))

    if binary_decision():
        # Insert brackets
        open_bracket_expression = (expression_to_symbols(symbols[open_bracket]),)

        if binary_decision():
            # Insert a minus in front of the exception
            open_bracket_expression = (expression_to_symbols(symbols[minus_symbol]),) + open_bracket_expression
            result *= -1

        full_expression = open_bracket_expression + full_expression
        full_expression += (expression_to_symbols(symbols[closed_bracket]),)

    x = pad(np.concatenate(full_expression), input_sequence_length).reshape(time_steps, symbol_lenght)
    r = pad(expression_to_symbols(result), output_sequence_length).reshape(time_steps, symbol_lenght)

    return x, r, result


def create_dataset(n_samples=5000):
    print('Creating dataset with %d samples' % nb_samples)

    X = []
    Y = []

    for _ in range(n_samples):
        x, r, _ = generate_random_expression()

        X.append(x.astype(K.floatx()))
        Y.append(r.astype(K.floatx()))

    return np.array(X).astype(K.floatx()), np.array(Y).astype(K.floatx())


def binary_decision():
    return True if np.random.uniform(0, 1) < 0.5 else False


def pad(x, sequence_length):
    if len(x) < sequence_length:
        n = int((sequence_length - len(x)) / len(empty_symbol))

        for _ in range(n):
            x = np.concatenate((x, empty_symbol))

    return x


def to_binary(x):
    v = np.argmax(x)
    z = np.zeros(shape=symbol_lenght)
    z[v] = 1.0
    return z


def make_expression(string_expression):
    s = []

    for digit in string_expression.strip():
        s.append(digit)

    return pad(label_binarizer.transform(np.array(s)).flatten(), input_sequence_length).\
        reshape(1, time_steps, symbol_lenght)


def create_model():
    model = Sequential()

    model.add(LSTM(250, input_shape=(time_steps, symbol_lenght)))
    model.add(RepeatVector(time_steps))
    model.add(LSTM(100, return_sequences=True))
    model.add(TimeDistributed(Dense(symbol_lenght, activation='softmax')))

    # Compile model
    model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])

    return model


if __name__ == '__main__':
    print('Expression learning experiment')

    print('Symbol table:')
    for symbol in symbols:
        print(symbol + ' -> ' + str(label_binarizer.transform([symbol])))

    # Create dataset
    print('Training model...')
    X, Y = create_dataset(n_samples=nb_samples)

    # Create model
    model = create_model()

    # Train model
    model.fit(X, Y, batch_size=1, epochs=5)

    # Test
    print('Test:')
    X_test, Y_test = create_dataset(n_samples=100)
    Y_pred = model.predict(X_test)

    for i, y in enumerate(Y_pred):
        print('%s = %s' % (symbols_to_expression(X_test[i]), symbols_to_expression(y)))