RNN with single output

minimal.py

#!/usr/bin/env python3

import csv
import tensorflow as tf
from tensorflow.contrib.feature_column import sequence_categorical_column_with_identity, sequence_input_layer
from random import shuffle

import numpy as np

with open('train.tsv') as f:
    reader = csv.reader(f, delimiter='\t')
    # skip header
    next(reader)
    lines = [row for row in reader]

domain = set(''.join([r[2] for r in lines]))
domain_size = len(domain)
to_int = {term: i + 1 for i, term in enumerate(domain)}
to_term = {i: term for term, i in to_int.items()}

num_steps = 50


def to_xy(data):
    return [r[0] for r in data], [[r[1]] for r in data]

def pad_data(x):
    row_len = [len(r) for r in x]
    max_len = max(row_len)
    x = [[0] * (max_len - row_len[i]) + r for i, r in enumerate(x)]
    return np.array(x)

def prepare_data(lines):
    data_all = [[[to_int[c] for c in list(r[2])], int(r[3])] for r in lines if len(r[2]) <= num_steps]
    shuffle(data_all)

    data_x, data_y = to_xy(data_all)
    nclass = max(data_y)[0] + 1
    data_y = np.eye(nclass, dtype=np.int32)[np.array(data_y).reshape(-1)]

    data_x = pad_data(data_x)
    return data_x, data_y, nclass

x, y, nclass = prepare_data(lines)

train_split, val_split = int(0.8 * len(x)), int(0.9 * len(x))
train_x, train_y = x[:train_split], y[:train_split]
valid_x, valid_y = x[train_split:val_split], y[train_split:val_split]
test_x, test_y = x[val_split:], y[val_split:]

def rnn_model(features, labels, mode=None, params=None, config=None):
    chars = sequence_categorical_column_with_identity('chars', params['domain_size']+1)
    chars_emb = tf.feature_column.embedding_column(chars, dimension=10)
    columns = [chars_emb]

    input_layer, sequence_length = sequence_input_layer(features, columns)
    
    hidden_units = 32
    lstm = tf.nn.rnn_cell.LSTMCell(hidden_units, state_is_tuple=True)
    rnn_outputs, state = tf.nn.dynamic_rnn(lstm, 
                                          inputs = input_layer,
                                          sequence_length=sequence_length, dtype=tf.float32)

    output = rnn_outputs[:,-1,:]
    logits = tf.layers.dense(output, params['n_classes'], activation=tf.nn.tanh)
    # apply projection to every timestep.
    # Compute predictions.
    predicted_classes = tf.nn.softmax(logits)

    # Compute loss.

    loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels, logits=logits)
    # Compute evaluation metrics.
    accuracy = tf.metrics.accuracy(labels=labels,
                                   predictions=predicted_classes,
                                   name='acc_op')
    metrics = {'accuracy': accuracy}
    tf.summary.scalar('accuracy', accuracy[1])

    optimizer = tf.train.AdagradOptimizer(learning_rate=0.1)
    train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
    return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)

train_input_fn = tf.estimator.inputs.numpy_input_fn(
    x={"chars": train_x},
    y=train_y,
    num_epochs=1,
    shuffle=False,
    batch_size=8)

classifier = tf.estimator.Estimator(model_fn=rnn_model,
                                   params={'n_classes': nclass,
                                           'domain_size': domain_size,
                                           'num_steps': num_steps})
classifier.train(input_fn=train_input_fn) 

train.tsv