BOWENBOWEN.py

""" Recurrent Neural Network.
A Recurrent Neural Network (LSTM) implementation example using TensorFlow library.
This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)
Links:
    [Long Short Term Memory](http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf)
    [MNIST Dataset](http://yann.lecun.com/exdb/mnist/).
Author: Aymeric Damien
Project: https://github.com/aymericdamien/TensorFlow-Examples/
"""

import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn

# Training Parameters
EPOCHS = 5000
LEARNING_RATE = 0.0005
#training_steps = 10000
BATCH_SIZE = 128  # the number of samples that will be propagated through the network.
#display_step = 200

# Network Parameters
#init_scale = 0.05
#hidden_size = 300
#vocab_size = X_Train.shape[2]

#num_input = 1  # MNIST data input (img shape: 28*28)
#timesteps = 5  # timesteps
#num_hidden = 128  # hidden layer num of features
#num_classes = 5  # MNIST total classes (0-9 digits)

# tf Graph input
#X = tf.placeholder("float", [None, timesteps, num_input])
#Y = tf.placeholder("float", [None, num_classes])

# tf state
#cell_state = tf.placeholder(tf.float32, [batch_size, state_size])
#hidden_state = tf.placeholder(tf.float32, [batch_size, state_size])
#init_state = tf.nn.rnn_cell.LSTMStateTuple(cell_state, hidden_state)


def test():

    # BATCHING
    BATCH_SIZE = 4
    #x = np.random.sample((100, 2))
    x = np.arange(100)
    x = x.reshape((50,2))
    # make a dataset from a numpy array
    
    x_ph = tf.placeholder(tf.int32, shape=(50, 2))
    dataset = tf.data.Dataset.from_tensor_slices(x_ph).batch(BATCH_SIZE).repeat()
    iterr = dataset.make_initializable_iterator()
    
    with tf.Session() as sess:
        sess.run(iterr.initializer, feed_dict={x_ph: x})
        for k in range(int(np.ceil(len(x)/BATCH_SIZE))):
            el = iterr.get_next()
            
            print(sess.run(el))


def buildLayers(modelname, X_Train, X_Test, Y_Train, Y_Test, valid_length_train, fold, TimeSteps=100):
    def preparing_data():

        X_train = np.array(X_Train[fold])
        Y_train = np.array(Y_Train[fold], dtype=np.float)
        Y_train = Y_train * 0.2 - 0.1  # 0.1 / 0.3 / 0.5 / 0.7 / 0.9
        print(X_train.shape, Y_train.shape)

        X_test = np.array(X_Test[fold])
        Y_test = np.array(Y_Test[fold], dtype=np.int32)
        print(X_test.shape, Y_test.shape)

        #X_tuning = np.array(X_Tuning[fold])
        #Y_tuning = np.array(Y_Tuning[fold], dtype=np.float)
        #Y_tuning = Y_tuning * 0.2 - 0.1

        VL_train = np.array(valid_length_train[fold])
        VL_train = VL_train.flatten()
        print(VL_train.shape)

        return X_train, Y_train, VL_train
    
    tf.reset_default_graph()
    X_placeholder  = tf.placeholder(tf.float32, shape=[None, TimeSteps, 300])
    Y_placeholder  = tf.placeholder(tf.float32, shape=[None, 1])
    VL_placeholder = tf.placeholder(tf.int64, shape=[None]) 
    batch_placeholder = tf.placeholder(tf.int64)
    train_dataset = tf.data.Dataset.from_tensor_slices(
        (X_placeholder, Y_placeholder, VL_placeholder)).batch(batch_placeholder).repeat()
    train_iter = train_dataset.make_initializable_iterator()
    train_init_op = train_iter.make_initializer(train_dataset)
    
    x, y, vl = train_iter.get_next()
    #x = tf.unstack(x, 100, 1)  # (batch_size, n_steps, n_input) to (batch_size, n_input)

    num_layers = 2
    state_size = 300

    #lstm_init_state = init_state = tf.placeholder(tf.float32, [num_layers, 2, None, state_size]) # None is for batch_size
    #state_per_layer_list = tf.unstack(lstm_init_state, axis=0)
    with tf.device('/gpu:0'):
        '''
        lstm_cell1 = tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(100)
        lstm_cell2 = tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(50)
        lstm_cell1_dropout = tf.nn.rnn_cell.DropoutWrapper(
            lstm_cell1, input_keep_prob=0.8)
        lstm_cell2_dropout = tf.nn.rnn_cell.DropoutWrapper(
            lstm_cell2, input_keep_prob=0.8)

        cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell1_dropout, lstm_cell2_dropout], state_is_tuple=True)
        #init_state = cells.zero_state(None, tf.float32)

        lstm_outputs, lstm_state = tf.nn.dynamic_rnn(
            cells, x, sequence_length=None, parallel_iterations=128, time_major=False, dtype=tf.float32)
        '''

        lstm = tf.contrib.cudnn_rnn.CudnnLSTM(num_layers=2, num_units=100)
        lstm_outputs, lstm_state = lstm(x, training=True)
        
        dense1 = tf.contrib.layers.fully_connected(
            lstm_outputs[:,-1,:], 10, activation_fn=tf.nn.leaky_relu)
        dropout1 = tf.nn.dropout(dense1, rate=0.2)
        dense2 = tf.contrib.layers.fully_connected(dropout1, 5, activation_fn=tf.nn.leaky_relu)
        dropout2 = tf.nn.dropout(dense2, rate=0.2)
        dense3 = tf.contrib.layers.fully_connected(dropout2, 1, activation_fn=tf.nn.leaky_relu)

        loss = tf.losses.mean_squared_error(dense3, y)
        optimizer = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss)
    saver = tf.train.Saver()
    history = {"loss": []}

    with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
        X_train, Y_train, VL_train = preparing_data()
        best_loss = 10.0
        n_batches = int(np.ceil(len(X_train) / BATCH_SIZE))
        sess.run(tf.global_variables_initializer())
        sess.run(train_init_op, feed_dict={
                 X_placeholder: X_train, Y_placeholder: Y_train, batch_placeholder: BATCH_SIZE, VL_placeholder: VL_train})
        print('Training...')
        for ep in range(EPOCHS):
            tot_loss = 0
            for i in range(n_batches):
                tot_loss = 0
                _, loss_value = sess.run([optimizer, loss], feed_dict={
                    X_placeholder: X_train, Y_placeholder: Y_train, batch_placeholder: BATCH_SIZE, VL_placeholder: VL_train})
                tot_loss += loss_value
        
            print("Epoch: {}, Loss: {:.4f}".format(ep, tot_loss))
            if tot_loss < best_loss:
                best_loss = tot_loss
                saver.save(sess, '{}best-{}-model.ckpt'.format(modelname, str(fold)))
            history['loss'].append(tot_loss)
        
        saver.save(sess, '{}{}-model.ckpt'.format(modelname, str(fold)))
        
def main():
    pass    

if __name__ == "__main__":
    test()