learn.py

import tensorflow as tf
import numpy as np
import math
from tensorflow.python.framework import ops
import matplotlib as mpl
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from itertools import chain
import pandas as pd
from matplotlib import cm


# Normalization function (min-max norm)
def normalize_cols(m):
    col_max = m.max(axis=0)
    col_min = m.min(axis=0)
    return (m - col_min) / (col_max - col_min)


# Set Goal and Max number of iteration
ERROR_GOAL = 0.25
MAX_EPOCH_NUMBER = 10000


# Set Random Seed
SEED = 2
tf.set_random_seed(SEED)
np.random.seed(SEED)

# Load datasets
data = np.loadtxt("iris.data")
x_values = data[:, 0:4]
y_values = data[:, 4]


"""iris = datasets.load_iris()
x_values = np.array([x[0:4] for x in iris.data])
y_values = np.array([x[4] for x in iris.data])"""

print(y_values)

train_indices = np.random.choice(len(x_values), round(len(x_values)*0.8), replace=False)
test_indices = np.array(list(set(range(len(x_values))) - set(train_indices)))

x_vals_train = x_values[train_indices]
x_vals_test = x_values[test_indices]
y_vals_train = y_values[train_indices]
y_vals_test = y_values[test_indices]

sorting_indexes = y_values.argsort()
x_sorted = x_values[sorting_indexes]
y_sorted = y_values[sorting_indexes]

test_sorting_indexes = y_vals_test.argsort()
x_test_sorted = x_vals_test[test_sorting_indexes]
y_test_sorted = y_vals_test[test_sorting_indexes]

# Perform normalization
x_values_train = np.nan_to_num(normalize_cols(x_vals_train))
x_values_test = np.nan_to_num(normalize_cols(x_vals_test))
#x_values_train = np.nan_to_num(normalize_cols(x_values))


# Generate Spread Constant table
sc_table = [0.01, 0.1, 0.3, 0.5, 0.7, 0.9, 1, 3, 5, 7, 9]

#sc_table = [0.5, 0.7, 0.9, 1, 3, 5]

# Generate Learning Rate Table
lr_table = [0.001, 0.01, 0.1, 1]

# Initial Spread Constant
SC = 0.01


# Creation of custom NN function
def gaussian_function(input_layer):
    initial = math.exp(-SC*math.pow(input_layer, 2))
    return initial


np_gaussian_function = np.vectorize(gaussian_function)


def d_gaussian_function(input_layer):
    initial = -2*SC*input_layer * math.exp(-SC * math.pow(input_layer, 2))
    return initial


np_d_gaussian_function = np.vectorize(d_gaussian_function)


def np_d_gaussian_function_32(input_layer):
    return np_d_gaussian_function(input_layer).astype(np.float32)


def tf_d_gaussian_function(input_layer, name=None):
    with ops.name_scope(name, "d_gaussian_function", [input_layer]) as name:
        y = tf.py_func(np_d_gaussian_function_32, [input_layer],[tf.float32], name=name, stateful=False)
    return y[0]


def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
    rnd_name = 'PyFunGrad' + str(np.random.randint(0, 1E+8))

    tf.RegisterGradient(rnd_name)(grad)
    g = tf.get_default_graph()
    with g.gradient_override_map({"PyFunc": rnd_name}):
        return tf.py_func(func, inp, Tout, stateful=stateful, name=name)


def gaussian_function_grad(op, grad):
    input_variable = op.inputs[0]
    n_gr = tf_d_gaussian_function(input_variable)
    return grad * n_gr


def np_gaussian_function_32(input_layer):
    return np_gaussian_function(input_layer).astype(np.float32)


def tf_gaussian_function(input_layer, name=None):
    with ops.name_scope(name, "gaussian_function", [input_layer]) as name:
        y = py_func(np_gaussian_function_32, [input_layer], [tf.float32], name=name, grad=gaussian_function_grad)
    return y[0]
# end of defining activation function


sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
plot_number = 0

for lr in lr_table:
    # First Layer Nodes Loop
    percentage_propper_class_a = []
    percentage_propper_class_b = []
    percentage_propper_class_c = []
    for first_layer_nodes in range(len(x_values_train), 0, -10):
        #for first_layer_nodes in range(20, 0, -10):
        # Spread Constant Loop
        for spread_constant in sc_table:
            SC -= SC
            SC += spread_constant
            # Initialize placeholders
            x_data = tf.placeholder(shape=[None, 4], dtype=tf.float32)
            y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

            # Create variables for NN layers
            #A1 = tf.Variable(tf.random_normal(shape=[4, first_layer_nodes]))  # input -> first layer nodes
            A2 = tf.Variable(tf.random_normal(shape=[first_layer_nodes, 1]))  # first_layer nodes -> sum node
            """c = tf.Variable(tf.random_normal(shape=[first_layer_nodes]))  # centroids
            # Declare NN
            inputs_with_weights = tf.matmul(x_data, A1)
            euclid_dist = tf.sqrt(tf.pow(tf.subtract(inputs_with_weights, c), 2))
            #first_output = tf_gaussian_function(euclid_dist)
            first_output = tf.exp(-SC* euclid_dist * euclid_dist)
            final_output = tf.matmul(first_output, A2)"""

            #with tf.name_scope("Hidden_layer") as scope:
            #c = tf.Variable(tf.random_uniform([first_layer_nodes, 4], dtype=tf.float32))
            r_i = np.random.choice(len(x_values_train), size=first_layer_nodes)
            init_data = x_values_train[r_i]
            #c = tf.get_variable('centroids', initializer=tf.constant_initializer(init_data), shape=[first_layer_nodes, 4], reuse=True)
            #c = tf.Variable(tf.random_uniform([first_layer_nodes, 4], dtype=tf.float32))
            c = tf.Variable(init_data, dtype=tf.float32)
            #c = c.assign(init_data)
            exp_list = []
            for i in range(first_layer_nodes):
                x_m_c = tf.subtract(x_data, c[i, :])
                euclid_distance = tf.reduce_sum(tf.multiply(x_m_c, x_m_c), 1)
                exp_list.append(tf.exp(tf.multiply(-SC, euclid_distance)))
            phi = tf.transpose(tf.stack(exp_list))

            final_output = tf.matmul(phi, A2)

            # Declare loss function (SSE)
            loss = tf.div(tf.reduce_sum(tf.square(tf.subtract(y_target, final_output))), 2)
            #loss = tf.losses.mean_squared_error(labels = y_target, predictions=final_output)
            # Declare optimizer
            my_opt = tf.train.GradientDescentOptimizer(lr)
            train_step = my_opt.minimize(loss)

            # Initialize variables
            init = tf.global_variables_initializer()

            sess.run(init)

            # Make loss vector
            loss_vec = []
            test_loss = []

            # rand_index = np.random.choice(len(x_values_train), size=len(x_values_train))

            for i in range(MAX_EPOCH_NUMBER):
                rand_index = np.random.choice(len(x_values_train), size=120)
                x_d = x_values_train[rand_index]
                y_d = np.transpose([y_values[rand_index]])

                sess.run(train_step, feed_dict={x_data: x_d, y_target: y_d})

                temp_loss = sess.run(loss, feed_dict={x_data: x_d, y_target: y_d})

                #f_o = sess.run(final_output, feed_dict={x_data: x_d})
                #l = tf.reduce_sum((y_d - f_o)**2)/2
                #print(l)
                loss_vec.append(temp_loss)

                test_temp_loss = sess.run(loss, feed_dict={x_data: x_vals_test, y_target: np.transpose([y_vals_test])})
                test_loss.append(test_temp_loss)
                with open('txt/Iris/txtOfLoss/{}_{}_{}.txt'.format(lr, first_layer_nodes, spread_constant), "a") as sf:
                    sf.write("{}\t{}\t{}\t{}\t{}\n".format(lr, first_layer_nodes, spread_constant, i+1, temp_loss))
                if (i + 1) % 50 == 0:
                    print("Learning Rate = {}, Nr of first layer nodes: {}, Spread Constant: {}, Generation: {}, Loss = {}".format(lr, first_layer_nodes, spread_constant, i+1, temp_loss))

                if np.isnan(temp_loss):
                    break
                if temp_loss < ERROR_GOAL:
                    break

            print(
                "FINISHED: Learning Rate = {}, Nr of first layer nodes: {}, Spread Constant: {}, Generation: {}, Loss = {}".format(
                    lr, first_layer_nodes, spread_constant, len(loss_vec), loss_vec[-1]))

            plot_number += 1
            plt.figure(plot_number)
            plt.plot(loss_vec, 'k-')
            plt.plot(test_loss, 'r--', label='Test Loss')
            plt.suptitle('Blad (SSE) na Epoke', fontsize=14, fontweight='bold')
            plt.title('lr = {}, fln = {}, sc = {}, ilosc epok = {}, blad koncowy = {}'.format(lr, first_layer_nodes, spread_constant, len(loss_vec), loss_vec[-1]))
            plt.legend(loc='upper right')
            plt.xlabel('Epoka')
            plt.ylabel('SSE')
            plt.savefig('Plots/Iris/PlotsOfLoss/{}_{}_{}.png'.format(lr, first_layer_nodes, spread_constant))
            plt.close()

            plot_number += 1
            plt.figure(plot_number)
            plt.plot(y_test_sorted, 'k-', label='Oczekiwane wyjscie')
            x_d = x_values_test
            f_o = sess.run(final_output, feed_dict={x_data: x_d})

            f_o_s = f_o[test_sorting_indexes]

            with open('txt/Iris/txtOfClassification/{}_{}_{}.txt'.format(lr, first_layer_nodes, spread_constant), 'a') as sf:
                for u in range(len(y_test_sorted)):
                    sf.write("{}\t{}\t{}\n".format(u+1, y_sorted[u], f_o_s[u]))
            plt.plot(f_o[test_sorting_indexes], 'b-', label="Wyjscie sieci")
            plt.suptitle('Wykres wyjscia sieci', fontsize=14, fontweight='bold')
            plt.title('lr = {}, fln = {}, sc = {}, ilosc epok = {}, blad koncowy = {}'.format(lr, first_layer_nodes,
                                                                                              spread_constant,
                                                                                              len(loss_vec),
                                                                                              loss_vec[-1]))
            plt.legend(loc='upper left')
            plt.xlabel('Nr danych')
            plt.ylabel('Klasa')
            plt.savefig('Plots/Iris/PlotsOfClassification/{}_{}_{}.png'.format(lr, first_layer_nodes, spread_constant))
            plt.close()

            n = 0
            if not np.isnan(loss_vec[-1]):
                for err in ((np.transpose(f_o)[0] - y_vals_test)**2):
                    if err < ERROR_GOAL:
                        n += 1

            print("Percentage of propper_classification = {}".format(100 * n / len(y_values)))
            percentage_propper_class_a.append(spread_constant)
            percentage_propper_class_b.append(first_layer_nodes)
            percentage_propper_class_c.append(100 * n / len(y_values))

    plot_number += 1
    fig = plt.figure(plot_number)
    ax = Axes3D(fig)
    a = percentage_propper_class_a
    b = percentage_propper_class_b
    c = percentage_propper_class_c

    df = pd.DataFrame({'x': a, 'y': b, 'z': c})

    ax.set_xlabel('Spread Constant')
    ax.set_ylabel('Liczba neuronow w pierwszej warstwie')

    ax.set_title('Wykres stosunku poprawnie sklasyfikowanych elementow dla lr = {}'.format(lr))

    ax.legend()
    surf = ax.plot_trisurf(df.x, df.y, df.z, color='b', cmap=cm.jet, linewidth=0.1)
    fig.colorbar(surf, shrink=0.5, aspect=5)

    fig.savefig('Plots/Iris/PlotsOfPercentage/{}.png'.format(lr))
    plt.close()

    with open('txt/Iris/txtOfPercentage/{}.txt'.format(lr), 'a') as sf:
        for u in range(len(a)):
            sf.write("{}\t{}\t{}\n".format(a[u], b[u], c[u]))