Python script version. It's same content as the former jupyter script.

3+layer+neural+net+smarter+.py

# coding: utf-8
# XOR, UCI_Iris datasetを解くNeuralNetwork

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import six, math, random, time
import pandas as pd
import matplotlib.pyplot as plt

# 非線形性のNeural Net
class InputLayer:
    """
    Neural netの入力層の機能を持つクラス
    コンストラクタの引数：入力データの次元dim(int)
    """
    def __init__(self, dim):
        self.dim = dim
        self.data = [0.0 for i in six.moves.range(self.dim)]

    def forward(self):
        pass

    def backward(self):
        pass

    def updateWeight(self, alpha):
        pass

class NeuroLayer:
    """
    Neural netの隠れ層の機能をもつクラス(中間層or出力層)
    コンストラクタの引数：入力データの次元dim(int)、データ入力を受ける層preLayer(Object)\
          重み初期値を決める乱数のシード(double)、バイアスの初期値bias(double)\
          重み初期値の範囲randA～randB(double)
    """
    def __init__(self, dim, preLayer, rand_choice, bias, randA, randB):
        self.dim = dim
        self.preLayer = preLayer
        self.data = [0.0 for i in six.moves.range(self.dim)]
        self.rand_choice = rand_choice
        self.weight = [[rand_choice(randA, randB)\
                        for i in six.moves.range(self.preLayer.dim)]\
                        for j in six.moves.range(self.dim)]
        self.bias = [bias for i in six.moves.range(self.dim)]
        self.nextLayer = None
        self.preLayer.nextLayer = self
        self.diff = [0.0 for i in six.moves.range(self.preLayer.dim)]
        self.diffWeight = [[0.0 for i in six.moves.range(self.preLayer.dim)]\
                           for j in six.moves.range(self.dim)]
        self.diffBias = [0.0 for i in six.moves.range(self.dim)]

    def forward(self):
        temp = [0.0 for i in six.moves.range(self.dim)]
        for m in six.moves.range(self.dim):
            for p in six.moves.range(self.preLayer.dim):
                temp[m] += self.preLayer.data[p] * self.weight[m][p]
            self.data[m] = temp[m] + self.bias[m]

    def backward(self):
        for m in six.moves.range(self.dim):
            self.diffBias[m] += self.nextLayer.diff[m] * 1
        for p in six.moves.range(self.preLayer.dim):
            for m in six.moves.range(self.dim):
                self.diffWeight[m][p] += self.nextLayer.diff[m] * self.preLayer.data[p]
                self.diff[p] = self.nextLayer.diff[m] * self.weight[m][p]

    def updateWeight(self, alpha):
        for m in six.moves.range(self.dim):
            self.bias[m] -= alpha * self.diffBias[m]
            for p in six.moves.range(self.preLayer.dim):
                self.weight[m][p] -= alpha * self.diffWeight[m][p]
        self.diffBias = [0.0 for i in six.moves.range(self.dim)]
        self.diffWeight = [[0.0 for i in six.moves.range(self.preLayer.dim)]\
                           for j in six.moves.range(self.dim)]

class ActionLayer:
    """
    活性関数の機能をもつクラス
    コンストラクタの引数：データ入力を受ける層prelayer(object)
    """
    def __init__(self, preLayer):
        self.preLayer = preLayer
        self.dim = self.preLayer.dim
        self.data = [0.0 for i in six.moves.range(self.preLayer.dim)]
        self.nextLayer = None
        self.preLayer.nextLayer = self
        self.diff = [0.0 for i in six.moves.range(self.preLayer.dim)]

    def activation(self, x):
        return 1.0 / (1.0 + math.exp(-x))

    def deactivation(self, y):
        return y * (1 - y)

    def forward(self):
        for m in six.moves.range(self.dim):
            self.data[m] = self.activation(self.preLayer.data[m])

    def backward(self):
        for m in six.moves.range(self.dim):
            self.diff[m] = self.nextLayer.diff[m] * self.deactivation(self.data[m])

    def updateWeight(self, alpha):
        pass

class ErrorLayer:
    """
    出力層の出力と教師ラベルとの誤差を求める機能をもつクラス
    コンストラクタの引数：入力を受ける層preLayer(Object)
    """
    def __init__(self, preLayer):
        self.preLayer = preLayer
        self.dim = self.preLayer.dim
        self.data = 0.0
        self.target = [0.0 for i in six.moves.range(self.dim)]
        self.diff = [0.0 for i in six.moves.range(self.preLayer.dim)]
        self.preLayer.nextLayer = self
        self.result = [0.0 for i in six.moves.range(self.dim)]

    def forward(self):
        for m in six.moves.range(self.dim):
            self.data += (self.preLayer.data[m] - self.target[m]) ** 2
            if self.preLayer.data[m] > 0.5:
                self.result[m] = 1.
            else:
                self.result[m] = 0.

    def backward(self):
        for p in six.moves.range(self.preLayer.dim):
            self.diff[p] = 2 * (self.preLayer.data[p] - self.target[p])

    def updateWeight(self, alpha):
        pass


def train_nn(alpha, iteration, batchsize, neuralNetwork,\
        trainingData, trainingTarget, testData, testTarget):
    """
    neural netを学習させる
    入力：学習率appha(double)、学習epoch(int)、neuralnet(list)、
        　学習用データ(list)、学習ラベル(list)、テストデータ(list)、テストラベル(list)
    出力：それぞれのepoch終了時でのlossの値をもつlist
    """
    start_time = time.clock()
    loss_list = [0 for i in six.moves.range(iteration)]
    perm = [i for i in six.moves.range(len(trainingData))]
    random.shuffle(perm)
    
    # train
    for itr in six.moves.range(iteration):
        for i in six.moves.range(0, len(trainingData), batchsize):
            x = [ trainingData[idx] for idx in perm[i: i + batchsize]]
            t = [trainingTarget[idx] for idx in perm[i: i + batchsize]]
            for (d, t) in zip(x, t):            
                neuralNetwork[0].data = d
                neuralNetwork[5].target = t
                for layer in neuralNetwork:
                    layer.forward()
                for layer in reversed(neuralNetwork):
                    layer.backward()
            for layer in neuralNetwork:
                layer.updateWeight(alpha)
        loss_list[itr] = neuralNetwork[5].data
        neuralNetwork[5].data = 0
            
    # test
    correct = 0
    for (d, t) in zip(testData, testTarget):
        neuralNetwork[0].data = d
        neuralNetwork[5].target = t
        for layer in neuralNetwork:
            layer.forward()
        if neuralNetwork[5].result == t:
            correct += 1
    
    elapsed_time = time.clock() - start_time
    print("経過時間",elapsed_time)
    print("train epoch={}, test accuracy={}%".format(\
        iteration, (correct / len(testData) * 100)))
    
    return loss_list, (correct / len(testData) * 100)


def get_dataset(d_dir, N_train):
    """
    IrisデータセットのCSVファイルからデータを抽出および整形
    入力：ファイルディレクトリ(str)、教師ラベル名(list)、学習に使うデータ数(int)
    出力：学習データ(list)、学習ラベル(list)、テストデータ(list)、テストラベル(list)
    """
    csv_data = pd.read_csv(d_dir, header=None)
    input_data = csv_data[[0, 1, 2, 3]]
    input_data = input_data.as_matrix()
    change_target = csv_data[4]
    input_target = [([1., 0., 0.]) if (d == "Iris-setosa") else (\
        ([0., 1., 0.]) if (d == "Iris-versicolor") else (\
        ([0., 0., 1.]) if (d == "Iris-virginica") else (\
        ))) for d in change_target]
    
    perm = [i for i in six.moves.range(len(input_data))]
    random.shuffle(perm)
    input_data = [input_data[idx] for idx in perm]
    input_target = [input_target[idx] for idx in perm]
    
    train_d = input_data[: N_train]
    train_t = input_target[: N_train]
    test_d = input_data[N_train:]
    test_t = input_target[N_train:]    
    
    return train_d, train_t, test_d, test_t


# 排他的論理和問題にNeural Netを適用
"""
ハイパーパラメータ
alpha     :学習係数
iteration :学習epoch数
batchsize :学習時のバッチサイズ
bias      :バイアスの初期値
hiddenDim :隠れ層の次元数
randA     :重みの初期値を決める乱数の下限
randB     :重みの初期値を決める乱数の上限
"""

alpha = 0.7
iteration = 700
batchsize = 4
bias = 0.6
randA = -0.5
randB = 0.5
hiddenDim = 5
train_d = [[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]]
train_t = [[0.0], [1.0], [1.0], [0.0]]
test_d = [[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]]
test_t = [[0.0], [1.0], [1.0], [0.0]]

inputLayer = InputLayer(len(train_d[0]))
hiddenLayer = NeuroLayer(hiddenDim, inputLayer, random.uniform, bias, randA, randB)
hiddenActionLayer = ActionLayer(hiddenLayer)
outputLayer = NeuroLayer(len(train_t[0]), hiddenActionLayer,\
    random.uniform, bias, randA, randB)
outputActionLayer = ActionLayer(outputLayer)
errorLayer = ErrorLayer(outputActionLayer)

neuralNetwork = [inputLayer, hiddenLayer, hiddenActionLayer,\
    outputLayer, outputActionLayer, errorLayer]

loss_list, acc = train_nn(alpha, iteration, batchsize, neuralNetwork,\
    train_d, train_t, test_d, test_t)

# plot loss values on training
plt.plot(loss_list)
plt.title('XOR train loss, accuracy={}'.format(acc))
plt.xlabel('epoch')
plt.ylabel('loss')
plt.xlim([0, len(loss_list)-1])
# plt.savefig("xor_loss.png")
plt.show()


# CUI Iris datasetにNeural Netを適用
"""
ハイパーパラメータ
alpha     :学習係数
iteration :学習epoch数
batchsize :学習時のバッチサイズ
bias      :バイアスの初期値
hiddenDim :隠れ層の次元数
randA     :重みの初期値を決める乱数の下限
randB     :重みの初期値を決める乱数の上限
N_train   :学習に使うサンプル数
f_name    :ファイルのパス
"""

alpha = 0.02
iteration = 500
batchsize = 20
bias  = 0.9
hiddenDim = 12
randA = -0.3
randB = 0.3
N_train = 120
f_name = "./iris.csv"

train_d, train_t, test_d, test_t  = get_dataset(f_name, N_train)

inputLayer = InputLayer(len(train_d[0]))
hiddenLayer = NeuroLayer(hiddenDim, inputLayer, random.uniform, bias, randA, randB)
hiddenActionLayer = ActionLayer(hiddenLayer)
outputLayer = NeuroLayer(len(train_t[0]), hiddenActionLayer,\
    random.uniform, bias, randA, randB)
outputActionLayer = ActionLayer(outputLayer)
errorLayer = ErrorLayer(outputActionLayer)

neuralNetwork = [inputLayer, hiddenLayer, hiddenActionLayer,\
    outputLayer, outputActionLayer, errorLayer]

loss_list, acc = train_nn(alpha, iteration, batchsize, neuralNetwork,\
    train_d, train_t, test_d, test_t)

# plot loss value on training
plt.plot(loss_list)
plt.title('Iris train loss, accuracy={}'.format(acc))
plt.xlabel('epoch')
plt.ylabel('loss')
plt.xlim([0, len(loss_list)-1])
# plt.savefig("iris_loss.png")
plt.show()