Gitmoko/MyMLP.py

## MyMLP.py

# coding: utf-8

# In[41]:


from matplotlib import pyplot as plt
import numpy as np
from sklearn import datasets


# In[42]:


class Cell():
    def __init__(self, name="sigmoid"):
        self.name = name

    def activate(self, Z):
        if self.name=="sigmoid":
            return self.sigmoid(Z)
        elif self.name=="tanh":
            return self.tanh(Z)
        elif self.name=="relu":
            return self.relu(Z)
        elif self.name=="linear":
            return self.linear(Z)

    def grad(self,Z):
        if self.name=="sigmoid":
            return self.grad_sigmoid(Z)
        elif self.name=="tanh":
            return self.grad_tanh(Z)
        elif self.name=="relu":
            return self.grad_relu(Z)
        elif self.name=="linear":
            return self.grad_linear(Z)

    def sigmoid(self, Z):
        return 1.0/(1.0001 + np.exp(-Z))

    def grad_sigmoid(self, Z):
        return self.sigmoid(Z)*(1.0 - self.sigmoid(Z))

    def tanh(self, Z):
        return np.tanh(Z)

    def grad_tanh(self, Z):
        return 1.0 - self.tanh(Z)*self.tanh(Z)

    def relu(self, Z):
        return Z * (Z > 0)

    def grad_relu(self, Z):
        return 1.0 * (Z > 0)

    def linear(self, Z):
        return Z

    def grad_linear(self, Z):
        return np.ones(Z.shape)


# In[43]:


class Output():
    def __init__(self,name="softmax"):
        self.name = name

    def output_calc(self,Z):
        if self.name=="softmax":
            return self.softmax(Z)
        elif self.name=="sigmoid":
            return self.sigmoid(Z)
        elif self.name == "linear":
            return self.linear(Z)


    def softmax(self,Z):
        e = np.exp(Z)

        return e / (np.sum(e))

    def sigmoid(self, Z):
        return 1.0/(1.00001 + np.exp(-Z))

    def linear(self, Z):
        return Z

    def dlossdin(self,output,answervector = None):
        if(self.name == "linear"):
            #損失関数は二乗誤差とする
            return output - answervector

        if(self.name == "softmax"):
            #損失関数は多クラス分類のクロスエントロピーとする
            return output - answervector

        if(self.name == "sigmoid" and answervector != None):
            #損失関数は２クラス分類のクロスエントロピーとする
            return output - answervector

        if(self.name == "sigmoid" and answervector == None):
            #損失関数はoutputを確率とみなした時のエントロピー
            #  dE/din = output*(1-output)*log((1-output)/output)
            coeff = output*(1-output)
            return  coeff * np.log(1.00-output) - coeff*np.log(output)


# In[60]:


import pprint
import copy
class Layer():
    def __init__(self,size,hidden_activation = "relu",output_activation="linear"):
        Cells = []
        for s in size:
            Cells.append([Cell() for i in range(0,s)])
        self.Cells = Cells
        self.Output_Calc = Output(output_activation)


    def getRandomInit(self,scaleW = 0.1,scaleB = 0.1):
        Ws = []
        Bs = []
        for i in range(0,len(self.Cells)-1):
            leftsize = len(self.Cells[i])
            rightsize = len(self.Cells[i+1])
            Ws.append(np.random.randn(rightsize,leftsize)*scaleW)
            Bs.append(np.random.randn(rightsize)*scaleB)
        return (Ws,Bs)


    #全セルへの中間入力と、出力層の結果を返す
    def forward(self,in_data,Ws,Bs,drop=0.0):
        in_list = []
        next_in = in_data
        in_list.append(next_in)
        for i in range(0,len(Ws)):
            next_in = [self.Cells[i][j].activate(next_in[j]) for j in range(0,len(next_in))]
            next_in = np.dot(Ws[i],next_in)
            next_in = next_in+Bs[i]
            #for i in range(0,len(next_in)):
             #   if(np.random.rand() < drop):
              #      next_in[i] = 0

            in_list.append(next_in)
        output = self.Output_Calc.output_calc(next_in)
        return (in_list,output)

    def update(self,Ws,Bs,in_list,output,delta=0.1,answervector = None,VerBose = False):

        dEdnextin = None
        Ws_fixed = []
        Bs_fixed= []
        dlossdW = []
        dlossdB = []
        for i in reversed(range(1,len(self.Cells))):
            if(i == len(self.Cells)-1):
                #損失関数を出力層セルへの入力で偏微分したもの dE/dO
                dEdin =  self.Output_Calc.dlossdin(output,answervector)
            else :
                #損失関数を中間層セルへの入力で偏微分したもの dE/dCnow =  ΣdE/dCnext * φ'(Cnow) * WCnow->Cnext
                dEdin = np.dot(Ws[i].T,dEdnextin).T #ΣdE/dCnext*WCnow->Cnext
                gradlist = []
                for cellnum in range(0,len(in_list[i])):
                    gradlist.append(self.Cells[i][cellnum].grad(in_list[i][cellnum]))
                dEdin = dEdin*gradlist# ΣdE/dCnext * φ'(Cnow) * WCnow->Cnext

            #損失関数をWで偏微分したもの  dE/dWpre_now = =dE/dWCnow * Cpre
            pre_out = np.array([self.Cells[i-1][j].activate(in_list[i-1][j]) for j in range(0,len(in_list[i-1]))])
            #print(pre_out.shape)
            dlossdWpre_now = np.multiply(np.matrix(dEdin).T,np.array(pre_out))
            #print(dlossdWpre_now.shape)
            dlossdBpre_now = dEdin
            dEdnextin = copy.copy(dEdin)

            Ws_fixed.insert(0,Ws[i-1] - delta*np.array((dlossdWpre_now)))
            Bs_fixed.insert(0,Bs[i-1] - delta*np.array(dlossdBpre_now))
        return (Ws_fixed,Bs_fixed)


# In[61]:


def Learn(layer,input_func,answervector_func,num=1000,delta=0.01,VerBose = False,initWs=None,initBs=None):
    if(initWs == None):
        Ws,Bs = layer.getRandomInit()
    else:
        Ws = initWs
        Bs= initBs

    #pprint.pprint(Ws)
    #pprint.pprint(Bs)
    print("\n")

    for i in range(0,num):
        #print("try"+str(i))
        inputs = input_func(i)
        #print(inputs)
        (in_list,output) = layer.forward(inputs,Ws,Bs)
        if(VerBose):
            print("in_list")
            print(in_list)
        newWs,newBs = layer.update(Ws,Bs,in_list,output,delta=delta,answervector= answervector_func(i))
        if(VerBose):
            print("newWs")
            #pprint.pprint(newWs)
            print("newBs")
            #pprint.pprint(newBs)
        Ws = newWs
        Bs = newBs

        #print("in")
        #pprint.pprint(inputs)
        #print("out")
        #pprint.pprint(output)

        #print("\n")

    return(Ws,Bs)


# In[62]:


layer = Layer([2,100,4],"relu","softmax")
times = 100
data = [np.random.randint(2,size=2) for x in range(0,times)]
testdata_func = lambda i : data[i]

code = [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]
answervector_func = lambda i : code[data[i][1]*2+data[i][0]]
(Ws,Bs) = Learn(layer,testdata_func,answervector_func,times,VerBose=False)


# In[63]:


in_list,out = layer.forward([1,1],Ws,Bs)
print(out)
in_list,out = layer.forward([0,1],Ws,Bs)
print(out)
in_list,out = layer.forward([1,0],Ws,Bs)
print(out)
in_list,out = layer.forward([0,0],Ws,Bs)
print(out)


# In[64]:


TrainingSampleNum = 2000 # 学習サンプル総数
TestSampleNum = 10000 # テストサンプル総数
ClassNum = 10 # クラス数（今回は10）
ImageSize = 28 # 画像サイズ（今回は縦横ともに28）
TrainingDataFile = './Images/TrainingSamples/{0:1d}-{1:04d}.png'
TestDataFile = './Images/TestSamples/{0:1d}-{1:04d}.png'
OutFile = './Images/OutSamples/gray_{0:1d}-{1:04d}.png'


# In[65]:


from skimage import io


# In[66]:


# LoadTemplates
def LoadTemplates () :
    labels = np.zeros( TrainingSampleNum, dtype=np.uint8 )
    templates = np.zeros( (TrainingSampleNum,ImageSize,ImageSize), dtype=np.uint8 )

    i = 0
    for label in range (0, ClassNum):
        for sample in range (0, TrainingSampleNum // ClassNum ):
            filename = TrainingDataFile.format(label,sample)
            templates[i,:,:] = io.imread(filename)
            labels[i]=label
            i += 1

    return templates, labels


# In[67]:


from scipy.ndimage.interpolation import shift
templates, labels = LoadTemplates ()

templates = templates/255
templates = templates.reshape(2000,784)


#indices = np.hstack((indices,indices))


# In[68]:


def MnistLearn(layer,initWs=None,initBs=None):
    global templates
    global labels

    cpytemplates = copy.copy(templates)

    #for i in range(0,len(cpytemplates)):
        #dx = np.random.randint(-3,4)
        #dy = np.random.randint(-3,4)
        #cpytemplates[i] = shift(cpytemplates[i],[dx,dy],cval=0)


    numinputs = len(cpytemplates)
    indices = list(range(0,numinputs))
    np.random.shuffle(indices)
    #print(indices)
    times = len(indices)
    testdata_func = lambda i : cpytemplates[indices[i]]
    answervector_func = lambda i :  np.identity(10)[labels[indices[i]]]
    (Ws,Bs) = Learn(layer,testdata_func,answervector_func,times,VerBose=False,delta=0.001,initWs=initWs,initBs=initBs)
    return (Ws,Bs)


# In[69]:


X_test = []
y_test = []

each_class_num = TestSampleNum // ClassNum

for label in range (0, ClassNum):
    for sample in range (0, each_class_num ):
        filename = TestDataFile.format(label,sample)
        t_img = io.imread ( filename ) /255
        #print(type(t_img))
        X_test.append(t_img.flatten())
        y_test.append(label)


def counting(layer,Ws,Bs):
    conf_mat = np.zeros((10,10))
    count = 0
    results = np.zeros ( (ClassNum, ClassNum) )
    each_class_num = TestSampleNum // ClassNum
    #print("1")
    for ind in range(0,len(X_test)):
            #print(t_img)
            (_,output) = layer.forward(X_test[ind],Ws,Bs)
            #output = np.array([0,0,0,0,0,0,0,0,0,0])
            #print(label)
            #print(output)
            i = output.argmax()
            #print(i)
            if(y_test[ind] == i):
                count += 1
            #print(count)
            conf_mat[y_test[ind],i] += 1
            #print("\n")
    return (conf_mat,count)


# In[73]:


import warnings
warnings.filterwarnings('error')

layer = Layer([28*28,512,512,10],"relu","softmax")
(Ws,Bs) = layer.getRandomInit()
print(Ws,Bs)
for i in range(0,100):
    print("epoch:"+str(i))
    (Ws,Bs) = MnistLearn(layer,Ws,Bs)
    if(i % 10 == 0):
        mat,count = counting(layer,Ws,Bs)
        pprint.pprint(mat)
        print(count)
        #print("a")


# In[ ]:


mat,count = counting(layer,Ws,Bs)


# In[74]:


pprint.pprint(mat)
print(count)


# In[75]:


for i in range(0,10):
    print("epoch:"+str(i))
    (Ws,Bs) = MnistLearn(layer,Ws,Bs)
    mat,count = counting(layer,Ws,Bs)
    pprint.pprint(mat)
    print(count)

	# coding: utf-8

	# In[41]:


	from matplotlib import pyplot as plt
	import numpy as np
	from sklearn import datasets


	# In[42]:


	class Cell():
	def __init__(self, name="sigmoid"):
	self.name = name

	def activate(self, Z):
	if self.name=="sigmoid":
	return self.sigmoid(Z)
	elif self.name=="tanh":
	return self.tanh(Z)
	elif self.name=="relu":
	return self.relu(Z)
	elif self.name=="linear":
	return self.linear(Z)

	def grad(self,Z):
	if self.name=="sigmoid":
	return self.grad_sigmoid(Z)
	elif self.name=="tanh":
	return self.grad_tanh(Z)
	elif self.name=="relu":
	return self.grad_relu(Z)
	elif self.name=="linear":
	return self.grad_linear(Z)

	def sigmoid(self, Z):
	return 1.0/(1.0001 + np.exp(-Z))

	def grad_sigmoid(self, Z):
	return self.sigmoid(Z)*(1.0 - self.sigmoid(Z))

	def tanh(self, Z):
	return np.tanh(Z)

	def grad_tanh(self, Z):
	return 1.0 - self.tanh(Z)*self.tanh(Z)

	def relu(self, Z):
	return Z * (Z > 0)

	def grad_relu(self, Z):
	return 1.0 * (Z > 0)

	def linear(self, Z):
	return Z

	def grad_linear(self, Z):
	return np.ones(Z.shape)


	# In[43]:


	class Output():
	def __init__(self,name="softmax"):
	self.name = name

	def output_calc(self,Z):
	if self.name=="softmax":
	return self.softmax(Z)
	elif self.name=="sigmoid":
	return self.sigmoid(Z)
	elif self.name == "linear":
	return self.linear(Z)


	def softmax(self,Z):
	e = np.exp(Z)

	return e / (np.sum(e))

	def sigmoid(self, Z):
	return 1.0/(1.00001 + np.exp(-Z))

	def linear(self, Z):
	return Z

	def dlossdin(self,output,answervector = None):
	if(self.name == "linear"):
	#損失関数は二乗誤差とする
	return output - answervector

	if(self.name == "softmax"):
	#損失関数は多クラス分類のクロスエントロピーとする
	return output - answervector

	if(self.name == "sigmoid" and answervector != None):
	#損失関数は２クラス分類のクロスエントロピーとする
	return output - answervector

	if(self.name == "sigmoid" and answervector == None):
	#損失関数はoutputを確率とみなした時のエントロピー
	# dE/din = output(1-output)log((1-output)/output)
	coeff = output*(1-output)
	return coeff * np.log(1.00-output) - coeff*np.log(output)



	# In[60]:


	import pprint
	import copy
	class Layer():
	def __init__(self,size,hidden_activation = "relu",output_activation="linear"):
	Cells = []
	for s in size:
	Cells.append([Cell() for i in range(0,s)])
	self.Cells = Cells
	self.Output_Calc = Output(output_activation)


	def getRandomInit(self,scaleW = 0.1,scaleB = 0.1):
	Ws = []
	Bs = []
	for i in range(0,len(self.Cells)-1):
	leftsize = len(self.Cells[i])
	rightsize = len(self.Cells[i+1])
	Ws.append(np.random.randn(rightsize,leftsize)*scaleW)
	Bs.append(np.random.randn(rightsize)*scaleB)
	return (Ws,Bs)


	#全セルへの中間入力と、出力層の結果を返す
	def forward(self,in_data,Ws,Bs,drop=0.0):
	in_list = []
	next_in = in_data
	in_list.append(next_in)
	for i in range(0,len(Ws)):
	next_in = [self.Cells[i][j].activate(next_in[j]) for j in range(0,len(next_in))]
	next_in = np.dot(Ws[i],next_in)
	next_in = next_in+Bs[i]
	#for i in range(0,len(next_in)):
	# if(np.random.rand() < drop):
	# next_in[i] = 0

	in_list.append(next_in)
	output = self.Output_Calc.output_calc(next_in)
	return (in_list,output)

	def update(self,Ws,Bs,in_list,output,delta=0.1,answervector = None,VerBose = False):

	dEdnextin = None
	Ws_fixed = []
	Bs_fixed= []
	dlossdW = []
	dlossdB = []
	for i in reversed(range(1,len(self.Cells))):
	if(i == len(self.Cells)-1):
	#損失関数を出力層セルへの入力で偏微分したもの dE/dO
	dEdin = self.Output_Calc.dlossdin(output,answervector)
	else :
	#損失関数を中間層セルへの入力で偏微分したもの dE/dCnow = ΣdE/dCnext * φ'(Cnow) * WCnow->Cnext
	dEdin = np.dot(Ws[i].T,dEdnextin).T #ΣdE/dCnext*WCnow->Cnext
	gradlist = []
	for cellnum in range(0,len(in_list[i])):
	gradlist.append(self.Cells[i][cellnum].grad(in_list[i][cellnum]))
	dEdin = dEdingradlist# ΣdE/dCnext φ'(Cnow) * WCnow->Cnext

	#損失関数をWで偏微分したもの dE/dWpre_now = =dE/dWCnow * Cpre
	pre_out = np.array([self.Cells[i-1][j].activate(in_list[i-1][j]) for j in range(0,len(in_list[i-1]))])
	#print(pre_out.shape)
	dlossdWpre_now = np.multiply(np.matrix(dEdin).T,np.array(pre_out))
	#print(dlossdWpre_now.shape)
	dlossdBpre_now = dEdin
	dEdnextin = copy.copy(dEdin)

	Ws_fixed.insert(0,Ws[i-1] - delta*np.array((dlossdWpre_now)))
	Bs_fixed.insert(0,Bs[i-1] - delta*np.array(dlossdBpre_now))
	return (Ws_fixed,Bs_fixed)





	# In[61]:



	def Learn(layer,input_func,answervector_func,num=1000,delta=0.01,VerBose = False,initWs=None,initBs=None):
	if(initWs == None):
	Ws,Bs = layer.getRandomInit()
	else:
	Ws = initWs
	Bs= initBs

	#pprint.pprint(Ws)
	#pprint.pprint(Bs)
	print("\n")

	for i in range(0,num):
	#print("try"+str(i))
	inputs = input_func(i)
	#print(inputs)
	(in_list,output) = layer.forward(inputs,Ws,Bs)
	if(VerBose):
	print("in_list")
	print(in_list)
	newWs,newBs = layer.update(Ws,Bs,in_list,output,delta=delta,answervector= answervector_func(i))
	if(VerBose):
	print("newWs")
	#pprint.pprint(newWs)
	print("newBs")
	#pprint.pprint(newBs)
	Ws = newWs
	Bs = newBs

	#print("in")
	#pprint.pprint(inputs)
	#print("out")
	#pprint.pprint(output)

	#print("\n")

	return(Ws,Bs)



	# In[62]:


	layer = Layer([2,100,4],"relu","softmax")
	times = 100
	data = [np.random.randint(2,size=2) for x in range(0,times)]
	testdata_func = lambda i : data[i]

	code = [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]
	answervector_func = lambda i : code[data[i][1]*2+data[i][0]]
	(Ws,Bs) = Learn(layer,testdata_func,answervector_func,times,VerBose=False)


	# In[63]:


	in_list,out = layer.forward([1,1],Ws,Bs)
	print(out)
	in_list,out = layer.forward([0,1],Ws,Bs)
	print(out)
	in_list,out = layer.forward([1,0],Ws,Bs)
	print(out)
	in_list,out = layer.forward([0,0],Ws,Bs)
	print(out)


	# In[64]:


	TrainingSampleNum = 2000 # 学習サンプル総数
	TestSampleNum = 10000 # テストサンプル総数
	ClassNum = 10 # クラス数（今回は10）
	ImageSize = 28 # 画像サイズ（今回は縦横ともに28）
	TrainingDataFile = './Images/TrainingSamples/{0:1d}-{1:04d}.png'
	TestDataFile = './Images/TestSamples/{0:1d}-{1:04d}.png'
	OutFile = './Images/OutSamples/gray_{0:1d}-{1:04d}.png'


	# In[65]:


	from skimage import io


	# In[66]:


	# LoadTemplates
	def LoadTemplates () :
	labels = np.zeros( TrainingSampleNum, dtype=np.uint8 )
	templates = np.zeros( (TrainingSampleNum,ImageSize,ImageSize), dtype=np.uint8 )

	i = 0
	for label in range (0, ClassNum):
	for sample in range (0, TrainingSampleNum // ClassNum ):
	filename = TrainingDataFile.format(label,sample)
	templates[i,:,:] = io.imread(filename)
	labels[i]=label
	i += 1

	return templates, labels


	# In[67]:


	from scipy.ndimage.interpolation import shift
	templates, labels = LoadTemplates ()

	templates = templates/255
	templates = templates.reshape(2000,784)


	#indices = np.hstack((indices,indices))


	# In[68]:


	def MnistLearn(layer,initWs=None,initBs=None):
	global templates
	global labels

	cpytemplates = copy.copy(templates)

	#for i in range(0,len(cpytemplates)):
	#dx = np.random.randint(-3,4)
	#dy = np.random.randint(-3,4)
	#cpytemplates[i] = shift(cpytemplates[i],[dx,dy],cval=0)


	numinputs = len(cpytemplates)
	indices = list(range(0,numinputs))
	np.random.shuffle(indices)
	#print(indices)
	times = len(indices)
	testdata_func = lambda i : cpytemplates[indices[i]]
	answervector_func = lambda i : np.identity(10)[labels[indices[i]]]
	(Ws,Bs) = Learn(layer,testdata_func,answervector_func,times,VerBose=False,delta=0.001,initWs=initWs,initBs=initBs)
	return (Ws,Bs)


	# In[69]:


	X_test = []
	y_test = []

	each_class_num = TestSampleNum // ClassNum

	for label in range (0, ClassNum):
	for sample in range (0, each_class_num ):
	filename = TestDataFile.format(label,sample)
	t_img = io.imread ( filename ) /255
	#print(type(t_img))
	X_test.append(t_img.flatten())
	y_test.append(label)


	def counting(layer,Ws,Bs):
	conf_mat = np.zeros((10,10))
	count = 0
	results = np.zeros ( (ClassNum, ClassNum) )
	each_class_num = TestSampleNum // ClassNum
	#print("1")
	for ind in range(0,len(X_test)):
	#print(t_img)
	(_,output) = layer.forward(X_test[ind],Ws,Bs)
	#output = np.array([0,0,0,0,0,0,0,0,0,0])
	#print(label)
	#print(output)
	i = output.argmax()
	#print(i)
	if(y_test[ind] == i):
	count += 1
	#print(count)
	conf_mat[y_test[ind],i] += 1
	#print("\n")
	return (conf_mat,count)





	# In[73]:


	import warnings
	warnings.filterwarnings('error')

	layer = Layer([28*28,512,512,10],"relu","softmax")
	(Ws,Bs) = layer.getRandomInit()
	print(Ws,Bs)
	for i in range(0,100):
	print("epoch:"+str(i))
	(Ws,Bs) = MnistLearn(layer,Ws,Bs)
	if(i % 10 == 0):
	mat,count = counting(layer,Ws,Bs)
	pprint.pprint(mat)
	print(count)
	#print("a")


	# In[ ]:


	mat,count = counting(layer,Ws,Bs)


	# In[74]:


	pprint.pprint(mat)
	print(count)


	# In[75]:


	for i in range(0,10):
	print("epoch:"+str(i))
	(Ws,Bs) = MnistLearn(layer,Ws,Bs)
	mat,count = counting(layer,Ws,Bs)
	pprint.pprint(mat)
	print(count)