Skip to content

Instantly share code, notes, and snippets.

@takatakamanbou

takatakamanbou/00_dAEbyTensorFlow2.md Secret

Last active September 17, 2017 11:55
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save takatakamanbou/01a529f38ac40f39a343c2bf619dccb6 to your computer and use it in GitHub Desktop.
Save takatakamanbou/01a529f38ac40f39a343c2bf619dccb6 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
00_dAEbyTensorFlow2.md

TensorFlow で denoising autoencoder (CNN版)

Raw
cnnAE170917.py
import tensorflow as tf
import tensorflow.contrib as tfc
import numpy as np
class CNNAE(object):
def __init__(self, Xshape, optimizer = None):
self.X = tf.placeholder(tf.float32, shape = [None] + Xshape)
self.Xshape = Xshape
self.params = dict()
self.tensors = dict()
### starting the session
#
self.sess = tf.InteractiveSession()
### definition of the network
H1 = 32
H2 = 2*H1
H3 = 2*H2
H4 = H2
H5 = H1
H6 = 1
ksize = 4
BN_offset = None
BN_scale = None
BN_eps = 0.001
Z = self.X
### conv1
#
lname = 'conv1'
Wshape = [ksize, ksize, 1, H1]
och = Wshape[-1]
with tf.variable_scope(lname):
W = tf.get_variable('W', shape = Wshape, initializer = tfc.layers.xavier_initializer_conv2d())
#b = tf.get_variable('b', shape = och, initializer = tf.zeros_initializer)
mean = tf.get_variable('mean', shape = och, initializer = tf.zeros_initializer)
var = tf.get_variable('var', shape = och, initializer = tf.ones_initializer)
WX = tf.nn.conv2d(Z, W, [1, 1, 1, 1], padding = 'SAME')
Y = tf.nn.batch_normalization(WX, mean, var, BN_offset, BN_scale, BN_eps)
#Z = tf.nn.relu(Y + b)
Z = tf.nn.relu(Y)
self.params[W.name] = W
#self.params[b.name] = b
self.params[mean.name] = mean
self.params[var.name] = var
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### pool1
#
lname = 'pool1'
Wshape = [1, 2, 2, 1]
Z = tf.nn.max_pool(Z, Wshape, [1, 2, 2, 1], padding = 'SAME')
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### conv2
#
lname = 'conv2'
Wshape = [ksize, ksize, H1, H2]
och = Wshape[-1]
with tf.variable_scope(lname):
W = tf.get_variable('W', shape = Wshape, initializer = tfc.layers.xavier_initializer_conv2d())
mean = tf.get_variable('mean', shape = och, initializer = tf.zeros_initializer)
var = tf.get_variable('var', shape = och, initializer = tf.ones_initializer)
WX = tf.nn.conv2d(Z, W, [1, 1, 1, 1], padding = 'SAME')
Y = tf.nn.batch_normalization(WX, mean, var, BN_offset, BN_scale, BN_eps)
Z = tf.nn.relu(Y)
self.params[W.name] = W
self.params[mean.name] = mean
self.params[var.name] = var
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### pool2
#
lname = 'pool2'
Wshape = [1, 2, 2, 1]
Z = tf.nn.max_pool(Z, Wshape, [1, 2, 2, 1], padding = 'SAME')
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### conv3
#
lname = 'conv3'
Wshape = [ksize, ksize, H2, H3]
och = Wshape[-1]
with tf.variable_scope(lname):
W = tf.get_variable('W', shape = Wshape, initializer = tfc.layers.xavier_initializer_conv2d())
mean = tf.get_variable('mean', shape = och, initializer = tf.zeros_initializer)
var = tf.get_variable('var', shape = och, initializer = tf.ones_initializer)
WX = tf.nn.conv2d(Z, W, [1, 1, 1, 1], padding = 'SAME')
Y = tf.nn.batch_normalization(WX, mean, var, BN_offset, BN_scale, BN_eps)
Z = tf.nn.relu(Y)
self.params[W.name] = W
self.params[mean.name] = mean
self.params[var.name] = var
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### pool3
#
lname = 'pool3'
Wshape = [1, 2, 2, 1]
Z = tf.nn.max_pool(Z, Wshape, [1, 2, 2, 1], padding = 'SAME')
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### conv4 (T)
#
lname = 'conv4_T'
Wshape = [ksize, ksize, H4, H3] # [h, w, och, ich]
och = Wshape[-2]
oshape = [tf.shape(Z)[0], 7, 7, och]
with tf.variable_scope(lname):
W = tf.get_variable('W', shape = Wshape, initializer = tfc.layers.xavier_initializer_conv2d())
mean = tf.get_variable('mean', shape = och, initializer = tf.zeros_initializer)
var = tf.get_variable('var', shape = och, initializer = tf.ones_initializer)
WX = tf.nn.conv2d_transpose(Z, W, output_shape = tf.stack(oshape), strides = [1, 2, 2, 1], padding = 'SAME')
Y = tf.nn.batch_normalization(WX, mean, var, BN_offset, BN_scale, BN_eps)
Z = tf.nn.relu(Y)
Z.set_shape([None] + oshape[1:])
self.params[W.name] = W
self.params[mean.name] = mean
self.params[var.name] = var
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### conv5 (T)
#
lname = 'conv5_T'
Wshape = [4, 4, H5, H4] # [h, w, och, ich]
och = Wshape[-2]
oshape = [tf.shape(Z)[0], 14, 14, och]
with tf.variable_scope(lname):
W = tf.get_variable('W', shape = Wshape, initializer = tfc.layers.xavier_initializer_conv2d())
mean = tf.get_variable('mean', shape = och, initializer = tf.zeros_initializer)
var = tf.get_variable('var', shape = och, initializer = tf.ones_initializer)
WX = tf.nn.conv2d_transpose(Z, W, output_shape = tf.stack(oshape), strides = [1, 2, 2, 1], padding = 'SAME')
Y = tf.nn.batch_normalization(WX, mean, var, BN_offset, BN_scale, BN_eps)
Z = tf.nn.relu(Y)
Z.set_shape([None] + oshape[1:])
self.params[W.name] = W
self.params[mean.name] = mean
self.params[var.name] = var
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### conv6 (T)
#
lname = 'conv6_T'
Wshape = [4, 4, H6, H5] # [h, w, och, ich]
och = Wshape[-2]
oshape = [tf.shape(Z)[0], 28, 28, och]
with tf.variable_scope(lname):
W = tf.get_variable('W', shape = Wshape, initializer = tfc.layers.xavier_initializer_conv2d())
Y = tf.nn.conv2d_transpose(Z, W, output_shape = tf.stack(oshape), strides = [1, 2, 2, 1], padding = 'SAME')
Z = Y
Z.set_shape([None] + oshape[1:])
self.params[W.name] = W
self.tensors[lname] = Z
print('# ' + lname + ':', Z.get_shape().as_list())
### definition for output computation
#
self.Z = Z
self.cg_output = self.Z
### definition for cost
#
self.Zt = tf.placeholder(tf.float32, shape = self.X.shape)
self.sqe = tf.reduce_sum(tf.squared_difference(self.Z, self.Zt), axis = [1, 2])
self.cg_test = self.sqe
cost = tf.reduce_mean(self.sqe)
### definition for training
#
self.optimizer = optimizer
if optimizer != None:
self.cg_train = self.optimizer.minimize(cost)
### definition for parameter initialization
#
self.cg_init = tf.global_variables_initializer()
def init(self):
rv = self.sess.run(self.cg_init)
return rv
def output(self, X):
d = {self.X: X}
rv = self.sess.run(self.cg_output, feed_dict = d)
return rv
def train(self, X, Zt):
d = {self.X: X, self.Zt: Zt}
rv = self.sess.run(self.cg_train, feed_dict = d)
return rv
def test(self, X, Zt):
d = {self.X: X, self.Zt: Zt}
rv = self.sess.run(self.cg_test, feed_dict = d)
return rv
def getWeight(self):
return self.sess.run(self.params)
def setWeight(self, vals_dict):
L = []
for k in vals_dict.keys():
L.append(tf.assign(self.params[k], vals_dict[k]))
self.sess.run(L)
if __name__ == '__main__':
with tf.Graph().as_default():
nn = CNNAE([28, 28, 1])
print('#', nn)
print('#', nn.params)
nn.init()
Raw
ex170917daeL.py
import tensorflow as tf
import numpy as np
import sys
import os
import getnoisy170817 as getnoisy
import cnnAE170917 as cnnAE
def evaluate(nn, X, Zt, bindex):
nbatch, ndat = bindex.shape
sqebuf = np.empty(nbatch)
for ib in range(nbatch):
ii = bindex[ib, :]
sqebuf[ib] = np.sum(nn.test(X[ii], Zt[ii]))
return np.sum(sqebuf)/ndat
if __name__ == '__main__':
dirResult = 'result_ex170917'
np.random.seed(0)
### /gpu:0 GTX 1080, /gpu:1 Tesla K20c on tortoise3
#
if len(sys.argv) == 1:
dev = '/cpu:0'
elif len(sys.argv) == 2:
dev = sys.argv[1]
else:
sys.exit('usage: %s [device]' % sys.argv[0])
### reading and preparing the training data
#
data = getnoisy.Data('../150117-mnist', nV = 10000)
D = data.nrow * data.ncol
XL = data.getData('L')
XV = data.getData('V')
NL = XL.shape[0]
NV = XV.shape[0]
xm = np.mean(XL, axis = 0)
XnoisyL = getnoisy.addNoise(XL) - xm # noisy input
XnoisyV = getnoisy.addNoise(XV) - xm
ZtL = XL - xm # output teacher
ZtV = XV - xm
### initializing the network
#
'''
eta = 0.001
mu = 0.9
optimizer = tf.train.MomentumOptimizer(eta, mu)
'''
optimizer = tf.train.AdamOptimizer(learning_rate = 0.001)
g = tf.Graph()
with g.as_default():
with g.device(dev):
nn = cnnAE.CNNAE([28, 28, 1], optimizer)
print('#', nn)
#print('#', nn.params)
nn.init()
### training
#
batchsize = 100
bindexL = getnoisy.makeBatchIndex(NL, batchsize)
nbatchL = bindexL.shape[0]
bindexV = getnoisy.makeBatchIndex(NV, batchsize)
XnoisyLd = XnoisyL.reshape((-1, 28, 28, 1))
XnoisyVd = XnoisyV.reshape((-1, 28, 28, 1))
ZtLd = ZtL.reshape((-1, 28, 28, 1))
ZtVd = ZtV.reshape((-1, 28, 28, 1))
print('# sqeL sqeV')
nitr = 10000
nd = 0
for i in range(nitr+1):
if (i < 500 and i % 100 == 0) or (i % 500 == 0):
sqeL = evaluate(nn, XnoisyLd, ZtLd, bindexL)
sqeV = evaluate(nn, XnoisyVd, ZtVd, bindexV)
print(i, nd, sqeL, sqeV)
if i == nitr:
break
ib = np.random.randint(0, nbatchL)
ii = bindexL[ib, :]
nn.train(XnoisyLd[ii], ZtLd[ii])
nd += XnoisyL[ii].shape[0]
fnParams = os.path.join(dirResult, os.path.splitext(sys.argv[0])[0] + '-params.npz')
params_dict = nn.getWeight()
np.savez_compressed(fnParams, **params_dict)
Raw
ex170917daeLT-result.txt
< ex170917daeL >
#optimizer = tf.train.MomentumOptimizer(0.001, 0.9)
optimizer = tf.train.AdamOptimizer(learning_rate = 0.001)
ksize = 4
# conv1: [28, 28, 32]
# pool1: [14, 14, 32]
# conv2: [14, 14, 64]
# pool2: [7, 7, 64]
# conv3: [7, 7, 128]
# pool3: [4, 4, 128]
# conv4_T: [7, 7, 64]
# conv5_T: [14, 14, 32]
# conv6_T: [28, 28, 1]
# <cnnAE170917.CNNAE object at 0x7f7ea106e198>
# sqeL sqeV
0 0 52.8518796875 52.1970689453
100 10000 31.8654773975 31.9284056641
200 20000 24.7436936816 24.8462027832
300 30000 21.6042201733 21.7156482178
400 40000 19.5454629175 19.6235557861
500 50000 18.2169079565 18.3421573608
1000 100000 15.3980160791 15.7435752319
1500 150000 13.7756991382 14.2387478149
2000 200000 13.02452979 13.6217053711
2500 250000 12.5354158008 13.2798163086
3000 300000 11.8780600378 12.7432520264
3500 350000 11.740074967 12.6600704346
4000 400000 11.4871323889 12.5264744019
4500 450000 11.1293309192 12.3025267944
5000 500000 10.8680347717 12.088534314
5500 550000 10.6261997375 12.0081892944
6000 600000 10.4202412646 11.9249944946
6500 650000 10.3151186853 11.8977365967
7000 700000 10.1717956042 11.8529559082
7500 750000 10.1684105237 11.899611084
8000 800000 9.82382151123 11.6893477295
8500 850000 10.0081835339 11.9517093506
9000 900000 9.87618809814 11.8711589966
9500 950000 9.57349972656 11.6822946289
10000 1000000 9.72242623047 11.9365574951
tortoise3 + /gpu:0 GTX 1080
real 2m52.885s
user 3m16.184s
sys 0m18.124s
tortoise3: CPU だと
real 35m48.430s
user 322m24.580s
sys 35m33.868s
< ex170917daeT >
# sqeL sqeV sqeT
9.72 11.94 11.67
Raw
ex170917daeT.py
import tensorflow as tf
import numpy as np
import cv2
import sys
import os
import getnoisy170817 as getnoisy
import cnnAE170917 as cnnAE
import ex170917daeL as daeL
if __name__ == '__main__':
dirResult = 'result_ex170917'
np.random.seed(0)
### reading and preparing the training data
#
data = getnoisy.Data('../150117-mnist', nV = 10000)
D = data.nrow * data.ncol
XL = data.getData('L')
XV = data.getData('V')
XT = data.getData('T')
NL = XL.shape[0]
NV = XV.shape[0]
NT = XV.shape[0]
xm = np.mean(XL, axis = 0)
XnoisyL = getnoisy.addNoise(XL) - xm # noisy input
XnoisyV = getnoisy.addNoise(XV) - xm
XnoisyT = getnoisy.addNoise(XT) - xm
ZtL = XL - xm # output teacher
ZtV = XV - xm
ZtT = XT - xm
XnoisyLd = XnoisyL.reshape((-1, 28, 28, 1))
XnoisyVd = XnoisyV.reshape((-1, 28, 28, 1))
XnoisyTd = XnoisyT.reshape((-1, 28, 28, 1))
ZtLd = ZtL.reshape((-1, 28, 28, 1))
ZtVd = ZtV.reshape((-1, 28, 28, 1))
ZtTd = ZtT.reshape((-1, 28, 28, 1))
### initializing the network
#
g = tf.Graph()
with g.as_default():
nn = cnnAE.CNNAE([28, 28, 1])
fnParams = os.path.join(dirResult, 'ex170917daeL-params.npz')
#fnParams = os.path.join(dirResult, 'ex170917daeL-cnnAE170917d-params.npz')
with np.load(fnParams) as hoge:
nn.setWeight(hoge)
print('#', nn)
print('#', nn.params)
### reconstruction
#
nx, ny = 10, 5
Xin = XnoisyT[:nx*ny]
img = getnoisy.visualize(255*(Xin+xm), nx, ny)
cv2.imwrite(os.path.join(dirResult, 'Xnoisy.png'), img)
Zd = nn.output(Xin.reshape((-1, 28, 28, 1)))
Z = Zd.reshape((Zd.shape[0], -1))
img = getnoisy.visualize(255*(Z+xm), nx, ny)
cv2.imwrite(os.path.join(dirResult, 'Z.png'), img)
Xtrue = XT[:nx*ny]
img = getnoisy.visualize(255*Xtrue, nx, ny)
cv2.imwrite(os.path.join(dirResult, 'Xtrue.png'), img)
### test
#
batchsize = 100
bindexL = getnoisy.makeBatchIndex(NL, batchsize)
bindexV = getnoisy.makeBatchIndex(NV, batchsize)
bindexT = getnoisy.makeBatchIndex(NT, batchsize)
sqeL = daeL.evaluate(nn, XnoisyLd, ZtLd, bindexL)
sqeV = daeL.evaluate(nn, XnoisyVd, ZtVd, bindexV)
sqeT = daeL.evaluate(nn, XnoisyTd, ZtTd, bindexT)
print('# sqeL sqeV sqeT')
print('%.2f %.2f %.2f' % (sqeL, sqeV, sqeT))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment