00_dAEbyTensorFlow.md

TensorFlow で denoising autoencoder

• TensorFlow で MLP & CNN ver.20170415
• TensorFlow で MLP & CNN ver.20170817

ver. 20170817

• http://www-tlab.math.ryukoku.ac.jp/tlab/index.php?takataka%2Fnote%2F2017-08-17

• http://takatakamanbou.hatenablog.com/entry/2017/08/18/185356

• ex170817daeL.py & ex170817daeT.py

  • linearAE170817.py 実験で使う autoencoder のネットワーク構造の定義（線形）
  • reluAE170817.py 実験で使う autoencoder のネットワーク構造の定義（ReLU）
  • mnist.py MNISTデータの読み込み
  • getnoisy170817.py MNIST データの前処理，ノイズ付加など
  • ex170817datL-result.txt 実験結果

ver. 20170729

古い

• http://www-tlab.math.ryukoku.ac.jp/tlab/index.php?takataka%2Fnote%2F2017-07-29

• ex170729dAE_L.py & ex170729dAE_T.py

  • autoencoder170729.py ニューラルネット識別器 nnclassifier170729.py の AutoEncoder 版
  • ae170729.py 実験で使う autoencoder のネットワーク構造の定義
  • mnist.py MNISTデータの読み込み
  • mnist_sub170729.py MNIST データの前処理，ノイズ付加など

ae170729.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import tensorflow.contrib as tfc


class LinearAE(object):

  def __init__(self, Xdim):

    self.inputs = tf.placeholder(tf.float32, shape = [None, Xdim])
    params = {'units':1000, 'activation':None, 'use_bias':False,
              'kernel_initializer':tfc.layers.xavier_initializer(),
              'bias_initializer':None}
    self.hidden1 = tf.layers.dense(inputs = self.inputs, **params)
    params = {'units':Xdim, 'activation':None, 'use_bias':False,
              'kernel_initializer':tfc.layers.xavier_initializer(),
              'bias_initializer':None}
    self.outputs = tf.layers.dense(inputs = self.hidden1, **params)


class ReLUAE(object):

  def __init__(self, Xdim):

    self.inputs = tf.placeholder(tf.float32, shape = [None, Xdim])
    params = {'units':1000, 'activation':tf.nn.relu, 'use_bias':True,
              'kernel_initializer':tfc.layers.xavier_initializer(),
              'bias_initializer':tf.zeros_initializer()}
    self.hidden1 = tf.layers.dense(inputs = self.inputs, **params)
    params = {'units':Xdim, 'activation':None, 'use_bias':False,
              'kernel_initializer':tfc.layers.xavier_initializer(),
              'bias_initializer':None}
    self.outputs = tf.layers.dense(inputs = self.hidden1, **params)


class ReLUAE3(object):

  def __init__(self, Xdim):

    self.inputs = tf.placeholder(tf.float32, shape = [None, Xdim])
    params = {'units':1000, 'activation':tf.nn.relu, 'use_bias':True,
              'kernel_initializer':tfc.layers.xavier_initializer(),
              'bias_initializer':tf.zeros_initializer()}
    self.hidden1 = tf.layers.dense(inputs = self.inputs, **params)
    self.hidden2 = tf.layers.dense(inputs = self.hidden1, **params)
    params = {'units':Xdim, 'activation':None, 'use_bias':False,
              'kernel_initializer':tfc.layers.xavier_initializer(),
              'bias_initializer':None}
    self.outputs = tf.layers.dense(inputs = self.hidden2, **params)

autoencoder170729.py

rom __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import tensorflow.contrib as tfc


class AutoEncoder():

    def __init__(self, netcfg, optimizer = None):

        ### definition for output computation
        #
        self.X = netcfg.inputs
        self.Z = netcfg.outputs
        self.cg_output = self.Z

        ### definition for cost
        #
        self.Zt = tf.placeholder(tf.float32, shape = self.X.shape)
        self.sqe = tf.reduce_sum(tf.squared_difference(self.Z, self.Zt), axis = 1)
        cost = tf.reduce_mean(self.sqe)
        #cost = tf.losses.mean_squared_error(self.Zt, self.Z)
        self.cg_test = cost

        ### definition for training
        #
        self.optimizer = optimizer
        if optimizer != None:
            self.cg_train = self.optimizer.minimize(cost)

        ### definition for parameter initialization
        #
        self.cg_init = tf.global_variables_initializer()

        ### starting the session
        #
        self.sess = tf.InteractiveSession()

        
    def init(self):

        rv = self.sess.run(self.cg_init)
        return rv

  
    def output(self, X):

        d = {self.X: X}
        rv = self.sess.run(self.cg_output, feed_dict = d)

        return rv


    def train(self, X, Zt):

        d = {self.X: X, self.Zt: Zt}
        rv = self.sess.run(self.cg_train, feed_dict = d)
        
        return rv


    def test(self, X, Zt):

        d = {self.X: X, self.Zt: Zt}
        rv = self.sess.run(self.cg_test, feed_dict = d)
        
        return rv

  
    def save(self, path):

        saver = tf.train.Saver()
        saver.save(self.sess, path)


    def restore(self, path):
        
        saver = tf.train.Saver()
        saver.restore(self.sess, path)

ex170729dAE_L.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import numpy as np

import mnist
import mnist_sub170729 as mnist_sub

import ae170729 as ae
import autoencoder170729 as autoencoder

def addNoise( X ):

    #Xnoisy = mnist_sub.addSaltPepperNoise( X, 0.1 )

    #Xnoisy = mnist_sub.shift( X, 2 )

    X2     = mnist_sub.shift( X, 2 )
    Xnoisy = mnist_sub.addSaltPepperNoise( X2, 0.1 )

    return Xnoisy


if __name__ == '__main__':
  

    ### reading and preparing the training data
    #
    mn = mnist.MNIST(pathMNIST = '../../150117-mnist')
    XL, labL, XV, labV = mnist_sub.getDataLV(mn)
    D = XL.shape[1]
    xm = np.mean(XL, axis = 0)  # mean subtraction is not performed here
    NL = XL.shape[0]
    NV = XV.shape[0]

    XLnoisy = addNoise(XL)
    XVnoisy = addNoise(XV)
        
    ### initializing the network
    #
    #nncfg = ae.LinearAE(D)
    #nncfg = ae.ReLUAE(D)
    nncfg = ae.ReLUAE3(D)
        
    eta = 0.001
    mu = 0.9
    optimizer = tf.train.MomentumOptimizer(eta, mu)
    nn = autoencoder.AutoEncoder(nncfg, optimizer)
    nn.init()

    print('# nncfg:', nncfg.__class__)
    print('# nn:', nn.__class__)
        
    ### training
    #
    batchsize = 100
    idxB = mnist_sub.makeBatchIndex(NL, batchsize)
    nbatch = idxB.shape[0]

    print('#     sqeL sqeV')
    
    nitr = 20
    for i in range(nitr):
        if i <= 10 or i % 10 == 0:
            sqeL = nn.test(XLnoisy - xm, XL - xm)
            sqeV = nn.test(XVnoisy - xm, XV - xm)
            print('%d  %.2f  %.2f' % (i, sqeL, sqeV))
        for ib in np.random.permutation(nbatch):
            ii = idxB[ib, :]
            Xin = XLnoisy[ii] - xm
            Zt  = XL[ii] - xm
            nn.train(Xin, Zt)

    nn.save('dAE')

  

ex170729dAE_T.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import numpy as np
import cv2

import mnist
import mnist_sub170729 as mnist_sub

import ae170729 as ae
import autoencoder170729 as autoencoder
#import ex170729i
import ex170729dAE_L

def addNoise(X):

    return ex170729dAE_L.addNoise(X)
    

if __name__ == '__main__':
  
    ### reading and preparing the training data
    #
    mn = mnist.MNIST(pathMNIST = '../../150117-mnist')
    XL, labL, XV, labV = mnist_sub.getDataLV(mn)
    D = XL.shape[1]
    xm = np.mean(XL, axis = 0)  # mean subtraction is not performed here

    XT, labT = mnist_sub.getDataT(mn)
    NL = XL.shape[0]
    NV = XV.shape[0]
    NT = XT.shape[0]

    XLnoisy = addNoise(XL)
    XVnoisy = addNoise(XV)
    XTnoisy = addNoise(XT)
        
    ### initializing the network
    #
    #nncfg = ae.LinearAE(D)
    #nncfg = ae.ReLUAE(D)
    nncfg = ae.ReLUAE3(D)
    nn = autoencoder.AutoEncoder(nncfg)
    nn.restore('dAE')

    print('# nncfg:', nncfg.__class__)
    print('# nn:', nn.__class__)

    ### test
    #
    sqeL = nn.test(XLnoisy - xm, XL - xm)
    sqeV = nn.test(XVnoisy - xm, XV - xm)
    sqeT = nn.test(XTnoisy - xm, XT - xm)
    print('%.2f  %.2f  %.2f' % (sqeL, sqeV, sqeT))
  
    nx, ny = 10, 5
    Xin = addNoise(XL[:nx*ny]) - xm
    Z = nn.output(Xin) + xm

    img = mnist_sub.visualize(255*(Xin+xm), nx, ny)
    cv2.imwrite('X.png', img)

    img = mnist_sub.visualize(255*Z, nx, ny)
    cv2.imwrite('Z.png', img)

ex170817daeL-result.txt

In [12]: %time %run ex170817daeL.py /gpu:0     # GeForce GTX 1080
# <linearAE170817.LinearAE object at 0x7f406f0eb390>
#     sqeL   sqeV
0 0 134.667979551 133.316992969
100 10000 36.8419604199 37.4187404297
200 20000 34.7458608545 35.4624817627
300 30000 34.5803067285 35.331162207
400 40000 34.3753529346 35.1260460693
500 50000 34.1441767871 34.9703582764
1000 100000 34.0689229346 34.8956889404
1500 150000 33.8404356641 34.5914494385
2000 200000 33.5332493701 34.3849062012
2500 250000 33.3508528955 34.152530542
3000 300000 33.3689177295 34.1587166504
3500 350000 33.0659213672 33.8709844727
4000 400000 32.947115957 33.7168620605
4500 450000 32.9565215332 33.7343044189
5000 500000 32.9019669678 33.6755770996
5500 550000 32.9164221191 33.6913230225
6000 600000 32.8613183398 33.6579268066
6500 650000 32.8997485889 33.6877947266
7000 700000 32.9398715869 33.7341349365
7500 750000 32.7980756152 33.588655249
8000 800000 32.8414224219 33.6326763184
8500 850000 32.8726254932 33.7171114502
9000 900000 32.9318401709 33.7842064697
9500 950000 32.8239343945 33.6591986816
10000 1000000 32.839681875 33.6528670166
CPU times: user 1min 30s, sys: 8.85 s, total: 1min 39s
Wall time: 1min 3s

In [64]: %run ex170817daeT.py
# <linearAE170817.LinearAE object at 0x159a1c908>
# sqeL sqeV sqeT
32.84  33.65  33.47

In [5]: %time %run ex170817daeL.py /gpu:0     # GeForce GTX 1080
# <reluAE170817.ReLUAE object at 0x7f405c543748>
#     sqeL   sqeV
0 0 94.1615285938 93.177168457
100 10000 32.5207284521 32.9867842285
200 20000 26.6384485205 27.5547973145
300 30000 23.7986352856 25.0091011963
400 40000 22.1090672559 23.5483374756
500 50000 20.8981465259 22.6398469971
1000 100000 18.2016388818 20.9610388306
1500 150000 16.8670344238 20.4675469238
2000 200000 15.9073584021 20.2453117798
2500 250000 15.4007348523 20.3088085571
3000 300000 14.9222910107 20.3802709229
3500 350000 14.5632112354 20.4816333984
4000 400000 14.2205513818 20.5549073242
4500 450000 13.9771353027 20.6636340088
5000 500000 13.8773649451 20.8368718262
5500 550000 13.6312556006 20.8960320557
6000 600000 13.4619291199 21.0348988403
6500 650000 13.3087240979 21.1374243042
7000 700000 13.257743269 21.2749579956
7500 750000 13.1724190149 21.4199958252
8000 800000 13.0830199683 21.5523608887
8500 850000 12.9690893665 21.6853352417
9000 900000 12.9208887622 21.7975457397
9500 950000 12.7863040588 21.8200642578
10000 1000000 12.740861156 21.9447665527
CPU times: user 1min 29s, sys: 8.13 s, total: 1min 37s
Wall time: 1min 3s

In [65]: %run ex170817daeT.py
# <reluAE170817.ReLUAE object at 0x187637160>
# sqeL sqeV sqeT
12.74  21.94  21.73

In [1]: %time %run ex170817daeL.py /gpu:0     # GeForce GTX 1080
# <reluAE3170817.ReLUAE3 object at 0x7f40a4c268d0>
#     sqeL   sqeV
0 0 74.7140612988 73.9807450684
100 10000 29.0753343896 29.2979303711
200 20000 24.0497760742 24.6967222412
300 30000 21.5675712671 22.4639517578
400 40000 20.1020632715 21.2733384888
500 50000 19.0573139941 20.4971976196
1000 100000 16.394145835 18.8423206055
1500 150000 14.9872722656 18.3109795898
2000 200000 13.8976362622 18.0409084717
2500 250000 13.211768667 18.0200608154
3000 300000 12.6059124866 18.0257837036
3500 350000 12.0897991577 18.1371610229
4000 400000 11.6678141748 18.1972028809
4500 450000 11.3611565991 18.2901698975
5000 500000 11.1345474158 18.4935889038
5500 550000 10.8190674414 18.5200787598
6000 600000 10.5923006726 18.5990333496
6500 650000 10.343986781 18.7050669678
7000 700000 10.2723618799 18.8658595459
7500 750000 10.1506798145 19.0366557495
8000 800000 10.009450448 19.1695141479
8500 850000 9.79464683472 19.2149471313
9000 900000 9.74745519775 19.3498310303
9500 950000 9.6134655481 19.4565813354
10000 1000000 9.5610569104 19.5329217651
CPU times: user 1min 39s, sys: 9.54 s, total: 1min 49s
Wall time: 1min 10s

In [66]: %run ex170817daeT.py
# <reluAE3170817.ReLUAE3 object at 0x16cfd76d8>
# sqeL sqeV sqeT
9.56  19.53  19.43

ex170817daeL.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import numpy as np
import sys
import os

#AEtype = 'linear'
AEtype = 'relu'
#AEtype = 'relu3'

import getnoisy170817 as getnoisy
import linearAE170817 as linearAE
import reluAE170817 as reluAE
import reluAE3170817 as reluAE3


def evaluate(nn, X, Zt, bindex):

    nbatch, ndat = bindex.shape
    sqebuf = np.empty(nbatch)
    for ib in range(nbatch):
        ii = bindex[ib, :]
        sqebuf[ib] = np.sum(nn.test(X[ii], Zt[ii]))

    return np.sum(sqebuf)/ndat
    

if __name__ == '__main__':

    dirResult = 'result_ex170817'

    np.random.seed(0)

    ###  /gpu:0  GTX 1080,  /gpu:1  Tesla K20c  on tortoise3
    #
    if len(sys.argv) == 1:
        dev = '/cpu:0'
    elif len(sys.argv) == 2:
        dev = sys.argv[1]
    else:
        sys.exit('usage: %s [device]' % sys.argv[0])

    
    ### reading and preparing the training data
    #
    data = getnoisy.Data('../150117-mnist', nV = 10000)
    D = data.nrow * data.ncol
    XL = data.getData('L')
    XV = data.getData('V')
    NL = XL.shape[0]
    NV = XV.shape[0]
    xm = np.mean(XL, axis = 0)
    XnoisyL = getnoisy.addNoise(XL) - xm  # noisy input
    XnoisyV = getnoisy.addNoise(XV) - xm
    ZtL = XL - xm  # output teacher
    ZtV = XV - xm
        
    ### initializing the network
    #
    #eta = 0.01
    #mu = 0.9
    #optimizer = tf.train.MomentumOptimizer(eta, mu)
    optimizer = tf.train.AdamOptimizer(learning_rate = 0.001)

    g = tf.Graph()
    with g.as_default():
        with g.device(dev):
            if AEtype == 'linear':
                nn = linearAE.LinearAE(D, 1000, optimizer)
            elif AEtype == 'relu':
                nn = reluAE.ReLUAE(D, 1000, optimizer)
            elif AEtype == 'relu3':
                nn = reluAE3.ReLUAE3(D, 1000, 1000, optimizer)

    print('#', nn)
    nn.init()


    ### training
    #
    batchsize = 100
    bindexL = getnoisy.makeBatchIndex(NL, batchsize)
    nbatchL = bindexL.shape[0]
    bindexV = getnoisy.makeBatchIndex(NV, batchsize)

    print('#     sqeL   sqeV')
    
    nitr = 10000
    nd = 0

    for i in range(nitr+1):
        
        if (i < 500 and i % 100 == 0) or (i % 500 == 0):
            sqeL = evaluate(nn, XnoisyL, ZtL, bindexL)
            sqeV = evaluate(nn, XnoisyV, ZtV, bindexV)
            print(i, nd, sqeL, sqeV)

        if i == nitr:
            break
            
        ib = np.random.randint(0, nbatchL)
        ii = bindexL[ib, :]
        nn.train(XnoisyL[ii], ZtL[ii])
        nd += XnoisyL[ii].shape[0]


    fnParams = os.path.join(dirResult, os.path.splitext(sys.argv[0])[0] + '-params.npz')
    params_dict = nn.getWeight()
    np.savez_compressed(fnParams, **params_dict)

ex170817daeT.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import numpy as np
import cv2
import sys
import os

import getnoisy170817 as getnoisy
import linearAE170817 as linearAE
import reluAE170817 as reluAE
import reluAE3170817 as reluAE3
import ex170817daeL


if __name__ == '__main__':

    dirResult = 'result_ex170817'

    np.random.seed(0)

    ### reading and preparing the training data
    #
    data = getnoisy.Data('../150117-mnist', nV = 10000)
    D = data.nrow * data.ncol
    XL = data.getData('L')
    XV = data.getData('V')
    XT = data.getData('T')
    NL = XL.shape[0]
    NV = XV.shape[0]
    NT = XV.shape[0]
    xm = np.mean(XL, axis = 0)
    XnoisyL = getnoisy.addNoise(XL) - xm  # noisy input
    XnoisyV = getnoisy.addNoise(XV) - xm
    XnoisyT = getnoisy.addNoise(XT) - xm
    ZtL = XL - xm  # output teacher
    ZtV = XV - xm
    ZtT = XT - xm

    ### initializing the network
    #
    g = tf.Graph()
    with g.as_default():
        if ex170817daeL.AEtype == 'linear':
            nn = linearAE.LinearAE(D, 1000)
        elif ex170817daeL.AEtype == 'relu':
            nn = reluAE.ReLUAE(D, 1000)
        elif ex170817daeL.AEtype == 'relu3':
            nn = reluAE3.ReLUAE3(D, 1000, 1000)

    fnParams = os.path.join(dirResult, 'ex170817daeL-params.npz')
    with np.load(fnParams) as hoge:
        nn.setWeight(hoge)

    print('#', nn)

    
    ### reconstruction
    #
    nx, ny = 10, 5
    Xin = XnoisyT[:nx*ny]
    img = getnoisy.visualize(255*(Xin+xm), nx, ny)
    cv2.imwrite(os.path.join(dirResult, 'Xnoisy.png'), img)
    Z = nn.output(Xin)
    img = getnoisy.visualize(255*(Z+xm), nx, ny)
    cv2.imwrite(os.path.join(dirResult, 'Z.png'), img)
    Xtrue = XT[:nx*ny]
    img = getnoisy.visualize(255*Xtrue, nx, ny)
    cv2.imwrite(os.path.join(dirResult, 'Xtrue.png'), img)


    ### test
    #
    batchsize = 100
    bindexL = getnoisy.makeBatchIndex(NL, batchsize)
    bindexV = getnoisy.makeBatchIndex(NV, batchsize)
    bindexT = getnoisy.makeBatchIndex(NT, batchsize)
    sqeL = ex170817daeL.evaluate(nn, XnoisyL, ZtL, bindexL)
    sqeV = ex170817daeL.evaluate(nn, XnoisyV, ZtV, bindexV)
    sqeT = ex170817daeL.evaluate(nn, XnoisyT, ZtT, bindexT)
    print('# sqeL sqeV sqeT')
    print('%.2f  %.2f  %.2f' % (sqeL, sqeV, sqeT))

getnoisy170817.py

import numpy as np
import mnist

class Data(object):

    def __init__(self, pathMNIST, nV = 0):

        self.mnist = mnist.MNIST(pathMNIST = pathMNIST)
        self.nrow = self.mnist.nrow
        self.ncol = self.mnist.ncol
        self.nclass = self.mnist.nclass

        self.nV = nV
        #self._getLV()
        #self._getT()

        
    def _getLV(self):

        X   = self.mnist.getImage('L') / 255
        self.nL = X.shape[0] - self.nV
        self.XL = X[:self.nL]
        self.XV = X[self.nL:]

        
    def _getT(self):

        X = self.mnist.getImage('T') / 255
        self.XT = X
        self.nT = X.shape[0]

        
    def getData(self, LVT):

        if LVT == 'L':
            if not hasattr(self, 'XL'):
                self._getLV()
            return self.XL
        elif LVT == 'V':
            if not hasattr(self, 'XV'):
                self._getLV()
            return self.XV
        else:            
            if not hasattr(self, 'XT'):
                self._getT()
            return self.XT



def makeBatchIndex(N, batchsize):
    
    idx = np.random.permutation(N)
    nbatch = int(np.ceil(float(N) / batchsize))
    idxB = np.zeros((nbatch, N), dtype = bool)
    for ib in range(nbatch - 1):
        idxB[ib, idx[ib*batchsize:(ib+1)*batchsize]] = True
    ib = nbatch - 1
    idxB[ib, idx[ib*batchsize:]] = True

    return idxB


def addSaltPepperNoise(Xbatch, p):

    Xnoisy = np.copy(Xbatch)
    pvec = np.random.random_sample(Xbatch.shape[0] * Xbatch.shape[1])
    idxB = pvec < p/2
    idxW = ( p/2 <= pvec ) * ( pvec < p )
    Xnoisy2 = Xnoisy.reshape( -1 )
    Xnoisy2[idxB] = 0.0
    Xnoisy2[idxW] = 1.0

    return Xnoisy


def shift(Xbatch, dmax):

    nrow = ncol = 28
    N = Xbatch.shape[0]
    delta = np.random.randint(-dmax, dmax + 1, ( N, 2 ))
    Xnoisy = np.zeros_like(Xbatch)
    for i in range(N):
        src = Xbatch[i, :].reshape((nrow, ncol))
        dst = np.zeros((nrow + 2*dmax, ncol + 2*dmax))
        dx, dy = delta[i, 0], delta[i, 1]
        dst[dmax+dy:dmax+dy+nrow, dmax+dx:dmax+dx+ncol] = src
        Xnoisy[i, :] = dst[dmax:dmax+nrow, dmax:dmax+ncol].reshape(-1)

    return Xnoisy


def addNoise(Xbatch):

    #Xnoisy = addSaltPepperNoise(Xbatch, 0.1)

    #Xnoisy = shift(Xbatch, 2)

    X2     = shift(Xbatch, 2)
    Xnoisy = addSaltPepperNoise(X2, 0.1)

    return Xnoisy

        

def visualize(src, nx, ny, nrow = 28, ncol = 28, gap = 4):

    src2 = src.reshape((ny, nx, nrow, ncol))
    
    w = nx * ( ncol + gap ) + gap
    h = ny * ( nrow + gap ) + gap
    img = np.zeros((h, w), dtype = int) + 128

    for iy in range(ny):
        lty = iy * ( nrow + gap ) + gap
        for ix in range(nx):
            ltx = ix * ( ncol + gap ) + gap
            img[lty:lty+nrow, ltx:ltx+ncol] = src2[iy, ix, :, :]

    return img

linearAE170817.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import tensorflow.contrib as tfc
import numpy as np


class LinearAE(object):

  def __init__(self, Xdim, Hdim, optimizer = None):

    self.X = tf.placeholder(tf.float32, shape = [None, Xdim])
    self.Xdim = Xdim
    self.Hdim = Hdim
    self.params = dict()

    ### definition of the network
    
    ### fc1: input => hidden
    #
    Wshape = [Xdim, Hdim]
    W = tf.get_variable('fc1/W', shape = Wshape, initializer = tfc.layers.xavier_initializer())
    fc1 = tf.matmul(self.X, W)
    self.params['fc1/W'] = W

    ### fc2: hidden => output
    #
    Wshape = [Hdim, Xdim]
    W = tf.get_variable('fc2/W', shape = Wshape, initializer = tfc.layers.xavier_initializer())
    fc2 = tf.matmul(fc1, W)
    self.params['fc2/W'] = W


    ### definition for output computation
    #
    self.Z = fc2
    self.cg_output = self.Z

    ### definition for cost
    #
    self.Zt = tf.placeholder(tf.float32, shape = self.X.shape)
    self.sqe = tf.reduce_sum(tf.squared_difference(self.Z, self.Zt), axis = 1)
    self.cg_test = self.sqe
    cost = tf.reduce_mean(self.sqe)
    
    ### definition for training
    #
    self.optimizer = optimizer
    if optimizer != None:
      self.cg_train = self.optimizer.minimize(cost)

    ### definition for parameter initialization
    #
    self.cg_init = tf.global_variables_initializer()

    ### starting the session
    #
    self.sess = tf.InteractiveSession()


  def init(self):

    rv = self.sess.run(self.cg_init)
    return rv

  
  def output(self, X):

    d = {self.X: X}
    rv = self.sess.run(self.cg_output, feed_dict = d)
    return rv


  def train(self, X, Zt):

    d = {self.X: X, self.Zt: Zt}
    rv = self.sess.run(self.cg_train, feed_dict = d)
    return rv


  def test(self, X, Zt):

    d = {self.X: X, self.Zt: Zt}
    rv = self.sess.run(self.cg_test, feed_dict = d)
    return rv


  def getWeight(self):

    return self.sess.run(self.params)


  def setWeight(self, vals_dict):

    L = []
    for k in vals_dict.keys():
      L.append(tf.assign(self.params[k], vals_dict[k]))
    self.sess.run(L)
    
    
if __name__ == '__main__':

  with tf.Graph().as_default():
    ae = LinearAE(768, 100)

  ae.init()

mnist_sub170729.py

import numpy as np
import mnist


def getDataLV(mn):

    X = mn.getImage('L') / 255.0
    lab = mn.getLabel('L')
    XL, labL = X[:50000], lab[:50000]
    XV, labV = X[50000:], lab[50000:]

    return XL, labL, XV, labV


def getDataT(mn):

    XT = mn.getImage('T') / 255.0
    labT = mn.getLabel('T')

    return XT, labT


def makeBatchIndex(N, batchsize):
    
    idx = np.random.permutation(N)
    nbatch = int(np.ceil(float(N) / batchsize))
    idxB = np.zeros((nbatch, N), dtype = bool)
    for ib in range(nbatch - 1):
        idxB[ib, idx[ib*batchsize:(ib+1)*batchsize]] = True
    ib = nbatch - 1
    idxB[ib, idx[ib*batchsize:]] = True

    return idxB


def addSaltPepperNoise(Xbatch, p):

    Xnoisy = np.copy(Xbatch)
    pvec = np.random.random_sample(Xbatch.shape[0] * Xbatch.shape[1])
    idxB = pvec < p/2
    idxW = ( p/2 <= pvec ) * ( pvec < p )
    Xnoisy2 = Xnoisy.reshape( -1 )
    Xnoisy2[idxB] = 0.0
    Xnoisy2[idxW] = 1.0

    return Xnoisy


def shift(Xbatch, dmax):

    nrow = ncol = 28
    N = Xbatch.shape[0]
    delta = np.random.randint(-dmax, dmax + 1, ( N, 2 ))
    Xnoisy = np.zeros_like(Xbatch)
    for i in range(N):
        src = Xbatch[i, :].reshape((nrow, ncol))
        dst = np.zeros((nrow + 2*dmax, ncol + 2*dmax))
        dx, dy = delta[i, 0], delta[i, 1]
        dst[dmax+dy:dmax+dy+nrow, dmax+dx:dmax+dx+ncol] = src
        Xnoisy[i, :] = dst[dmax:dmax+nrow, dmax:dmax+ncol].reshape(-1)

    return Xnoisy
        


def visualize(src, nx, ny, nrow = 28, ncol = 28, gap = 4):

    src2 = src.reshape((ny, nx, nrow, ncol))
    
    w = nx * ( ncol + gap ) + gap
    h = ny * ( nrow + gap ) + gap
    img = np.zeros((h, w), dtype = int) + 128

    for iy in range(ny):
        lty = iy * ( nrow + gap ) + gap
        for ix in range(nx):
            ltx = ix * ( ncol + gap ) + gap
            img[lty:lty+nrow, ltx:ltx+ncol] = src2[iy, ix, :, :]

    return img

reluAE170817.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import tensorflow as tf
import tensorflow.contrib as tfc
import numpy as np


class ReLUAE(object):

  def __init__(self, Xdim, Hdim, optimizer = None):

    self.X = tf.placeholder(tf.float32, shape = [None, Xdim])
    self.Xdim = Xdim
    self.Hdim = Hdim
    self.params = dict()

    ### definition of the network
    
    ### fc1
    #
    Wshape = [Xdim, Hdim]
    W = tf.get_variable('fc1/W', shape = Wshape, initializer = tfc.layers.xavier_initializer())
    b = tf.get_variable('fc1/b', shape = Wshape[-1], initializer = tf.zeros_initializer)
    fc1 = tf.nn.relu(tf.matmul(self.X, W) + b)
    self.params['fc1/W'] = W
    self.params['fc1/b'] = b

    ### fc2
    #
    Wshape = [Hdim, Xdim]
    W = tf.get_variable('fc2/W', shape = Wshape, initializer = tfc.layers.xavier_initializer())
    #b = tf.get_variable('fc2/b', shape = Wshape[-1], initializer = tf.zeros_initializer)
    #fc2 = tf.matmul(fc1, W) + b
    fc2 = tf.matmul(fc1, W)
    self.params['fc2/W'] = W
    #self.params['fc2/b'] = b


    ### definition for output computation
    #
    self.Z = fc2
    self.cg_output = self.Z

    ### definition for cost
    #
    self.Zt = tf.placeholder(tf.float32, shape = self.X.shape)
    self.sqe = tf.reduce_sum(tf.squared_difference(self.Z, self.Zt), axis = 1)
    self.cg_test = self.sqe
    cost = tf.reduce_mean(self.sqe)
    
    ### definition for training
    #
    self.optimizer = optimizer
    if optimizer != None:
      self.cg_train = self.optimizer.minimize(cost)

    ### definition for parameter initialization
    #
    self.cg_init = tf.global_variables_initializer()

    ### starting the session
    #
    self.sess = tf.InteractiveSession()


  def init(self):

    rv = self.sess.run(self.cg_init)
    return rv

  
  def output(self, X):

    d = {self.X: X}
    rv = self.sess.run(self.cg_output, feed_dict = d)
    return rv


  def train(self, X, Zt):

    d = {self.X: X, self.Zt: Zt}
    rv = self.sess.run(self.cg_train, feed_dict = d)
    return rv


  def test(self, X, Zt):

    d = {self.X: X, self.Zt: Zt}
    rv = self.sess.run(self.cg_test, feed_dict = d)
    return rv


  def getWeight(self):

    return self.sess.run(self.params)


  def setWeight(self, vals_dict):

    L = []
    for k in vals_dict.keys():
      L.append(tf.assign(self.params[k], vals_dict[k]))
    self.sess.run(L)
    
    
if __name__ == '__main__':

  with tf.Graph().as_default():
    ae = ReLUAE(768, 100)

  ae.init()