Skip to content

Instantly share code, notes, and snippets.

@takatakamanbou takatakamanbou/00_convnet.md Secret
Last active Sep 6, 2016

Embed
What would you like to do?
Learn more about clone URLs
Download ZIP
convnet.py
Raw
convnet150907.py
import numpy as np
import theano
import theano.tensor as T
import theano.tensor.signal.downsample as Tsd
import nnet150903 as nnet
########## Input Layer for 4D tensor inputs ##########
class T4InputLayer( object ):
def __init__( self, Xdim, dropout = 1.0 ):
self.Xshape = Xdim # Xnch, Xrow, Xcol
self.dropout = dropout
def Top_output( self, X ):
if self.dropout < 1.0:
return X * self.dropout
else:
return X
def Top_generateMask( self, rng ):
return rng.uniform( self.Xshape ) <= self.dropout
def Top_outputMasked( self, X, mask ):
return X * mask
########## Convolution Layer ##########
class ConvLayer( object ):
def __init__( self, Xdim, Wdim, afunc, withBias = True, Wini = 0.01,
border_mode = 'valid', dropout = 1.0 ):
# dimension of the input
Xnch, Xrow, Xcol = Xdim
self.Xshape = Xdim
# dimension of the convolution filters
Wnch, Wrow, Wcol = Wdim
self.Wshape = ( Wnch, Xnch, Wrow, Wcol )
# dimension of the output
if border_mode == 'valid':
Yrow, Ycol = Xrow - Wrow + 1, Xcol - Wcol + 1
else:
Yrow, Ycol = Xrow + Wrow - 1, Xcol + Wcol - 1
self.Yshape = ( Wnch, Yrow, Ycol )
self.Dout = Wnch * Yrow * Ycol
# activation function of the layer
self.afunc = afunc
self.withBias = withBias
# border mode
assert border_mode in [ 'valid', 'full' ]
self.border_mode = border_mode
# dropout
self.dropout = dropout
# theano shared variables
floatX = theano.config.floatX
W = Wini * np.random.standard_normal( self.Wshape )
self.W = theano.shared( np.asarray( W, dtype = floatX ) )
self.dW = theano.shared( np.zeros( self.Wshape, dtype = floatX ) )
if withBias:
self.b = theano.shared( np.zeros( Wnch, dtype = floatX ) )
self.db = theano.shared( np.zeros( Wnch, dtype = floatX ) )
'''
def Tfunc_setWeight( self ):
# under construction
'''
def getWeight( self ):
W = self.W.get_value()
if self.withBias:
b = self.b.get_value()
return [ W, b ]
else:
return W
def Top_outputRaw( self, X ):
# X: Ndat x Xshape, Y: Ndat x Yshape
Xs = ( None, self.Xshape[0], self.Xshape[1], self.Xshape[2] )
Ws = self.Wshape
Y = T.nnet.conv.conv2d( X, self.W, image_shape = Xs, filter_shape = Ws, border_mode = self.border_mode )
if self.withBias:
b = self.b.dimshuffle( 'x', 0, 'x', 'x' ) # 1 x nch x 1 x 1
Y += b
Z = nnet.d_afunc[self.afunc]( Y )
return Y, Z # Ndat x Yshape
def Top_output( self, X ):
Y, Z = self.Top_outputRaw( X )
if self.dropout < 1.0:
Z *= self.dropout
return Y, Z
def Top_generateMask( self, rng ):
return rng.uniform( ( self.Yshape ) ) <= self.dropout
def Top_outputMasked( self, X, mask ):
Y, Z = self.Top_outputRaw( X )
return Y, Z * mask
def T_update( self, cost, eta, mu, lam ):
gradW = T.grad( cost, self.W )
dWnew = -eta * ( gradW + lam * self.W ) + mu * self.dW
Wnew = self.W + dWnew
upList = [ ( self.W, Wnew ), ( self.dW, dWnew ) ]
if self.withBias:
gradb = T.grad( cost, self.b )
# no weight decay for bias
dbnew = -eta * gradb + mu * self.db
bnew = self.b + dbnew
upList += [ ( self.b, bnew ), ( self.db, dbnew ) ]
return upList
########## Pooling Layer ##########
class PoolLayer( object ):
def __init__( self, Xdim, ds, st = None, dropout = 1.0 ):
# dimension of the input
Xnch, Xrow, Xcol = Xdim
self.Xshape = Xdim
# parameters of the pooling layer
self.ds = ds
self.st = st
self.ignore_border = False
rv = Tsd.DownsampleFactorMax.out_shape( self.Xshape, ds, ignore_border = self.ignore_border, st = st )
#self.Yshape = ( Xnch, rv[1], rv[2] )
self.Yshape = tuple( rv )
self.Dout = np.prod( self.Yshape )
# dropout
self.dropout = dropout
def Top_outputRaw( self, X ):
# X: Ndat x Xshape
Y = Tsd.max_pool_2d( X, self.ds, ignore_border = self.ignore_border, st = self.st ) # Ndat x Yshape
return Y, Y
def Top_output( self, X ):
Y, Y = self.Top_outputRaw( X )
if self.dropout < 1.0:
Y *= self.dropout
return Y, Y
def Top_generateMask( self, rng ):
return rng.uniform( ( self.Yshape ) ) <= self.dropout
def Top_outputMasked( self, X, mask ):
Y, Y = self.Top_outputRaw( X )
Ymask = Y * mask
return Ymask, Ymask
########## Full-Connection Layer ##########
class FullLayer( nnet.Layer ):
def __init__( self, Din, Nunit, afunc, withBias = True, Wini = 0.01,
dropout = 1.0, T4toMat = False ):
super( FullLayer, self ).__init__( Din, Nunit, afunc, withBias, Wini, dropout )
self.T4toMat = T4toMat
def super_Top_outputRaw( self, X ):
return super( FullLayer, self ).Top_outputRaw( X )
def Top_outputRaw( self, X ):
if self.T4toMat:
return self.super_Top_outputRaw( X.reshape( ( X.shape[0], -1 ) ) )
else:
return self.super_Top_outputRaw( X )
########## Convolutional Neural Net ##########
class CNN( object ):
def __init__( self, Layers, rng = None ):
# layers - list of Layer instances
self.Layers = Layers
assert isinstance( Layers[0], T4InputLayer )
# random number generator
if rng == None:
self.rng = nnet.randomstreams( 0 )
else:
self.rng = rng
# theano functions
self.output = self.Tfunc_output()
self.cost = self.Tfunc_cost()
self.train = self.Tfunc_train()
# theano op for output computation ( for test )
def Top_output( self, X ):
# input layer
layer = self.Layers[0]
Zprev = layer.Top_output( X )
# hidden layers
for layer in self.Layers[1:]:
Y, Z = layer.Top_output( Zprev )
Zprev = Z
# output
Zsoftmax = T.nnet.softmax( Zprev )
return Zsoftmax
# theano function for output computation ( for test )
def Tfunc_output( self ):
X = T.tensor4() # Ndat x Xnch x Xrow x Xcol
Z = self.Top_output( X )
return theano.function( [ X ] , Z )
# theano op for cost computation ( error term )
def Top_cost( self, Z, lab ):
cost = T.nnet.categorical_crossentropy( Z, lab )
return T.mean( cost )
# theano function for cost computation
def Tfunc_cost( self ):
Z = T.matrix() # N x K
lab = T.ivector() # N-dim
return theano.function( [ Z, lab ], self.Top_cost( Z, lab ) )
# theano function for gradient descent learning
def Tfunc_train( self ):
X = T.tensor4( 'X' )
lab = T.ivector( 'lab' )
eta = T.scalar( 'eta' )
mu = T.scalar( 'mu' )
lam = T.scalar( 'lambda' )
# input layer
layer = self.Layers[0]
if layer.dropout < 1.0:
mask = layer.Top_generateMask( self.rng )
Zprev = layer.Top_outputMasked( X, mask )
else:
Zprev = layer.Top_output( X )
# hidden layers
for layer in self.Layers[1:]:
if layer.dropout < 1.0:
mask = layer.Top_generateMask( self.rng )
Y, Z = layer.Top_outputMasked( Zprev, mask )
else:
Y, Z = layer.Top_output( Zprev )
Zprev = Z
# output & cost
Z = T.nnet.softmax( Zprev )
cost = self.Top_cost( Z, lab )
# updatesList
updatesList = []
for layer in self.Layers:
if isinstance( layer, T4InputLayer ) or isinstance( layer, PoolLayer ):
continue
updatesList += layer.T_update( cost, eta, mu, lam )
return theano.function( [ X, lab, eta, mu, lam ], [ Z, cost ], updates = updatesList )
Raw
convnet160323.py
import numpy as np
import theano
import theano.tensor as T
import theano.tensor.signal.pool as Tsp
import nnet151219 as nnet
########## Input Layer for 4D tensor inputs ##########
class T4InputLayer( object ):
def __init__( self, Xdim, rng = None, dropout = 1.0 ):
self.Xshape = Xdim # Xnch, Xrow, Xcol
self.dropout = dropout
if rng == None:
self.rng = nnet.randomstreams( 0 )
else:
self.rng = rng
def Top_outputTrain( self, X ):
if self.dropout < 1.0:
mask = self.rng.uniform( self.Xshape ) <= self.dropout
return X * mask
else:
return X
def Top_outputInference( self, X ):
if self.dropout < 1.0:
return X * self.dropout
else:
return X
########## Convolution Layer ##########
class ConvLayer( object ):
def __init__( self, Xdim, Wdim, afunc, rng = None, withBias = True,
Wini = 0.01, border_mode = 'valid', dropout = 1.0 ):
# dimension of the input
Xnch, Xrow, Xcol = Xdim
self.Xshape = Xdim
# dimension of the convolution filters
Wnch, Wrow, Wcol = Wdim
self.Wshape = ( Wnch, Xnch, Wrow, Wcol )
# dimension of the output
if border_mode == 'valid':
Yrow, Ycol = Xrow - Wrow + 1, Xcol - Wcol + 1
else:
Yrow, Ycol = Xrow + Wrow - 1, Xcol + Wcol - 1
self.Yshape = ( Wnch, Yrow, Ycol )
self.Dout = Wnch * Yrow * Ycol
# activation function of the layer
self.afunc = afunc
self.withBias = withBias
# border mode
assert border_mode in [ 'valid', 'full' ]
self.border_mode = border_mode
# dropout
self.dropout = dropout
# random number generator
if rng == None:
self.rng = nnet.randomstreams( 0 )
else:
self.rng = rng
# theano shared variables
floatX = theano.config.floatX
W = Wini * np.random.standard_normal( self.Wshape )
#self.W = theano.shared( np.zeros( self.Wshape, dtype = floatX ) )
self.W = theano.shared( np.ones( self.Wshape, dtype = floatX ) )
self.dW = theano.shared( np.zeros( self.Wshape, dtype = floatX ) )
if withBias:
self.b = theano.shared( np.zeros( Wnch, dtype = floatX ) )
self.db = theano.shared( np.zeros( Wnch, dtype = floatX ) )
# theano functions
self.initWeight = self.Tfunc_initWeight()
self.setWeight = self.Tfunc_setWeight()
# weight initialization
self.initWeight( Wini )
def Tfunc_initWeight( self ):
Wini = T.scalar()
#W = self.rng.normal( self.Wshape, avg = 0.0, std = Wini )
''' Using the next line & 'W = np.ones' in __init__ to prevent the error raised by the above.'''
W = self.W * self.rng.normal( self.Wshape, avg = 0.0, std = Wini )
inList = [ Wini ]
upList = [ ( self.W, W ) ]
return theano.function( inList, None, updates = upList )
def Tfunc_setWeight( self ):
W = T.tensor4()
if self.withBias:
b = T.vector()
inList = [ W, b ]
upList = [ ( self.W, W ), ( self.b, b ) ]
else:
inList = [ W ]
upList = [ ( self.W, W ) ]
return theano.function( inList, None, updates = upList )
def getWeight( self ):
W = self.W.get_value()
if self.withBias:
b = self.b.get_value()
return [ W, b ]
else:
return W
def Top_outputRaw( self, X ):
# X: Ndat x Xshape, Y: Ndat x Yshape
Xs = ( None, self.Xshape[0], self.Xshape[1], self.Xshape[2] )
Ws = self.Wshape
Y = T.nnet.conv2d( X, self.W, input_shape = Xs, filter_shape = Ws, border_mode = self.border_mode )
if self.withBias:
b = self.b.dimshuffle( 'x', 0, 'x', 'x' ) # 1 x nch x 1 x 1
Y += b
Z = nnet.d_afunc[self.afunc]( Y )
return Y, Z # Ndat x Yshape
def Top_outputTrain( self, X ):
Y, Z = self.Top_outputRaw( X )
if self.dropout < 1.0:
mask = self.rng.uniform( self.Yshape ) <= self.dropout
return Y, Z * mask
else:
return Y, Z
def Top_outputInference( self, X ):
Y, Z = self.Top_outputRaw( X )
if self.dropout < 1.0:
return Y, Z * self.dropout
else:
return Y, Z
def T_update( self, cost, eta, mu, lam ):
gradW = T.grad( cost, self.W )
dWnew = -eta * ( gradW + lam * self.W ) + mu * self.dW
Wnew = self.W + dWnew
upList = [ ( self.W, Wnew ), ( self.dW, dWnew ) ]
if self.withBias:
gradb = T.grad( cost, self.b )
# no weight decay for bias
dbnew = -eta * gradb + mu * self.db
bnew = self.b + dbnew
upList += [ ( self.b, bnew ), ( self.db, dbnew ) ]
return upList
########## Pooling Layer ##########
class PoolLayer( object ):
def __init__( self, Xdim, ds, st = None, rng = None, dropout = 1.0 ):
# dimension of the input
Xnch, Xrow, Xcol = Xdim
self.Xshape = Xdim
# parameters of the pooling layer
self.ds = ds
self.st = st
#self.ignore_border = False
self.ignore_border = True
rv = Tsp.Pool.out_shape( self.Xshape, ds, ignore_border = self.ignore_border, st = st )
self.Yshape = tuple( rv )
self.Dout = np.prod( self.Yshape )
# dropout
self.dropout = dropout
# random number generator
if rng == None:
self.rng = nnet.randomstreams( 0 )
else:
self.rng = rng
def Top_outputRaw( self, X ):
# X: Ndat x Xshape
Y = Tsp.pool_2d( X, self.ds, ignore_border = self.ignore_border, st = self.st ) # Ndat x Yshape
return Y, Y
def Top_outputTrain( self, X ):
Y, Y = self.Top_outputRaw( X )
if self.dropout < 1.0:
mask = self.rng.uniform( self.Yshape ) <= self.dropout
Ymask = Y * mask
return Ymask, Ymask
else:
return Y, Y
def Top_outputInference( self, X ):
Y, Y = self.Top_outputRaw( X )
if self.dropout < 1.0:
Ymask = Y * self.dropout
return Ymask, Ymask
else:
return Y, Y
########## Full-Connection Layer ##########
class FullLayer( nnet.Layer ):
def __init__( self, Din, Nunit, afunc, rng = None, withBias = True,
Wini = 0.01, dropout = 1.0, T4toMat = False ):
super( FullLayer, self ).__init__( Din, Nunit, afunc, rng = rng, withBias = withBias, Wini = Wini, dropout = dropout )
self.T4toMat = T4toMat
def super_Top_outputRaw( self, X ):
return super( FullLayer, self ).Top_outputRaw( X )
def Top_outputRaw( self, X ):
if self.T4toMat:
return self.super_Top_outputRaw( X.reshape( ( X.shape[0], -1 ) ) )
else:
return self.super_Top_outputRaw( X )
########## Convolutional Neural Net ##########
class CNN( object ):
def __init__( self, Layers ):
# layers - list of Layer instances
self.Layers = Layers
assert isinstance( Layers[0], T4InputLayer )
# theano functions
self.output = self.Tfunc_output()
self.cost = self.Tfunc_cost()
self.train = self.Tfunc_train()
# theano op for output computation ( for inference )
def Top_output( self, X ):
# input layer
layer = self.Layers[0]
Zprev = layer.Top_outputInference( X )
# hidden layers
for layer in self.Layers[1:]:
Y, Z = layer.Top_outputInference( Zprev )
Zprev = Z
# output
Zsoftmax = T.nnet.softmax( Zprev )
return Zsoftmax
# theano function for output computation ( for inference )
def Tfunc_output( self ):
X = T.tensor4() # Ndat x Xnch x Xrow x Xcol
Z = self.Top_output( X )
return theano.function( [ X ] , Z )
# theano op for cost computation ( error term )
def Top_cost( self, Z, lab ):
cost = T.nnet.categorical_crossentropy( Z, lab )
return T.mean( cost )
# theano function for cost computation
def Tfunc_cost( self ):
Z = T.matrix() # N x K
lab = T.ivector() # N-dim
return theano.function( [ Z, lab ], self.Top_cost( Z, lab ) )
# theano function for gradient descent learning
def Tfunc_train( self ):
X = T.tensor4( 'X' )
lab = T.ivector( 'lab' )
eta = T.scalar( 'eta' )
mu = T.scalar( 'mu' )
lam = T.scalar( 'lambda' )
# input layer
layer = self.Layers[0]
Zprev = layer.Top_outputTrain( X )
# hidden layers
for layer in self.Layers[1:]:
Y, Z = layer.Top_outputTrain( Zprev )
Zprev = Z
# output & cost
Z = T.nnet.softmax( Zprev )
cost = self.Top_cost( Z, lab )
# updatesList
updatesList = []
for layer in self.Layers:
if isinstance( layer, T4InputLayer ) or isinstance( layer, PoolLayer ):
continue
updatesList += layer.T_update( cost, eta, mu, lam )
return theano.function( [ X, lab, eta, mu, lam ], [ Z, cost ], updates = updatesList )
Raw
convnet160327.py
from __future__ import print_function
import numpy as np
import theano
import theano.tensor as T
import theano.tensor.signal.pool as Tsp
import nnet151219 as nnet
########## Input Layer for 4D tensor inputs ##########
class T4InputLayer( object ):
def __init__( self, Xdim, rng = None, dropout = 1.0 ):
self.Xshape = Xdim # Xnch, Xrow, Xcol
self.dropout = dropout
if rng == None:
self.rng = nnet.randomstreams( 0 )
else:
self.rng = rng
def Top_outputTrain( self, X ):
if self.dropout < 1.0:
mask = self.rng.uniform( self.Xshape ) <= self.dropout
return X * mask
else:
return X
def Top_outputInference( self, X ):
if self.dropout < 1.0:
return X * self.dropout
else:
return X
########## Convolution Layer ##########
class ConvLayer( object ):
def __init__( self, Xdim, Wdim, afunc, withBias = True, Wini = 0.01,
rng = None, dropout = 1.0, **kwargs4conv2d ):
# dimension of the input
Xnch, Xrow, Xcol = Xdim
self.Xshape = Xdim
# dimension of the convolution filters
Wnch, Wrow, Wcol = Wdim
self.Wshape = ( Wnch, Xnch, Wrow, Wcol )
# activation function & bias
self.afunc = afunc
self.withBias = withBias
# random number generator
if rng == None:
self.rng = nnet.randomstreams( 0 )
else:
self.rng = rng
# dropout
self.dropout = dropout
# keyword arguments for conv2d
kwargs4conv2d['input_shape'] = ( None, Xnch, Xrow, Xcol )
kwargs4conv2d['filter_shape'] = self.Wshape
if not 'border_mode' in kwargs4conv2d:
kwargs4conv2d['border_mode'] = 'valid'
if not 'subsample' in kwargs4conv2d:
kwargs4conv2d['subsample'] = ( 1, 1 )
if not 'filter_flip' in kwargs4conv2d:
kwargs4conv2d['filter_flip'] = True
self.kwargs4conv2d = kwargs4conv2d
# dimension of the output
args = ( self.kwargs4conv2d['input_shape'],
self.kwargs4conv2d['filter_shape'],
self.kwargs4conv2d['border_mode'],
self.kwargs4conv2d['subsample'] )
rv = T.nnet.abstract_conv.get_conv_output_shape( *args )
self.Yshape = rv[1:] # Wnch, Yrow, Ycol
self.Dout = np.prod( self.Yshape )
# theano shared variables
floatX = theano.config.floatX
W = Wini * np.random.standard_normal( self.Wshape )
#self.W = theano.shared( np.zeros( self.Wshape, dtype = floatX ) )
self.W = theano.shared( np.ones( self.Wshape, dtype = floatX ) )
self.dW = theano.shared( np.zeros( self.Wshape, dtype = floatX ) )
if withBias:
self.b = theano.shared( np.zeros( Wnch, dtype = floatX ) )
self.db = theano.shared( np.zeros( Wnch, dtype = floatX ) )
# theano functions
self.initWeight = self.Tfunc_initWeight()
self.setWeight = self.Tfunc_setWeight()
# weight initialization
self.initWeight( Wini )
def Tfunc_initWeight( self ):
Wini = T.scalar()
#W = self.rng.normal( self.Wshape, avg = 0.0, std = Wini )
''' Using the next line & 'W = np.ones' in __init__ to prevent the error raised by the above.'''
W = self.W * self.rng.normal( self.Wshape, avg = 0.0, std = Wini )
inList = [ Wini ]
upList = [ ( self.W, W ) ]
return theano.function( inList, None, updates = upList )
def Tfunc_setWeight( self ):
W = T.tensor4()
if self.withBias:
b = T.vector()
inList = [ W, b ]
upList = [ ( self.W, W ), ( self.b, b ) ]
else:
inList = [ W ]
upList = [ ( self.W, W ) ]
return theano.function( inList, None, updates = upList )
def getWeight( self ):
W = self.W.get_value()
if self.withBias:
b = self.b.get_value()
return [ W, b ]
else:
return W
def Top_outputRaw( self, X ):
Y = T.nnet.conv2d( X, self.W, **self.kwargs4conv2d )
if self.withBias:
b = self.b.dimshuffle( 'x', 0, 'x', 'x' ) # 1 x nch x 1 x 1
Y += b
Z = nnet.d_afunc[self.afunc]( Y )
return Y, Z # Ndat x Yshape
def Top_outputTrain( self, X ):
Y, Z = self.Top_outputRaw( X )
if self.dropout < 1.0:
mask = self.rng.uniform( self.Yshape ) <= self.dropout
return Y, Z * mask
else:
return Y, Z
def Top_outputInference( self, X ):
Y, Z = self.Top_outputRaw( X )
if self.dropout < 1.0:
return Y, Z * self.dropout
else:
return Y, Z
def T_update( self, cost, eta, mu, lam ):
gradW = T.grad( cost, self.W )
dWnew = -eta * ( gradW + lam * self.W ) + mu * self.dW
Wnew = self.W + dWnew
upList = [ ( self.W, Wnew ), ( self.dW, dWnew ) ]
if self.withBias:
gradb = T.grad( cost, self.b )
# no weight decay for bias
dbnew = -eta * gradb + mu * self.db
bnew = self.b + dbnew
upList += [ ( self.b, bnew ), ( self.db, dbnew ) ]
return upList
########## Pooling Layer ##########
class PoolLayer( object ):
def __init__( self, Xdim, ds, rng = None, dropout = 1.0, **kwargs4pool_2d ):
# dimension of the input
Xnch, Xrow, Xcol = Xdim
self.Xshape = Xdim
# downscaling factor
self.ds = ds
# dropout
self.dropout = dropout
# random number generator
if rng == None:
self.rng = nnet.randomstreams( 0 )
else:
self.rng = rng
# keyword arguments for pool_2d
if not 'ignore_border' in kwargs4pool_2d:
kwargs4pool_2d['ignore_border'] = True
if kwargs4pool_2d['ignore_border'] == False:
print( '# PoolLayer warning: ignore_border is set to False' )
if not 'st' in kwargs4pool_2d:
kwargs4pool_2d['st'] = None
if not 'padding' in kwargs4pool_2d:
kwargs4pool_2d['padding'] = ( 0, 0 )
if not 'mode' in kwargs4pool_2d:
kwargs4pool_2d['mode'] = 'max'
if kwargs4pool_2d['mode'] != 'max':
print( '# PoolLayer warning: mode != max' )
self.kwargs4pool_2d = kwargs4pool_2d
# dimension of the output
kwargs = {}
for kn in ( 'ignore_border', 'st', 'padding' ):
kwargs[kn] = self.kwargs4pool_2d[kn]
rv = Tsp.Pool.out_shape( self.Xshape, ds, **kwargs )
self.Yshape = tuple( rv )
self.Dout = np.prod( self.Yshape )
def Top_outputRaw( self, X ):
# X: Ndat x Xshape
Y = Tsp.pool_2d( X, self.ds, **self.kwargs4pool_2d ) # Ndat x Yshape
return Y, Y
def Top_outputTrain( self, X ):
Y, Y = self.Top_outputRaw( X )
if self.dropout < 1.0:
mask = self.rng.uniform( self.Yshape ) <= self.dropout
Ymask = Y * mask
return Ymask, Ymask
else:
return Y, Y
def Top_outputInference( self, X ):
Y, Y = self.Top_outputRaw( X )
if self.dropout < 1.0:
Ymask = Y * self.dropout
return Ymask, Ymask
else:
return Y, Y
########## Full-Connection Layer ##########
class FullLayer( nnet.Layer ):
def __init__( self, Din, Nunit, afunc, rng = None, withBias = True,
Wini = 0.01, dropout = 1.0, T4toMat = False ):
super( FullLayer, self ).__init__( Din, Nunit, afunc, rng = rng, withBias = withBias, Wini = Wini, dropout = dropout )
self.T4toMat = T4toMat
def super_Top_outputRaw( self, X ):
return super( FullLayer, self ).Top_outputRaw( X )
def Top_outputRaw( self, X ):
if self.T4toMat:
return self.super_Top_outputRaw( X.reshape( ( X.shape[0], -1 ) ) )
else:
return self.super_Top_outputRaw( X )
########## Convolutional Neural Net ##########
class CNN( object ):
def __init__( self, Layers ):
# layers - list of Layer instances
self.Layers = Layers
assert isinstance( Layers[0], T4InputLayer )
# theano functions
self.output = self.Tfunc_output()
self.cost = self.Tfunc_cost()
self.train = self.Tfunc_train()
# theano op for output computation ( for inference )
def Top_output( self, X ):
# input layer
layer = self.Layers[0]
Zprev = layer.Top_outputInference( X )
# hidden layers
for layer in self.Layers[1:]:
Y, Z = layer.Top_outputInference( Zprev )
Zprev = Z
# output
Zsoftmax = T.nnet.softmax( Zprev )
return Zsoftmax
# theano function for output computation ( for inference )
def Tfunc_output( self ):
X = T.tensor4() # Ndat x Xnch x Xrow x Xcol
Z = self.Top_output( X )
return theano.function( [ X ] , Z )
# theano op for cost computation ( error term )
def Top_cost( self, Z, lab ):
cost = T.nnet.categorical_crossentropy( Z, lab )
return T.mean( cost )
# theano function for cost computation
def Tfunc_cost( self ):
Z = T.matrix() # N x K
lab = T.ivector() # N-dim
return theano.function( [ Z, lab ], self.Top_cost( Z, lab ) )
# theano function for gradient descent learning
def Tfunc_train( self ):
X = T.tensor4( 'X' )
lab = T.ivector( 'lab' )
eta = T.scalar( 'eta' )
mu = T.scalar( 'mu' )
lam = T.scalar( 'lambda' )
# input layer
layer = self.Layers[0]
Zprev = layer.Top_outputTrain( X )
# hidden layers
for layer in self.Layers[1:]:
Y, Z = layer.Top_outputTrain( Zprev )
Zprev = Z
# output & cost
Z = T.nnet.softmax( Zprev )
cost = self.Top_cost( Z, lab )
# updatesList
updatesList = []
for layer in self.Layers:
if isinstance( layer, T4InputLayer ) or isinstance( layer, PoolLayer ):
continue
updatesList += layer.T_update( cost, eta, mu, lam )
return theano.function( [ X, lab, eta, mu, lam ], [ Z, cost ], updates = updatesList )
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.