Skip to content

Instantly share code, notes, and snippets.

@takatakamanbou takatakamanbou/cifar10_convnet0215.py Secret
Last active Aug 29, 2015

Embed
What would you like to do?
import numpy as np
import scipy as sp
import cv2
import datetime
import cifar10
import convnet0211 as convnet
def recograte( mlp, X, t, label ):
Y, Z = mlp.output( X )
mnLL = np.mean( mlp.cost( Z, t ) )
rr = np.mean( label == np.argmax( Z, axis = 1 ) )
return mnLL, rr
# Conv-Pool-Softmax
def CPS( Xnch, Xrow, Xcol, K ):
Xdim = ( Xnch, Xrow, Xcol )
W1dim = ( 16, 5, 5 )
ds1 = ( 4, 4 )
L1conv = convnet.ConvLayer( Xdim, W1dim, 'linear', withBias = False )
#L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'linear', withBias = False )
L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'ReLu', withBias = True )
H1 = L1pool.Dout
L2 = convnet.FullLayer( H1, K, 'softmax', withBias = True, T4toMat = True )
cnn = convnet.CNN( [ L1conv, L1pool, L2 ] )
print '### Conv-Pool-Softmax Xdim:', Xdim
print '# W1dim:', W1dim, ' ds1:', ds1, ' H1:', H1
return cnn
# Conv-Pool-Conv-Pool-Softmax
def CPCPS( Xnch, Xrow, Xcol, K ):
Xdim = ( Xnch, Xrow, Xcol )
W1dim = ( 16, 5, 5 )
ds1 = ( 4, 4 )
#ds1 = ( 2, 2 )
W2dim = ( 16, 5, 5 )
ds2 = ( 4, 4 )
#ds2 = ( 2, 2 )
L1conv = convnet.ConvLayer( Xdim, W1dim, 'linear', withBias = False )
#L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'linear', withBias = False )
L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'ReLu', withBias = True )
H1 = L1pool.Dout
L2conv = convnet.ConvLayer( L1pool.Yshape, W2dim, 'linear', withBias = False )
#L2pool = convnet.PoolLayer( L2conv.Yshape, ds2, 'linear', withBias = False )
L2pool = convnet.PoolLayer( L2conv.Yshape, ds2, 'ReLu', withBias = True )
H2 = L2pool.Dout
L3 = convnet.FullLayer( H2, K, 'softmax', withBias = True, T4toMat = True )
cnn = convnet.CNN( [ L1conv, L1pool, L2conv, L2pool, L3 ] )
print '### Conv-Pool-Conv-Pool-Softmax Xdim:', Xdim
print '# W1dim:', W1dim, ' ds1:', ds1, ' H1:', H1
print '# W2dim:', W2dim, ' ds2:', ds2, ' H2:', H2
return cnn
# Conv-Pool-ReLu-Softmax
def CPRS( Xnch, Xrow, Xcol, K ):
Xdim = ( Xnch, Xrow, Xcol )
#W1dim = ( 16, 5, 5 )
W1dim = ( 64, 5, 5 )
ds1 = ( 4, 4 )
L1conv = convnet.ConvLayer( Xdim, W1dim, 'linear', withBias = False )
#L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'linear', withBias = False )
L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'ReLu', withBias = True )
H1 = L1pool.Dout
H2 = 400
L2 = convnet.FullLayer( H1, H2, 'ReLu', withBias = True, T4toMat = True )
L3 = convnet.FullLayer( H2, K, 'softmax', withBias = True, T4toMat = False )
cnn = convnet.CNN( [ L1conv, L1pool, L2, L3 ] )
print '### Conv-Pool-ReLu-Softmax Xdim:', Xdim
print '# W1dim:', W1dim, ' ds1:', ds1, ' H1:', H1
print '# H2:', H2
return cnn
if __name__ == "__main__":
idstr = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
print '### ID: ', idstr
dirCIFAR10 = '../140823-pylearn2/data/cifar10/cifar-10-batches-py'
cifar = cifar10.CIFAR10( dirCIFAR10 )
ZCAwhitening = True
##### setting the training data & the validation data
#
Xraw, label, t = cifar.loadData( 'L' )
Xraw /= 255
xm = np.mean( Xraw, axis = 0 )
Xraw -= xm
if ZCAwhitening:
Xraw2 = Xraw.reshape( ( Xraw.shape[0], -1 ) )
C = np.dot( Xraw2.T, Xraw2 ) / Xraw2.shape[0]
U, eva, V = np.linalg.svd( C ) # U[:, i] is the i-th eigenvector
sqeva = np.sqrt( eva + 0.001 )
Uzca = np.dot( U / sqeva[np.newaxis, :], U.T )
X = np.dot( Xraw2, Uzca ).reshape( Xraw.shape )
else:
X = Xraw
idxL, idxV = cifar.genIndexLV( label )
XL, labelL, tL = X[idxL], label[idxL], t[idxL]
XV, labelV, tV = X[idxV], label[idxV], t[idxV]
NL, Xnch, Xrow, Xcol = XL.shape
NV, Xnch, Xrow, Xcol = XV.shape
K = cifar.nclass
Xdim = ( Xrow, Xcol )
##### mini batch indicies for stochastic gradient ascent
#
np.random.seed( 0 )
idx = np.random.permutation( NL )
batchsize = 100
nbatch = NL / batchsize
assert( NL % batchsize == 0 )
idxB = np.zeros( ( nbatch, NL ), dtype = bool )
for ib in range( nbatch ):
idxB[ib, idx.reshape( ( nbatch, batchsize ))[ib, :]] = True
##### training
#
#cnn = CPS( Xnch, Xrow, Xcol, K )
#cnn = CPCPS( Xnch, Xrow, Xcol, K )
cnn = CPRS( Xnch, Xrow, Xcol, K )
eta, mu, lam = 0.01, 0.9, 0.0001
nepoch = 50
print '# eta = ', eta, ' mu = ', mu, ' lam = ', lam
print '# ZCAwhitening = ', ZCAwhitening
print '### training: NL = ', NL, ' NV = ', NV, ' K = ', K, ' batchsize = ', batchsize
for i in range( nepoch ):
# printing error rates etc.
if (i <= 5 ) or ( i % 10 == 0 ):
mnLLL, rrL = recograte( cnn, XL, tL, labelL )
mnLLV, rrV = recograte( cnn, XV, tV, labelV )
print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100 )
# training (selecting each batch in random order)
for ib in np.random.permutation( nbatch ):
ii = idxB[ib, :]
cnn.train( XL[ii], tL[ii], eta, mu, lam )
i = nepoch
mnLLL, rrL = recograte( cnn, XL, tL, labelL )
mnLLV, rrV = recograte( cnn, XV, tV, labelV )
print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100 )
##### setting the test data
#
XTraw, labelT, tT = cifar.loadData( 'T' )
XTraw /= 255
XTraw -= xm
if ZCAwhitening:
XTraw2 = XTraw.reshape( ( XTraw.shape[0], -1 ) )
XT = np.dot( XTraw2, Uzca ).reshape( XTraw.shape )
else:
XT = XTraw
NT, Nstack, Xrow, Xcol = XT.shape
print '# NT = ', NT
mnLLT, rrT = recograte( cnn, XT, tT, labelT )
print '%d | %.4f %.2f | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100, mnLLT, rrT * 100 )
##### making convolution filter image
#
W = cnn.Layers[0].W.get_value() # Wnch x Xnch x Wrow x Wcol
Wnch, Xnch, Wrow, Wcol = W.shape
simg_ncol = 16
simg_nrow = Wnch / simg_ncol
simg_w = simg_ncol * ( Wcol + 1 ) + 1
simg_h = simg_nrow * ( Wrow + 1 ) + 1
simg = np.zeros( ( simg_h, simg_w, 3 ), dtype = int )
absmax = np.max( np.abs( W ) )
for iy in range( simg_nrow ):
lty = iy * ( Wrow + 1 ) + 1
for ix in range( simg_ncol ):
ltx = ix * ( Wcol + 1 ) + 1
ich = iy * simg_ncol + ix
#absmax = np.max( np.abs( W[ich, :, :, :] ) )
Wi = np.asarray( W[ich, :, :, :] / absmax * 127 + 128, dtype = int )
# BGR <= RGB
simg[lty:lty+Wrow, ltx:ltx+Wcol, 2] = Wi[0, :, :]
simg[lty:lty+Wrow, ltx:ltx+Wcol, 1] = Wi[1, :, :]
simg[lty:lty+Wrow, ltx:ltx+Wcol, 0] = Wi[2, :, :]
Wimg = np.zeros( ( simg_h * 5, simg_w * 5, 3 ), dtype = int )
for iy in range( simg_h * 5 ):
for ix in range( simg_w * 5 ):
Wimg[iy, ix, :] = simg[iy/5, ix/5, :]
cv2.imwrite( 'hoge-' + idstr + '.png', Wimg )
print '### ID: ', idstr
import numpy as np
import scipy as sp
import datetime
import cifar10
import nnet0211 as nnet
def recograte( mlp, X, t, label ):
Y, Z = mlp.output( X )
mnLL = np.mean( mlp.cost( Z, t ) )
rr = np.mean( label == np.argmax( Z, axis = 1 ) )
return mnLL, rr
def MLP2( D, H1, K ):
print '### 2-layer MLP: D =', D, ' H =', H1, ' K =', K
L1 = nnet.Layer( D, H1, 'ReLu', withBias = True, Wini = 0.01 )
L2 = nnet.Layer( H1, K, 'softmax', withBias = True, Wini = 0.01 )
mlp = nnet.MLP( [ L1, L2 ] )
return mlp
def MLP3( D, H1, H2, K ):
print '### 3-layer MLP: D =', D, ' H1 =', H1, ' H2 =', H2, ' K =', K
L1 = nnet.Layer( D, H1, 'ReLu', withBias = True, Wini = 0.01 )
L2 = nnet.Layer( H1, H2, 'ReLu', withBias = True, Wini = 0.01 )
L3 = nnet.Layer( H2, K, 'softmax', withBias = True, Wini = 0.01 )
mlp = nnet.MLP( [ L1, L2, L3 ] )
return mlp
if __name__ == "__main__":
idstr = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
print '### ID: ', idstr
dirCIFAR10 = '../140823-pylearn2/data/cifar10/cifar-10-batches-py'
cifar = cifar10.CIFAR10( dirCIFAR10 )
ZCAwhitening = False
##### setting the training data & the validation data
#
Xraw, label, t = cifar.loadData( 'L' )
Xraw /= 255
xm = np.mean( Xraw, axis = 0 )
Xraw -= xm
if ZCAwhitening:
Xraw2 = Xraw.reshape( ( Xraw.shape[0], -1 ) )
C = np.dot( Xraw2.T, Xraw2 ) / Xraw2.shape[0]
U, eva, V = np.linalg.svd( C ) # U[:, i] is the i-th eigenvector
sqeva = np.sqrt( eva + 0.001 )
Uzca = np.dot( U / sqeva[np.newaxis, :], U.T )
X = np.dot( Xraw2, Uzca )
else:
X = Xraw.reshape( ( Xraw.shape[0], -1 ) )
idxL, idxV = cifar.genIndexLV( label )
XL, labelL, tL = X[idxL], label[idxL], t[idxL]
XV, labelV, tV = X[idxV], label[idxV], t[idxV]
NL, D = XL.shape
NV, D = XV.shape
K = cifar.nclass
##### mini batch indicies for stochastic gradient ascent
#
np.random.seed( 0 )
idx = np.random.permutation( NL )
batchsize = 100
nbatch = NL / batchsize
assert( NL % batchsize == 0 )
idxB = np.zeros( ( nbatch, NL ), dtype = bool )
for ib in range( nbatch ):
idxB[ib, idx.reshape( ( nbatch, batchsize ))[ib, :]] = True
##### training
#
#mlp = MLP2( D, 1000, K )
mlp = MLP3( D, 1000, 500, K )
eta, mu, lam = 0.01, 0.9, 0.0001
nepoch = 100
print '# eta = ', eta, ' mu = ', mu, ' lam = ', lam
print '# ZCAwhitening = ', ZCAwhitening
print '### training: NL = ', NL, ' NV = ', NV, ' D = ', D, ' K = ', K, ' batchsize = ', batchsize
for i in range( nepoch ):
# printing error rates etc.
if (i <= 5 ) or ( i % 10 == 0 ):
mnLLL, erL = recograte( mlp, XL, tL, labelL )
mnLLV, erV = recograte( mlp, XV, tV, labelV )
print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, erL * 100, mnLLV, erV * 100 )
# training (selecting each batch in random order)
for ib in np.random.permutation( nbatch ):
ii = idxB[ib, :]
mlp.train( XL[ii], tL[ii], eta, mu, lam )
i = nepoch
mnLLL, erL = recograte( mlp, XL, tL, labelL )
mnLLV, erV = recograte( mlp, XV, tV, labelV )
print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, erL * 100, mnLLV, erV * 100 )
##### setting the test data
#
XTraw, labelT, tT = cifar.loadData( 'T' )
XTraw /= 255
XTraw -= xm
if ZCAwhitening:
XTraw2 = XTraw.reshape( ( XTraw.shape[0], -1 ) )
XT = np.dot( XTraw2, Uzca )
else:
XT = XTraw.reshape( ( XTraw.shape[0], -1 ) )
NT, D = XT.shape
print '# NT = ', NT
mnLLT, erT = recograte( mlp, XT, tT, labelT )
print '%d | %.4f %.2f | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, erL * 100, mnLLV, erV * 100, mnLLT, erT * 100 )
print '### ID: ', idstr
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.