ex150823.py & ex150823b.py

00readme.md

ex150823.py & ex150823b.py

• see http://takatakamanbou.hatenablog.com/entry/2015/08/24/173834
• cifar10.py & cifar10_sub150823.py https://gist.github.com/takatakamanbou/413467c6c36177c5d6e3
• convnet150821.py https://gist.github.com/takatakamanbou/b65f0048160f0ed07985

ex150823.py

import numpy as np
import scipy as sp
import datetime

import cifar10
import cifar10_sub150823 as cifar10_sub 
import convnet150821 as convnet



# computing the recognition rate
def recograte( cnn, X, label, batchsize, dstshape ):

    N = X.shape[0]
    nbatch = int( np.ceil( float( N ) / batchsize ) )

    LL = 0.0
    cnt = 0
    for ib in range( nbatch - 1 ):
        ii = np.arange( ib*batchsize, (ib+1)*batchsize )
        XX = cifar10_sub.clipcenter( X[ii], dstshape )
        Y, Z  = cnn.output( XX )
        LL += np.sum( cnn.cost( Z, label[ii] ) )
        cnt += np.sum( label[ii] == np.argmax( Z, axis = 1 ) )
    ib = nbatch - 1
    ii = np.arange( ib*batchsize, N )
    XX = cifar10_sub.clipcenter( X[ii], dstshape )
    Y, Z  = cnn.output( XX )
    LL += np.sum( cnn.cost( Z, label[ii] ) )
    cnt += np.sum( label[ii] == np.argmax( Z, axis = 1 ) )

    return LL / N, float( cnt ) / N


# Conv-Pool-ReLu-Softmax
def CPRS( Xnch, Xrow, Xcol, K ):
    
    Xdim = ( Xnch, Xrow, Xcol )
    W1dim = ( 64, 5, 5 )
    ds1 = ( 4, 4 )
    #ds1 = ( 2, 2 )
    st1 = None
    #border_mode = 'valid'
    border_mode = 'full'
    L1conv = convnet.ConvLayer( Xdim, W1dim, 'linear', withBias = False, border_mode = border_mode  )
    L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'ReLu', withBias = True, st = st1 )
    H1 = L1pool.Dout
    H2 = 1000
    L2 = convnet.FullLayer( H1, H2, 'ReLu', withBias = True, T4toMat = True )
    L3 = convnet.FullLayer( H2, K, 'softmax', withBias = True, T4toMat = False )
    cnn = convnet.CNN( [ L1conv, L1pool, L2, L3 ] )
    print '### Conv-Pool-ReLu-Softmax   Xdim:', Xdim
    print '#   W1dim:', W1dim, ' border_mode:', L1conv.border_mode, ' ds1:', ds1, ' st1:', st1, ' H1:', H1
    print '#   H2:', H2

    return cnn



    
if __name__ == "__main__":

    idstr = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
    print '### ID: ', idstr

    dirCIFAR10 = './cifar10/cifar-10-batches-py'
    cifar = cifar10.CIFAR10( dirCIFAR10 )
    ZCAwhitening = True
    tfunc = cifar10_sub.translate2
    dstshape = ( 24, 24 )
    #dstshape = ( 28, 28 )
    #dstshape = ( 32, 32 )

    ##### setting the training data & the validation data
    #
    Xraw, label, t = cifar.loadData( 'L' )
    Xraw /= 255
    xm = np.mean( Xraw, axis = 0 )
    Xraw -= xm
    if ZCAwhitening:
        X, Uzca = cifar10_sub.ZCAtrans( Xraw, Uzca = None )
    else:
        X = Xraw
    X = np.asarray( X, dtype = np.float32 )
    label = np.asarray( label, dtype = np.int32 )

    idxL, idxV = cifar.genIndexLV( label )
    XL, labelL = X[idxL], label[idxL]
    XV, labelV = X[idxV], label[idxV]
    NL, Xnch = XL.shape[0], XL.shape[1]
    Xrow, Xcol = dstshape[0], dstshape[1]
    NV = XV.shape[0]
    K = cifar.nclass
    Xdim = ( Xrow, Xcol )

    np.random.seed( 0 )
    batchsize = 100
    idxB = cifar10_sub.makebatchindex( NL, batchsize )
    nbatch = idxB.shape[0]
        

    ##### training
    #
    cnn = CPRS( Xnch, Xrow, Xcol, K )

    eta, mu, lam = 0.01, 0.9, 0.0001
    nepoch = 50
    print '# eta = ', eta, ' mu = ', mu, ' lam = ', lam
    print '# ZCAwhitening = ', ZCAwhitening,  ' tfunc = ', tfunc.__name__, ' dstshape = ', dstshape
    
    print '### training:   NL = ', NL, ' NV = ', NV, ' K = ', K, ' batchsize = ', batchsize

    for i in range( nepoch ):

        # printing error rates etc.
        if (i <= 5 ) or ( i % 10 == 0 ):
            mnLLL, rrL = recograte( cnn, XL, labelL, batchsize, dstshape )
            mnLLV, rrV = recograte( cnn, XV, labelV, batchsize, dstshape )
            print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100 )

        # training (selecting each batch in random order)
        for ib in np.random.permutation( nbatch ):
            ii = idxB[ib, :]
            XX = tfunc( XL[ii], dstshape )
            cnn.train( XX, labelL[ii], eta, mu, lam )

    i = nepoch

    mnLLL, rrL = recograte( cnn, XL, labelL, batchsize, dstshape )
    mnLLV, rrV = recograte( cnn, XV, labelV, batchsize, dstshape )
    print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100 )


    ##### setting the test data
    #
    XTraw, labelT, tT = cifar.loadData( 'T' )
    XTraw /= 255
    XTraw -= xm
    if ZCAwhitening:
        XT = cifar10_sub.ZCAtrans( XTraw, Uzca = Uzca )
    else:
        XT = XTraw
    XT = np.asarray( XT, dtype = np.float32 )
    labelT = np.asarray( labelT, dtype = np.int32 )
        
    NT, Nstack, Xrow, Xcol = XT.shape
    print '# NT = ', NT
    mnLLT, rrT = recograte( cnn, XT, labelT, batchsize, dstshape )
    print '%d | %.4f %.2f | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100, mnLLT, rrT * 100 )

    print '### ID: ', idstr

ex150823b.py

import numpy as np
import scipy as sp
import datetime

import cifar10
import cifar10_sub150823 as cifar10_sub 
import convnet150821 as convnet



# computing the recognition rate
def recograte( cnn, X, label, batchsize, dstshape ):

    N = X.shape[0]
    nbatch = int( np.ceil( float( N ) / batchsize ) )

    LL = 0.0
    cnt = 0
    for ib in range( nbatch - 1 ):
        ii = np.arange( ib*batchsize, (ib+1)*batchsize )
        XX = cifar10_sub.clipcenter( X[ii], dstshape )
        Y, Z  = cnn.output( XX )
        LL += np.sum( cnn.cost( Z, label[ii] ) )
        cnt += np.sum( label[ii] == np.argmax( Z, axis = 1 ) )
    ib = nbatch - 1
    ii = np.arange( ib*batchsize, N )
    XX = cifar10_sub.clipcenter( X[ii], dstshape )
    Y, Z  = cnn.output( XX )
    LL += np.sum( cnn.cost( Z, label[ii] ) )
    cnt += np.sum( label[ii] == np.argmax( Z, axis = 1 ) )

    return LL / N, float( cnt ) / N


# Conv-Pool-Conv-Pool-ReLu-Softmax
def CPCPRS( Xnch, Xrow, Xcol, K ):

    # 1st Layer 
    Xdim = ( Xnch, Xrow, Xcol )
    W1dim = ( 64, 5, 5 )
    b1 = 'full'
    L1conv = convnet.ConvLayer( Xdim, W1dim, 'linear', withBias = False, border_mode = b1 )
    #ds1 = ( 4, 4 )
    #ds1 = ( 3, 3 )
    ds1 = ( 2, 2 )
    st1 = None
    L1pool = convnet.PoolLayer( L1conv.Yshape, ds1, 'ReLu', withBias = True, st = st1 )
    H1 = L1pool.Dout

    # 2nd Layer 
    W2dim = ( 64, 5, 5 )
    #W2dim = ( 64, 3, 3 )
    b2 = 'full'
    L2conv = convnet.ConvLayer( L1pool.Yshape, W2dim, 'linear', withBias = False, border_mode = b2 )
    #ds2 = ( 4, 4 )
    #ds2 = ( 3, 3 )
    ds2 = ( 2, 2 )
    st2 = None
    L2pool = convnet.PoolLayer( L2conv.Yshape, ds2, 'ReLu', withBias = True, st = st2 )
    H2 = L2pool.Dout

    # 3rd Layer 
    H3 = 1000
    L3 = convnet.FullLayer( H2, H3, 'ReLu', withBias = True, T4toMat = True )

    # 4th Layer 
    L4 = convnet.FullLayer( H3, K, 'softmax', withBias = True, T4toMat = False )
    cnn = convnet.CNN( [ L1conv, L1pool, L2conv, L2pool, L3, L4 ] )

    print '### Conv-Pool-Conv-Pool-ReLu-Softmax   Xdim:', Xdim
    print '#   W1dim:', W1dim, ' ds1:', ds1, ' st1:', st1, ' border_mode:', L1conv.border_mode, ' H1:', H1
    print '#   W2dim:', W2dim, ' ds2:', ds2, ' st2:', st2, ' border_mode:', L2conv.border_mode, ' H2:', H2
    print '#   H3:', H3

    return cnn


    
if __name__ == "__main__":

    idstr = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
    print '### ID: ', idstr

    dirCIFAR10 = './cifar10/cifar-10-batches-py'
    cifar = cifar10.CIFAR10( dirCIFAR10 )
    ZCAwhitening = True
    tfunc = cifar10_sub.translate2
    #tfunc = cifar10_sub.clipcenter
    dstshape = ( 24, 24 )
    #dstshape = ( 32, 32 )


    ##### setting the training data & the validation data
    #
    Xraw, label, t = cifar.loadData( 'L' )
    Xraw /= 255
    xm = np.mean( Xraw, axis = 0 )
    Xraw -= xm
    if ZCAwhitening:
        X, Uzca = cifar10_sub.ZCAtrans( Xraw, Uzca = None )
    else:
        X = Xraw
    X = np.asarray( X, dtype = np.float32 )
    label = np.asarray( label, dtype = np.int32 )

    idxL, idxV = cifar.genIndexLV( label )
    XL, labelL = X[idxL], label[idxL]
    XV, labelV = X[idxV], label[idxV]
    NL, Xnch = XL.shape[0], XL.shape[1]
    Xrow, Xcol = dstshape[0], dstshape[1]
    NV = XV.shape[0]
    K = cifar.nclass
    Xdim = ( Xrow, Xcol )

    np.random.seed( 0 )
    batchsize = 100
    idxB = cifar10_sub.makebatchindex( NL, batchsize )
    nbatch = idxB.shape[0]
        

    ##### training
    #
    cnn = CPCPRS( Xnch, Xrow, Xcol, K )

    eta, mu, lam = 0.01, 0.9, 0.0001
    nepoch = 50
    print '# eta = ', eta, ' mu = ', mu, ' lam = ', lam
    print '# ZCAwhitening = ', ZCAwhitening,  ' tfunc = ', tfunc.__name__, ' dstshape = ', dstshape
    
    print '### training:   NL = ', NL, ' NV = ', NV, ' K = ', K, ' batchsize = ', batchsize

    for i in range( nepoch ):

        # printing error rates etc.
        if (i <= 5 ) or ( i % 10 == 0 ):
            mnLLL, rrL = recograte( cnn, XL, labelL, batchsize, dstshape )
            mnLLV, rrV = recograte( cnn, XV, labelV, batchsize, dstshape )
            print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100 )

        # training (selecting each batch in random order)
        for ib in np.random.permutation( nbatch ):
            ii = idxB[ib, :]
            XX = tfunc( XL[ii], dstshape )
            cnn.train( XX, labelL[ii], eta, mu, lam )

    i = nepoch

    mnLLL, rrL = recograte( cnn, XL, labelL, batchsize, dstshape )
    mnLLV, rrV = recograte( cnn, XV, labelV, batchsize, dstshape )
    print '%d | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100 )


    ##### setting the test data
    #
    XTraw, labelT, tT = cifar.loadData( 'T' )
    XTraw /= 255
    XTraw -= xm
    if ZCAwhitening:
        XT = cifar10_sub.ZCAtrans( XTraw, Uzca = Uzca )
    else:
        XT = XTraw
    XT = np.asarray( XT, dtype = np.float32 )
    labelT = np.asarray( labelT, dtype = np.int32 )
        
    NT, Nstack, Xrow, Xcol = XT.shape
    print '# NT = ', NT
    mnLLT, rrT = recograte( cnn, XT, labelT, batchsize, dstshape )
    print '%d | %.4f %.2f | %.4f %.2f | %.4f %.2f' % ( i, mnLLL, rrL * 100, mnLLV, rrV * 100, mnLLT, rrT * 100 )

    print '### ID: ', idstr