TessorFlow tutorial of Variable Scoping

README.md

TensorFlow の名前空間を理解して共有変数を使いこなす

関連ファイル:

1. mnist_ae_varscope.py
2. mnist_2nets.py

mnist_2nets.py

#
#   mnist_2nets.py
#       date. 11/28/2016, 12/6
#

import numpy as np
import tensorflow as tf

# Import data
from input_data import DataSet, extract_images, extract_labels

# Full-connected Layer   
class FullConnected(object):
    def __init__(self, input, n_in, n_out, vn=('W', 'b')):
        self.input = input

        weight_init = tf.truncated_normal_initializer(mean=0.0, stddev=0.05)
        bias_init = tf.constant_initializer(value=0.0)
        W = tf.get_variable(vn[0], [n_in, n_out], initializer=weight_init)
        b = tf.get_variable(vn[1], [n_out], initializer=bias_init)
        self.w = W
        self.b = b
        self.params = [self.w, self.b]
    
    def output(self):
        linarg = tf.matmul(self.input, self.w) + self.b
        self.output = tf.nn.relu(linarg)
        
        return self.output
#

# Read-out Layer
class ReadOutLayer(object):
    def __init__(self, input, n_in, n_out, vn=('W', 'b')):
        self.input = input

        weight_init = tf.random_normal_initializer(mean=0.0, stddev=0.05)
        bias_init = tf.constant_initializer(value=0.0)
        W = tf.get_variable(vn[0], [n_in, n_out], initializer=weight_init)
        b = tf.get_variable(vn[1], [n_out], initializer=bias_init)
        self.w = W
        self.b = b
        self.params = [self.w, self.b]
    
    def output(self):
        linarg = tf.matmul(self.input, self.w) + self.b
        self.output = tf.nn.softmax(linarg)  

        return self.output
#

def read_and_split(dirn='../data', one_hot=True):
    class DataSets(object):
        pass
    data_sets = DataSets()

    TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'
    TRAIN_LABELS = 'train-labels-idx1-ubyte.gz'
    TEST_IMAGES = 't10k-images-idx3-ubyte.gz'
    TEST_LABELS = 't10k-labels-idx1-ubyte.gz'
    TRAIN1_SIZE = 30000

    BASE_PATH = dirn + '/'
    train_images = extract_images(BASE_PATH+TRAIN_IMAGES)
    train_labels = extract_labels((BASE_PATH+TRAIN_LABELS), one_hot=one_hot)
    test_images = extract_images(BASE_PATH+TEST_IMAGES)
    test_labels = extract_labels((BASE_PATH+TEST_LABELS), one_hot=one_hot)

    train1_images = train_images[:TRAIN1_SIZE]
    train1_labels = train_labels[:TRAIN1_SIZE]
    train2_images = train_images[TRAIN1_SIZE:]
    train2_labels = train_labels[TRAIN1_SIZE:]

    data_sets.train1 = DataSet(train1_images, train1_labels)
    data_sets.train2 = DataSet(train2_images, train2_labels)
    data_sets.test = DataSet(test_images, test_labels)

    return data_sets
 


# Create the model
def mk_NN_model(scope='mlp', reuse=False):
    '''
      args.:
        scope   : variable scope ID of networks
        reuse   : reuse flag of weights/biases
    '''
    with tf.variable_scope(scope, reuse=reuse):
        hidden1 = FullConnected(x, 784, 625, vn=('W_hid_1','b_hid_1'))
        h1out = hidden1.output()
        hidden2 = FullConnected(h1out, 625, 625, vn=('W_hid_2','b_hid_2'))
        h2out = hidden2.output()    
        readout = ReadOutLayer(h2out, 625, 10, vn=('W_RO', 'b_RO'))
        y_pred = readout.output()
     
    cross_entropy = -tf.reduce_sum(y_*tf.log(y_pred))
    
    # Regularization terms (weight decay)
    L2_sqr = tf.nn.l2_loss(hidden1.w) + tf.nn.l2_loss(hidden2.w)
    lambda_2 = 0.01

    # the loss and accuracy
    with tf.name_scope('loss'):
        loss = cross_entropy + lambda_2 * L2_sqr
    with tf.name_scope('accuracy'):
        correct_prediction = tf.equal(tf.argmax(y_pred,1), tf.argmax(y_,1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    
    return y_pred, loss, accuracy
 
#

def test_averaging(predicts, actual):
    '''
      test classification process
        args.:
        predicts    : predictions lists by networks
        actual      : label data of test
    '''
    y_pred_ave = (predicts[0] + predicts[1]) / 2.

    correct_prediction = tf.equal(tf.argmax(y_pred_ave,1), tf.argmax(actual,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    return accuracy
#


if __name__ == '__main__':
    # Load Dataset
    mnist = read_and_split("../data/", one_hot=True)

    # Variables
    x = tf.placeholder(tf.float32, [None, 784])
    y_ = tf.placeholder(tf.float32, [None, 10])
    tf.set_random_seed(2016)
    y_pred1, loss1, accuracy1 = mk_NN_model(scope='mlp1')
    # y_pred2, loss2, accuracy2 = mk_NN_model(scope='mlp1', reuse=True)
    y_pred2, loss2, accuracy2 = mk_NN_model(scope='mlp2')

    # Train
    with tf.name_scope('train1'):
        train_step1 = tf.train.AdagradOptimizer(0.003).minimize(loss1)
    with tf.name_scope('train2'):
        train_step2 = tf.train.AdamOptimizer(0.001).minimize(loss2)
    
    init = tf.initialize_all_variables()

    with tf.Session() as sess:
        summary_writer = tf.train.SummaryWriter('/tmp/tflogs'+'/train', sess.graph)
        sess.run(init)
        print('Training...')

        print('  Network No.1 :')
        for i in range(5001):
            batch_xs, batch_ys = mnist.train1.next_batch(100)
            train_step1.run({x: batch_xs, y_: batch_ys})
            if i % 1000 == 0:
                accuracy1_i = accuracy1.eval({x: batch_xs, y_: batch_ys})
                loss1_i = loss1.eval({x: batch_xs, y_: batch_ys})
                print('  step, loss, accurary = {:>6d}:{:>8.3f},{:>8.3f}'\
                        .format(i, loss1_i, accuracy1_i))
        accuracy1_test = accuracy1.eval(
            {x: mnist.test.images, y_: mnist.test.labels})

        print('  Network No.2 :')
        for i in range(5001):
            batch_xs, batch_ys = mnist.train2.next_batch(100)
            train_step2.run({x: batch_xs, y_: batch_ys})
            if i % 1000 == 0:
                accuracy2_i = accuracy2.eval({x: batch_xs, y_: batch_ys})
                loss2_i = loss2.eval({x: batch_xs, y_: batch_ys})
                print('  step, loss, accurary = {:>6d}:{:>8.3f},{:>8.3f}'\
                        .format(i, loss2_i, accuracy2_i))
        accuracy2_test = accuracy2.eval(
            {x: mnist.test.images, y_: mnist.test.labels})

        # Test trained model
        with tf.name_scope('model_averaging'):
            accu_ave = test_averaging([y_pred1, y_pred2], y_)
            averaged = sess.run(accu_ave, 
                feed_dict={x: mnist.test.images, y_: mnist.test.labels})

        print('accuracy1 = {:>8.4f}'.format(accuracy1_test))
        print('accuracy2 = {:>8.4f}'.format(accuracy2_test))
        print('accuracy (model averaged) = {:>8.4f}'.format(averaged))

mnist_ae_varscope.py

#
#   mnist_ae_varscope.py   date. 12/2/2016
#   
#   Autoencoder tutorial code
#

import numpy as np
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import tensorflow as tf

# Import data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("../data/", one_hot=True)

# Encoder Layer   
class Encoder(object):
    def __init__(self, input, n_in, n_out, vs_enc='encoder'):
        self.input = input
        with tf.variable_scope(vs_enc):
            weight_init = tf.truncated_normal_initializer(mean=0.0, stddev=0.05)
            W = tf.get_variable('W', [n_in, n_out], initializer=weight_init)
            bias_init = tf.constant_initializer(value=0.0)
            b = tf.get_variable('b', [n_out], initializer=bias_init)
        self.w = W
        self.b = b
    
    def output(self):
        linarg = tf.matmul(self.input, self.w) + self.b
        self.output = tf.sigmoid(linarg)
        
        return self.output
#

# Decoder Layer
class Decoder(object):
    def __init__(self, input, n_in, n_out, vs_dec='decoder'):
        self.input = input
        if vs_dec == 'decoder': # independent weight
            with tf.variable_scope(vs_dec):
                weight_init = tf.truncated_normal_initializer(mean=0.0, stddev=0.05)
                W = tf.get_variable('W', [n_in, n_out], initializer=weight_init)
        else:                   # weight sharing (tying)
            with tf.variable_scope(vs_dec, reuse=True):     # set reuse option
                W = tf.get_variable('W', [n_out, n_in])
                W = tf.transpose(W)

        with tf.variable_scope('decoder'):  # in all case, need new bias
            bias_init = tf.constant_initializer(value=0.0)
            b = tf.get_variable('b', [n_out], initializer=bias_init)
        self.w = W
        self.b = b
    
    def output(self):
        linarg = tf.matmul(self.input, self.w) + self.b
        self.output = tf.sigmoid(linarg)
        
        return self.output
#

# Variables
x = tf.placeholder("float", [None, 784])
y_ = tf.placeholder("float", [None, 10])

# make neural network model
def make_model(x):
    enc_layer = Encoder(x, 784, 625, vs_enc='encoder')
    enc_out = enc_layer.output()
    dec_layer = Decoder(enc_out, 625, 784, vs_dec='encoder')
    dec_out = dec_layer.output()

    return enc_out, dec_out


encoded, decoded = make_model(x)

# Cost Function basic term
cross_entropy = -1. * x * tf.log(decoded) - (1. - x) * tf.log(1. - decoded)
loss = tf.reduce_mean(cross_entropy)
train_step = tf.train.AdagradOptimizer(0.1).minimize(loss)

# Train
init = tf.initialize_all_variables()

with tf.Session() as sess:
    sess.run(init)
    print('Training...')
    for i in range(10001):
        batch_xs, batch_ys = mnist.train.next_batch(128)
        train_step.run({x: batch_xs, y_: batch_ys})
        
        if i % 1000 == 0:
            train_loss = loss.eval({x: batch_xs, y_: batch_ys})
            print('  step, loss = %6d: %6.3f' % (i, train_loss))
        
    # generate decoded image with test data
    test_fd = {x: mnist.test.images, y_: mnist.test.labels}
    decoded_imgs = decoded.eval(test_fd)
    print('loss (test) = ', loss.eval(test_fd))
    
x_test = mnist.test.images

n = 10  # how many digits we will display
plt.figure(figsize=(20, 4))
for i in range(n):
    # display original
    ax = plt.subplot(2, n, i + 1)
    plt.imshow(x_test[i].reshape(28, 28))
    plt.gray()
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)

    # display reconstruction
    ax = plt.subplot(2, n, i + 1 + n)
    plt.imshow(decoded_imgs[i].reshape(28, 28))
    plt.gray()
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)

plt.savefig('mnist_ae1.png')