fig 3.10 Attempting to use uninitialized value test_recon_layer1/b

gistfile1.txt

"""
ch 3.5
https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist.py
"""

import time
import numpy as np
import tensorflow as tf
import tensorlayer as tl

import matplotlib as mpl
mpl.use('TkAgg')
import matplotlib.pyplot as plt

X_train, y_train, X_val, y_val, X_test, y_test = \
    tl.files.load_mnist_dataset(shape=(-1, 784))

X_train = np.asarray(X_train, dtype=np.float32)
y_train = np.asarray(y_train, dtype=np.int32)
X_val = np.asarray(X_val, dtype=np.float32)
y_val = np.asarray(y_val, dtype=np.int32)
X_test = np.asarray(X_test, dtype=np.float32)
y_test = np.asarray(y_test, dtype=np.int32)

print('X_train.shape', X_train.shape)
print('y_train.shape', y_train.shape)
print('X_val.shape', X_val.shape)
print('y_val.shape', y_val.shape)
print('X_test.shape', X_test.shape)
print('y_test.shape', y_test.shape)
print('X %s   y %s' % (X_test.dtype, y_test.dtype))

save_path = './ae_tl/autoencoder2.ckpt'

image_width = 28

saver = None

# 模型结构参数
hidden_size = 196
input_size = 784


def main_layers(model='relu', is_train=True, reuse=False):
    """build network
    :param model: 模型类别，可选Sigmoid或Relu
    """
    with tf.variable_scope("ae", reuse=reuse):
        # set model reuse
        tl.layers.set_name_reuse(reuse)

        # # 定义模型
        x = tf.placeholder(tf.float32, shape=[None, input_size], name='x')

        # 输入层 f(x)
        network = tl.layers.InputLayer(x, name='input')

        # ch3.4 introduce noise to input data
        network = tl.layers.DropoutLayer(network, keep=0.5, is_train=is_train, is_fix=False, name='denoising1')

        print('Build Network')
        if model == 'relu':
            network = tl.layers.DenseLayer(network, hidden_size, tf.nn.relu, name='relu1')

            # 隐层输出
            encoded_img = network.outputs

            # 重构层输出 g(h)
            # recon_layer1 = tl.layers.DenseLayer(network, input_size, tf.nn.softplus, name='recon_layer1')
            recon_layer1 = tl.layers.ReconLayer(network, x_recon=x, n_units=784, act=tf.nn.softplus,
                                                name='recon_layer1')

        elif model == 'sigmoid':
            network = tl.layers.DenseLayer(network, hidden_size, tf.nn.sigmoid, name='sigmoid1')

            # 隐层输出
            encoded_img = network.outputs

            # 重构层输出 g(h)
            # recon_layer1 = tl.layers.DenseLayer(network, input_size, tf.nn.sigmoid, name='recon_layer1')
            recon_layer1 = tl.layers.ReconLayer(network, x_recon=x, n_units=784, act=tf.nn.sigmoid,
                                                name='recon_layer1')

        return x, recon_layer1, encoded_img


def train_layers(model='relu'):
    global saver
    # 定义超参数
    learning_rate = 0.0001
    lambda_l2_w = 0.01
    n_epochs = 10
    batch_size = 128
    print_interval = 200

    x, recon_layer1, _ = main_layers(model, is_train=True, reuse=False)

    saver = tf.train.Saver()

    with tf.Session() as sess:
        tl.layers.initialize_global_variables(sess)
        recon_layer1.pretrain(sess, x=x, X_train=X_train, X_val=X_val, denoise_name='ae/denoising1', n_epoch=n_epochs,
                              batch_size=batch_size, print_freq=print_interval, save=True, save_name='w1pre_')

        # 保存模型为TensorFlow的ckpt格式
        saver.save(sess, save_path=save_path)
        print('model saved.')


def test_layers(reuse: bool, model='relu'):
    global saver

    x, recon_layer1, _ = main_layers(model, is_train=False, reuse=reuse)

    if saver is None:
        saver = tf.train.Saver()

    print('preparing visualization...')
    n_images = 15
    fig, axes = plt.subplots(nrows=2, ncols=n_images, sharex=True, sharey=True, figsize=(20, 2.5))
    # where is figsize in matplotlib documentation? No documentation, but has stackoverflow answer.
    # see https://stackoverflow.com/questions/332289/how-do-you-change-the-size-of-figures-drawn-with-matplotlib

    # 准备输入数据
    import random
    test_images = X_test[:n_images]
    # test_images = X_test[random.choices(range(X_test.shape[0]), k=n_images)]

    with tf.Session() as sess:
        # 加载训练好的模型
        saver.restore(sess, save_path=save_path)

        # 获取重构数据
        decoded = sess.run(recon_layer1.outputs, feed_dict={x: test_images})

        # 获取f(x)的参数
        if model == 'relu':
            weights = sess.run(tl.layers.get_variables_with_name('relu1/W:0', False, True))
        elif model == 'sigmoid':
            weights = sess.run(tl.layers.get_variables_with_name('sigmoid1/W:0', False, True))

        # 获取g(h)的参数
        recon_weights = sess.run(tl.layers.get_variables_with_name('recon_layer1/W:0', False, True))
        recon_bias = sess.run(tl.layers.get_variables_with_name('recon_layer1/b:0', False, True))

    # 保存图片
    for i in range(n_images):
        for ax, img in zip(axes, [test_images, decoded]):
            ax[i].imshow(img[i].reshape((image_width, image_width)), cmap='binary')

    plt.show()

    # TODO: 隐层权值可视化 fig 3.5, 3.7


def test_2to9_transition(model='relu', reuse=True):
    global saver

    x, recon_layer1, encoded_img = main_layers(model, is_train=False, reuse=reuse)

    if saver is None:
        saver = tf.train.Saver()

    # 准备输入数据
    n_images = 15
    import random
    test_images = X_test[:n_images]
    # test_images = X_test[random.choices(range(X_test.shape[0]), k=n_images)]

    # 首先准备2和9两个图片
    sample1_idx = 1
    sample2_idx = 12
    print('test image shape: {0}'.format(test_images[sample1_idx].shape))
    sample1 = test_images[sample1_idx].reshape([1, 784])
    sample2 = test_images[sample2_idx].reshape([1, 784])

    with tf.Session() as sess:
        # 加载训练好的网络
        saver.restore(sess, save_path)

        feed_dict = {x: sample1}
        encoded1 = sess.run(encoded_img, feed_dict=feed_dict)

        feed_dict = {x: sample2}
        encoded2 = sess.run(encoded_img, feed_dict=feed_dict)

        # 获取g(h)的参数
        recon_weights = sess.run(tl.layers.get_variables_with_name('ae/recon_layer1/W:0', False, True))
        recon_bias = sess.run(tl.layers.get_variables_with_name('ae/recon_layer1/b:0', False, True))
        # print('recon_weights: {0}'.format(recon_weights))
        # print('recon_bias: {0}'.format(recon_bias))

    encoded = tf.placeholder(tf.float32, shape=[None, hidden_size], name='encoded')
    recon_weights, recon_bias = recon_weights[0], recon_bias[0]

    # 构建图模型
    # with tf.variable_scope("test", reuse=reuse):
    test_network = tl.layers.InputLayer(encoded, name='test_input')
    if model == 'relu':
        test_recon_layer1 = tl.layers.DenseLayer(test_network, input_size, tf.nn.relu,
                                                 name='test_relu1')
    elif model == 'sigmoid':
        test_recon_layer1 = tl.layers.DenseLayer(test_network, input_size, tf.nn.sigmoid,
                                                 name='test_recon_layer1')

    # 通过对隐层提取到的隐特征向量插值实现(Interpolation)渐变
    diff = encoded2 - encoded1
    num_inter = 10
    delta = diff / num_inter
    encoded_all = encoded1
    for i in range(1, num_inter + 1):
        encoded_all = np.vstack((encoded_all, encoded1 + delta * i))
        # https://docs.scipy.org/doc/numpy/reference/generated/numpy.vstack.html

    print('encoded_all: {0}'.format(encoded_all.shape))

    # 输入h到解码器中，求出重构数据
    with tf.Session() as sess:
        # tl.layers.initialize_global_variables(sess)
        decoded_all = sess.run(test_recon_layer1.outputs, feed_dict={encoded: encoded_all})

    print('decoded_all: {0}'.format(decoded_all.shape))

    # 显示图3.10
    fig, axes = plt.subplots(nrows=1, ncols=num_inter+1, sharex=True, sharey=True, figsize=(15, 1.5))
    for i in range(num_inter+1):
        axes[i].imshow(decoded_all[i].reshape((image_width, image_width)), cmap='binary')

    plt.show()


if __name__ == '__main__':
    all_start_time = time.time()
    model = 'sigmoid'
    # train_layers(model=model)
    # test_layers(model=model, reuse=False)
    test_2to9_transition(model=model, reuse=False)

    print('all finished took %.2fs' % (time.time() - all_start_time))