dataset.py

# Importing data from csv file
csv_filename = './tweets.csv'
dataset = tf.contrib.data.make_csv_dataset(csv_filename, batch_size=32)
dataset = dataset.shuffle(buffer_size=100)
iter = dataset.make_one_shot_iterator()
next = iter.get_next()
features, labels = next['text'], next['sentiment']

with tf.Session() as sess:
	sess.run([features, labels])

# Creating an iterator
x = tf.placeholder(tf.float32, shape=[None,2])
dataset = tf.data.Dataset.from_tensor_slices(x)	# Create dataset instance from placeholder
data = np.random.sample((100,2))
iter = dataset.make_initializable_iterator()	# Creating iterator from dataset instance
el = iter.get_next()

with tf.Session() as sess:
    sess.run(iter.initializer, feed_dict={x:data})	# initialize iterator
    print(sess.run(el))


# Creating simple model using batching and switch between train & test dataset using initializable iterator
epochs = 10
batch_size = tf.placeholder(tf.int64)
BATCH_SIZE = 32

x,y = tf.placeholder(tf.float32, shape=[None, 2]), tf.placeholder(tf.float32, shape=[None,1])
dataset = tf.data.Dataset.from_tensor_slices((x,y)).batch(batch_size).repeat()

train_data = (np.random.sample((100,2)), np.random.sample((100,1)))
test_data = (np.random.sample((20,2)), np.random.sample((20,1)))

iter = dataset.make_initializable_iterator()
features, labels = iter.get_next()

net = tf.layers.dense(features, 8, activation=tf.tanh)
net = tf.layers.dense(net, 8, activation=tf.tanh)
prediction = tf.layers.dense(net, 1, activation=tf.tanh)

n_batches = train_data[0].shape[0] // BATCH_SIZE

loss = tf.losses.mean_squared_error(prediction, labels)
optimizer = tf.train.AdamOptimizer().minimize(loss)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    
    sess.run(iter.initializer, feed_dict={ x: train_data[0], y: train_data[1], batch_size: BATCH_SIZE})
    print("Training...")
    for i in range(epochs):
        total_loss = 0
        for _ in range(n_batches):
            _, loss_value = sess.run([optimizer, loss])
            total_loss += loss_value
        print("EPOCH: {}, Loss: {:.4f}".format(i, total_loss/n_batches))
    sess.run(iter.initializer, feed_dict={x:test_data[0], y:test_data[1], batch_size:test_data[0].shape[0]})
    print('Test Loss: {:.4f}'.format(sess.run(loss)))

input.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import tensorflow as tf


def input(eval_data, data_dir, batch_size):
	# Get file names from data dir

	# Create queue to produce filenames to read

	# Read examples from files in filename queue

	"""
	Preprocessing
	"""
	pass

def distorted_input(data_dir, batch_size):
	# Get file names from data dir

	pass

def _generate_image_label_batches(image, label, min_queue_examples,
                                    batch_size, shuffle):
	"""
	NOTE: purpose of function is just to differentiate between shuffling distorted and
	straight input batches
	"""
	# Read batch size from the queue

	# Add training images to the visualizer
	# tf.summary.image('images', images)

 #  	return images, tf.reshape(label_batch, [batch_size])
	pass

main_datacamp.py

def load_data(data_dir):
    # Get all subdirectories of data_dir. Each represents a label.
    directories = [d for d in os.listdir(data_dir) 
                   if os.path.isdir(os.path.join(data_dir, d))]
    # Loop through the label directories and collect the data in
    # two lists, labels and images.
    labels = []
    images = []
    for d in directories:
        label_dir = os.path.join(data_dir, d)
        file_names = [os.path.join(label_dir, f) 
                      for f in os.listdir(label_dir) 
                      if f.endswith(".ppm")]
        for f in file_names:
            images.append(data.imread(f))
            labels.append(int(d))
    return images, labels

ROOT_PATH = "/Users/ktl014/PycharmProjects/PersonalProjects/tensorflow_tutorial/tf_datacamp/TensorFlow Tutorial For Beginners"
train_data_dir = os.path.join(ROOT_PATH, "Training")
test_data_dir = os.path.join(ROOT_PATH, "Testing")

images, labels = load_data(train_data_dir)
images_array = np.array(images)
labels_array = np.array(labels)

# Resize images
images32 = [transform.resize(image, (28, 28)) for image in images]
images32 = np.array(images32)
images32 = rgb2gray(np.array(images32))

x = tf.placeholder(dtype = tf.float32, shape = [None, 28, 28])                  # placeholders ~ variables for feeding data into computation graph w/out data
y = tf.placeholder(dtype = tf.int32, shape = [None])
images_flat = tf.contrib.layers.flatten(x)                                      # flatten ~ flattens input while maintaining batch size; 1st dim = batch
logits = tf.contrib.layers.fully_connected(images_flat, 62, tf.nn.relu)         # fully_connected ~ adds fully connected layer, outputs feature vector
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels = y, logits = logits))  # reduce_mean ~ computes mean of elements across tensor dim
                                                                                                    # sparse_softmax_cross... ~ 
train_op = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)           # AdamOptimizer().minimize() ~ Class AdamOptimizer adds operatiosn to minimize loss (computing and applying gradient)
correct_pred = tf.argmax(logits, 1)                                             # argmax() ~ returns index with largest value across axes of a tensor
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

sess = tf.Session()

sess.run(tf.global_variables_initializer())                                     # global_variables... ~ initializes global variables in graph

for i in range(201):
        print('EPOCH', i)
        _, accuracy_val = sess.run([train_op, accuracy], feed_dict={x: images32, y: labels})
        if i % 10 == 0:
            print("Loss: ", loss)
        print('DONE WITH EPOCH')

# Load the test data
test_images, test_labels = load_data(test_data_dir)

# Transform the images to 28 by 28 pixels
test_images28 = [transform.resize(image, (28, 28)) for image in test_images]

# Convert to grayscale
from skimage.color import rgb2gray
test_images28 = rgb2gray(np.array(test_images28))

# Run predictions against the full test set.
predicted = sess.run([correct_pred], feed_dict={x: test_images28})[0]

# Calculate correct matches 
match_count = sum([int(y == y_) for y, y_ in zip(test_labels, predicted)])

# Calculate the accuracy
accuracy = match_count / float(len(test_labels))

# Print the accuracy
print("Accuracy: {:.3f}".format(accuracy))

main_finetune.py

import tensorflow as tf
import numpy as np
import sys

import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "1"

from datetime import datetime
#sys.path.insert(0,'/data6/lekevin/cayman/')
sys.path.insert(0, '/data4/plankton_wi17/mpl/source_domain/spcbench/bench_finetune/code/')
#from tools.dataset import SPCDataset
from create_dataset import PlanktonDataset
import tensorflow.contrib.slim.nets as nets
tf.reset_default_graph()

class CaffeNet(object):
    def __init__(self, x, keep_prob, num_classes, skip_layer, weights_path='DEFAULT'):
        """
        Inputs:
        - x: tf.placeholder, for the input images
        - keep_prob: tf.placeholder, for the dropout rate
        - num_classes: int, number of classes of the new dataset
        - skip_layer: list of strings, names of the layers you want to reinitialize
        - weights_path: path string, path to the pretrained weights,
                        (if bvlc_alexnet.npy is not in the same folder)
        """
        self.X = x
        self.NUM_CLASSES = num_classes
        self.KEEP_PROB = keep_prob
        self.SKIP_LAYER = skip_layer
        if weights_path == 'DEFAULT':
            self.WEIGHTS_PATH = '/data6/lekevin/cayman/tensorflow_code/caffenet.npy'
        else:
            self.WEIGHTS_PATH = weights_path
        
        # Build computational graph of AlexNet
        self.create()
    
    def create(self):
        """Create network graph"""
        # 1st Layer: Conv (w/ReLu) -> Pool -> Norm
        conv1 = conv(self.X, 11, 11, 96, 4, 4, padding='VALID', name='conv1')
        pool1 = max_pool(conv1, 3, 3, 2, 2, padding='VALID', name='pool1')
        norm1 = lrn(pool1, 2, 1e-04, 0.75, name='norm1')
        
        # 2nd Layer: Conv (w/ReLu) -> Pool -> Norm
        conv2 = conv(norm1, 5, 5, 256, 1, 1, groups=2, name='conv2')
        pool2 = max_pool(conv2, 3, 3, 2, 2, padding='VALID', name='pool2')
        norm2 = lrn(pool2, 2, 1e-04, 0.75, name='norm2_a')
        
        # 3rd Layer: Conv (w/ ReLu)
        conv3 = conv(norm2, 3, 3, 384, 1, 1, name='conv3')
        
        # 4th Layer: Conv(w/ ReLu)
        conv4 = conv(conv3, 3, 3, 384, 1, 1, groups=2, name='conv4')
        
        # 5th Layer: Conv (w/ ReLu)
        conv5 = conv(conv4, 3, 3, 256, 1, 1, groups=2, name='conv5')
        pool5 = max_pool(conv5, 3, 3, 2, 2, padding='VALID', name='pool5')
        
        # 6th Layer: Flatten --> FC (w/ ReLu) -> Dropout
        flattened = tf.reshape(pool5, [-1, 6*6*256])
        fc6 = fc(flattened, 6*6*256, 4096, name='fc6')
        dropout6 = dropout(fc6, self.KEEP_PROB)
        
        # 7th Layer: FC (w ReLu) -> Dropout
        fc7 = fc(dropout6, 4096, 4096, name='fc7')
        dropout7 = dropout(fc7, self.KEEP_PROB)

        # 8th Layer: FC and return unscaled activations
        self.fc8 = fc(dropout7, 4096, self.NUM_CLASSES, relu=False, name='fc8_spcbench')
    
    def load_initial_weights(self, session):
        weights_dict = np.load(self.WEIGHTS_PATH, encoding='bytes').item()
        for op_name in weights_dict:
            if op_name not in self.SKIP_LAYER:
                with tf.variable_scope(op_name, reuse=True):
                    for data in weights_dict[op_name]:
                        if len(weights_dict[op_name][data].shape) == 1:
                            var = tf.get_variable('biases', trainable=False)
                            session.run(var.assign(weights_dict[op_name][data]))
                        else:
                            var = tf.get_variable('weights', trainable=False)
                            session.run(var.assign(weights_dict[op_name][data]))

def conv(x, filter_height, filter_width, num_filters, stride_y, stride_x, name, padding='SAME', groups=1):
    input_channels = int(x.get_shape()[-1])
    
    convolve = lambda i, k: tf.nn.conv2d(i, k, strides = [1, stride_y, stride_x, 1],
                                        padding = padding)
    with tf.variable_scope(name) as scope:
        weights = tf.get_variable('weights', shape = [filter_height, filter_width, input_channels/groups, num_filters])
        biases = tf.get_variable('biases', shape = [num_filters])
    
        if groups == 1:
            conv = convolve(x, weights)
        else:
            # Split input and weights and convolve them separately
            input_groups = tf.split(axis=3, num_or_size_splits=groups, value=x)
            weight_groups = tf.split(axis=3, num_or_size_splits=groups,
                                     value=weights)
            output_groups = [convolve(i, k) for i, k in zip(input_groups, weight_groups)]

            # Concat the convolved output together again
            conv = tf.concat(axis=3, values=output_groups)
        
        # Add biases
        bias = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape().as_list())
        
        # Apply relu function
        relu = tf.nn.relu(bias, name = scope.name)
        return relu

def fc(x, num_in, num_out, name, relu=True):
    with tf.variable_scope(name) as scope:
        weights = tf.get_variable('weights', shape=[num_in, num_out], trainable=True)
        biases = tf.get_variable('biases', [num_out], trainable=True)
    act = tf.nn.xw_plus_b(x, weights, biases, name=scope.name)
    
    if relu == True:
        relu = tf.nn.relu(act)
        return relu
    else:
        return act

def max_pool(x, filter_height, filter_width, stride_y, stride_x, name, padding='SAME'):
    return tf.nn.max_pool(x, ksize=[1, filter_height, filter_width, 1], strides=[1, stride_y, stride_x, 1], padding=padding, name=name)

def lrn(x, radius, alpha, beta, name, bias=1.0):
    return tf.nn.local_response_normalization(x, depth_radius=radius, alpha=alpha, beta=beta, bias=bias, name=name)

def dropout(x, keep_prob):
    return tf.nn.dropout(x, keep_prob)

from tensorflow.python.framework import dtypes
from tensorflow.python.framework.ops import convert_to_tensor
IMAGENET_MEAN = tf.constant([123.68, 116.779, 103.939], dtype=tf.float32)

class ImageDataGenerator(object):
    def __init__(self, phase, batch_size, buffer_size=1000):        
        self.phase = phase
        
        self._get_filenames()
        
        # number of smaples in dataset
        self.data_size = len(self.lbls)
        
        self.img_paths = convert_to_tensor(self.img_paths, dtype=dtypes.string)
        self.lbls = convert_to_tensor(self.lbls, dtype=dtypes.int32)
        
        # create dataset
        data = tf.data.Dataset.from_tensor_slices((self.img_paths, self.lbls))
        
        data = data.map(self._my_input_fn)
        data = data.batch(batch_size)
        self.data = data
        
    def _get_filenames(self):
        # root = '/data6/lekevin/cayman'
        root = '/data4/plankton_wi17/mpl/source_domain/spcbench/bench_data'
        dataset_version = 'V1b'
        #img_dir = root + '/bench_data/'
        img_dir = '/data5/Plankton_wi18/rawcolor_db2/images/'
        csv_filename = os.path.join(str(root), str(dataset_version), '{}.csv')
        dataset = PlanktonDataset(csv_filename=csv_filename.format(self.phase), img_dir=img_dir, phase=self.phase)
        print(dataset)
        
        self.num_classes = 2
        
        self.img_paths, self.lbls = dataset.get_fns()
        
    def _my_input_fn(self, filename, label):
        one_hot = tf.one_hot(label, self.num_classes)
        
        img_string = tf.read_file(filename)
        img_decoded = tf.image.decode_png(img_string, channels=3)
        img_resized = tf.image.resize_images(img_decoded, [227, 227])
        img_centered = tf.subtract(img_resized, IMAGENET_MEAN)
        img_bgr = img_centered[:, :, ::-1]
        
        return img_bgr, one_hot

# Learning Params
batch_size = 256
learning_rate = 0.001
num_epochs = 10

# Network params
dropout_rate = 0.5
train_layers = ['fc8', 'fc7', 'fc6']
num_classes = 2

# How often to write tf.summary data to disk
display_step = 20

"""
Main Part of the Finetuning Script
"""

# Place data loading and processing on the gpu
with tf.device('/device:GPU:1'):
    data = {phase: ImageDataGenerator(phase=phase, batch_size=batch_size) for phase in ['train', 'valid']}

    # Create reinitializable iterator given the dataset structure
    iterator = tf.data.Iterator.from_structure(data['train'].data.output_types,
                                                              data['train'].data.output_shapes)
    next_batch = iterator.get_next()

# Ops for intializing the iterator
init_op = {phase: iterator.make_initializer(data[phase].data) for phase in ['train','valid']}

x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)

# Initialize Model
model = CaffeNet(x, keep_prob, num_classes, train_layers)

# Link variable to model output
score = model.fc8

# List of trainable variables of the layers to train
var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers]

# Operation for calculating the loss
with tf.name_scope('cross_ent'):
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))

# Training optimization
with tf.name_scope("train"):
	# Get gradients of all trainable variables
    gradients = tf.gradients(loss, var_list)
    gradients = list(zip(gradients, var_list))
    
    # Create optimizer and apply gradient descent to the trainable variables
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    train_op = optimizer.apply_gradients(grads_and_vars=gradients)

# Add gradients to summary
for gradient, var in gradients:
    tf.summary.histogram(var.name.replace(':', '_') + '/gradient', gradient)

# Add variables to train to the summary
for var in var_list:
    tf.summary.histogram(var.name.replace(':', '_') , var)

# Add loss to summary
tf.summary.scalar('cross_entropy', loss)

# Evaluation operation: accuracy of the model
with tf.name_scope("accuracy"):
    correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Add accuracy to the summary
tf.summary.scalar('accuracy', accuracy)

# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = "/data6/lekevin/cayman/tensorflow_code/tmp/finetune_alexnet/tensorboard"
checkpoint_path = "/data6/lekevin/cayman/tensorflow_code/tmp/finetune_alexnet/checkpoints"

if not os.path.exists(checkpoint_path):
    os.makedirs(checkpoint_path)
    
merged_summary = tf.summary.merge_all() # Merge all summaries together
writer = tf.summary.FileWriter(filewriter_path) # Initialize filewriter
saver = tf.train.Saver() # Initialize saver

# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(data['train'].data_size/batch_size))
val_batches_per_epoch = int(np.floor(data['valid'].data_size/batch_size))

# Start Tensorflow session
with tf.Session() as sess:

    # Initialize all variables
    sess.run(tf.global_variables_initializer())
    
    # Add the model graph to TensorBoard
    writer.add_graph(sess.graph)
    
    # Load pretrained weights into non-trainable layer
    model.load_initial_weights(sess)
    
    print("{} Start training...".format(datetime.now()))
    print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
                                                      filewriter_path))
    
    # Loop over number of epochs
    for epoch in range(num_epochs):
        print("{} Epoch number: {}".format(datetime.now(), epoch+1))

        # Initialize iterator with the training dataset
        sess.run(init_op['train'])
        
        for step in range(train_batches_per_epoch):

        	# Get next batch of data
            img_batch, label_batch = sess.run(next_batch)
        
        	# And run the training operation
            sess.run(train_op, feed_dict={x: img_batch,
                                             y: label_batch,
                                             keep_prob: 1.})

            # Generate summary with the current batch of data and write to file
            if step%display_step==0:
                s = sess.run(merged_summary, feed_dict={x: img_batch,
                                                 y: label_batch,
                                                 keep_prob: 1.})
                writer.add_summary(s, epoch*train_batches_per_epoch + step)
        
        # Validate the model on the entire validation set
        print("{} Start validation".format(datetime.now()))
        sess.run(init_op['valid'])
        test_acc = 0.
        test_count = 0
        for _ in range(val_batches_per_epoch):
            img_batch, label_batch = sess.run(next_batch)
            acc = sess.run(accuracy, feed_dict={x: img_batch,
                                             y: label_batch,
                                             keep_prob: 1.})
            test_acc += acc
            test_count += 1
        test_acc /= test_count
        print("{} Validation Accuracy = {:.4f}".format(datetime.now(), test_acc))
        print("Saving checkpoint of model...".format(datetime.now()))
        
        # Save checkpoint of the model
        checkpoint_name = os.path.join(checkpoint_path, 'model_epoch' + str(epoch+1) + '.ckpt')
        save_path = saver.save(sess, checkpoint_name)
        print("{} Model checkpoint saved at {}".format(datetime.now(), checkpoint_name))

model.py

def _activation_summary(x):
  """Helper to create summaries for activations.

  Creates a summary that provides a histogram of activations.
  Creates a summary that measures the sparsity of activations.

  Args:
    x: Tensor
  Returns:
    nothing
  """
  # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
  # session. This helps the clarity of presentation on tensorboard.
  tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name)
  tf.summary.histogram(tensor_name + '/activations', x)
  tf.summary.scalar(tensor_name + '/sparsity',
                                       tf.nn.zero_fraction(x))

def inference(images):
  """Build the CIFAR-10 model.

  Args:
    images: Images returned from distorted_inputs() or inputs().

  Returns:
    Logits.
  """
	pass

def _variable_on_cpu(name, shape, initializer):
  """Helper to create a Variable stored on CPU memory.

  Args:
    name: name of the variable
    shape: list of ints
    initializer: initializer for Variable

  Returns:
    Variable Tensor
  """
  with tf.device('/cpu:0'):
    dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
    var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype)
  return var