Skip to content

Instantly share code, notes, and snippets.

@ktl014
Created August 7, 2018 14:49
Show Gist options
  • Save ktl014/23f52f4b9e95b6d6c39ec1feb8ba31d9 to your computer and use it in GitHub Desktop.
Save ktl014/23f52f4b9e95b6d6c39ec1feb8ba31d9 to your computer and use it in GitHub Desktop.
# Importing data from csv file
csv_filename = './tweets.csv'
dataset = tf.contrib.data.make_csv_dataset(csv_filename, batch_size=32)
dataset = dataset.shuffle(buffer_size=100)
iter = dataset.make_one_shot_iterator()
next = iter.get_next()
features, labels = next['text'], next['sentiment']
with tf.Session() as sess:
sess.run([features, labels])
# Creating an iterator
x = tf.placeholder(tf.float32, shape=[None,2])
dataset = tf.data.Dataset.from_tensor_slices(x) # Create dataset instance from placeholder
data = np.random.sample((100,2))
iter = dataset.make_initializable_iterator() # Creating iterator from dataset instance
el = iter.get_next()
with tf.Session() as sess:
sess.run(iter.initializer, feed_dict={x:data}) # initialize iterator
print(sess.run(el))
# Creating simple model using batching and switch between train & test dataset using initializable iterator
epochs = 10
batch_size = tf.placeholder(tf.int64)
BATCH_SIZE = 32
x,y = tf.placeholder(tf.float32, shape=[None, 2]), tf.placeholder(tf.float32, shape=[None,1])
dataset = tf.data.Dataset.from_tensor_slices((x,y)).batch(batch_size).repeat()
train_data = (np.random.sample((100,2)), np.random.sample((100,1)))
test_data = (np.random.sample((20,2)), np.random.sample((20,1)))
iter = dataset.make_initializable_iterator()
features, labels = iter.get_next()
net = tf.layers.dense(features, 8, activation=tf.tanh)
net = tf.layers.dense(net, 8, activation=tf.tanh)
prediction = tf.layers.dense(net, 1, activation=tf.tanh)
n_batches = train_data[0].shape[0] // BATCH_SIZE
loss = tf.losses.mean_squared_error(prediction, labels)
optimizer = tf.train.AdamOptimizer().minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(iter.initializer, feed_dict={ x: train_data[0], y: train_data[1], batch_size: BATCH_SIZE})
print("Training...")
for i in range(epochs):
total_loss = 0
for _ in range(n_batches):
_, loss_value = sess.run([optimizer, loss])
total_loss += loss_value
print("EPOCH: {}, Loss: {:.4f}".format(i, total_loss/n_batches))
sess.run(iter.initializer, feed_dict={x:test_data[0], y:test_data[1], batch_size:test_data[0].shape[0]})
print('Test Loss: {:.4f}'.format(sess.run(loss)))
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tensorflow as tf
def input(eval_data, data_dir, batch_size):
# Get file names from data dir
# Create queue to produce filenames to read
# Read examples from files in filename queue
"""
Preprocessing
"""
pass
def distorted_input(data_dir, batch_size):
# Get file names from data dir
pass
def _generate_image_label_batches(image, label, min_queue_examples,
batch_size, shuffle):
"""
NOTE: purpose of function is just to differentiate between shuffling distorted and
straight input batches
"""
# Read batch size from the queue
# Add training images to the visualizer
# tf.summary.image('images', images)
# return images, tf.reshape(label_batch, [batch_size])
pass
def load_data(data_dir):
# Get all subdirectories of data_dir. Each represents a label.
directories = [d for d in os.listdir(data_dir)
if os.path.isdir(os.path.join(data_dir, d))]
# Loop through the label directories and collect the data in
# two lists, labels and images.
labels = []
images = []
for d in directories:
label_dir = os.path.join(data_dir, d)
file_names = [os.path.join(label_dir, f)
for f in os.listdir(label_dir)
if f.endswith(".ppm")]
for f in file_names:
images.append(data.imread(f))
labels.append(int(d))
return images, labels
ROOT_PATH = "/Users/ktl014/PycharmProjects/PersonalProjects/tensorflow_tutorial/tf_datacamp/TensorFlow Tutorial For Beginners"
train_data_dir = os.path.join(ROOT_PATH, "Training")
test_data_dir = os.path.join(ROOT_PATH, "Testing")
images, labels = load_data(train_data_dir)
images_array = np.array(images)
labels_array = np.array(labels)
# Resize images
images32 = [transform.resize(image, (28, 28)) for image in images]
images32 = np.array(images32)
images32 = rgb2gray(np.array(images32))
x = tf.placeholder(dtype = tf.float32, shape = [None, 28, 28]) # placeholders ~ variables for feeding data into computation graph w/out data
y = tf.placeholder(dtype = tf.int32, shape = [None])
images_flat = tf.contrib.layers.flatten(x) # flatten ~ flattens input while maintaining batch size; 1st dim = batch
logits = tf.contrib.layers.fully_connected(images_flat, 62, tf.nn.relu) # fully_connected ~ adds fully connected layer, outputs feature vector
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels = y, logits = logits)) # reduce_mean ~ computes mean of elements across tensor dim
# sparse_softmax_cross... ~
train_op = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss) # AdamOptimizer().minimize() ~ Class AdamOptimizer adds operatiosn to minimize loss (computing and applying gradient)
correct_pred = tf.argmax(logits, 1) # argmax() ~ returns index with largest value across axes of a tensor
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
sess = tf.Session()
sess.run(tf.global_variables_initializer()) # global_variables... ~ initializes global variables in graph
for i in range(201):
print('EPOCH', i)
_, accuracy_val = sess.run([train_op, accuracy], feed_dict={x: images32, y: labels})
if i % 10 == 0:
print("Loss: ", loss)
print('DONE WITH EPOCH')
# Load the test data
test_images, test_labels = load_data(test_data_dir)
# Transform the images to 28 by 28 pixels
test_images28 = [transform.resize(image, (28, 28)) for image in test_images]
# Convert to grayscale
from skimage.color import rgb2gray
test_images28 = rgb2gray(np.array(test_images28))
# Run predictions against the full test set.
predicted = sess.run([correct_pred], feed_dict={x: test_images28})[0]
# Calculate correct matches
match_count = sum([int(y == y_) for y, y_ in zip(test_labels, predicted)])
# Calculate the accuracy
accuracy = match_count / float(len(test_labels))
# Print the accuracy
print("Accuracy: {:.3f}".format(accuracy))
import tensorflow as tf
import numpy as np
import sys
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
from datetime import datetime
#sys.path.insert(0,'/data6/lekevin/cayman/')
sys.path.insert(0, '/data4/plankton_wi17/mpl/source_domain/spcbench/bench_finetune/code/')
#from tools.dataset import SPCDataset
from create_dataset import PlanktonDataset
import tensorflow.contrib.slim.nets as nets
tf.reset_default_graph()
class CaffeNet(object):
def __init__(self, x, keep_prob, num_classes, skip_layer, weights_path='DEFAULT'):
"""
Inputs:
- x: tf.placeholder, for the input images
- keep_prob: tf.placeholder, for the dropout rate
- num_classes: int, number of classes of the new dataset
- skip_layer: list of strings, names of the layers you want to reinitialize
- weights_path: path string, path to the pretrained weights,
(if bvlc_alexnet.npy is not in the same folder)
"""
self.X = x
self.NUM_CLASSES = num_classes
self.KEEP_PROB = keep_prob
self.SKIP_LAYER = skip_layer
if weights_path == 'DEFAULT':
self.WEIGHTS_PATH = '/data6/lekevin/cayman/tensorflow_code/caffenet.npy'
else:
self.WEIGHTS_PATH = weights_path
# Build computational graph of AlexNet
self.create()
def create(self):
"""Create network graph"""
# 1st Layer: Conv (w/ReLu) -> Pool -> Norm
conv1 = conv(self.X, 11, 11, 96, 4, 4, padding='VALID', name='conv1')
pool1 = max_pool(conv1, 3, 3, 2, 2, padding='VALID', name='pool1')
norm1 = lrn(pool1, 2, 1e-04, 0.75, name='norm1')
# 2nd Layer: Conv (w/ReLu) -> Pool -> Norm
conv2 = conv(norm1, 5, 5, 256, 1, 1, groups=2, name='conv2')
pool2 = max_pool(conv2, 3, 3, 2, 2, padding='VALID', name='pool2')
norm2 = lrn(pool2, 2, 1e-04, 0.75, name='norm2_a')
# 3rd Layer: Conv (w/ ReLu)
conv3 = conv(norm2, 3, 3, 384, 1, 1, name='conv3')
# 4th Layer: Conv(w/ ReLu)
conv4 = conv(conv3, 3, 3, 384, 1, 1, groups=2, name='conv4')
# 5th Layer: Conv (w/ ReLu)
conv5 = conv(conv4, 3, 3, 256, 1, 1, groups=2, name='conv5')
pool5 = max_pool(conv5, 3, 3, 2, 2, padding='VALID', name='pool5')
# 6th Layer: Flatten --> FC (w/ ReLu) -> Dropout
flattened = tf.reshape(pool5, [-1, 6*6*256])
fc6 = fc(flattened, 6*6*256, 4096, name='fc6')
dropout6 = dropout(fc6, self.KEEP_PROB)
# 7th Layer: FC (w ReLu) -> Dropout
fc7 = fc(dropout6, 4096, 4096, name='fc7')
dropout7 = dropout(fc7, self.KEEP_PROB)
# 8th Layer: FC and return unscaled activations
self.fc8 = fc(dropout7, 4096, self.NUM_CLASSES, relu=False, name='fc8_spcbench')
def load_initial_weights(self, session):
weights_dict = np.load(self.WEIGHTS_PATH, encoding='bytes').item()
for op_name in weights_dict:
if op_name not in self.SKIP_LAYER:
with tf.variable_scope(op_name, reuse=True):
for data in weights_dict[op_name]:
if len(weights_dict[op_name][data].shape) == 1:
var = tf.get_variable('biases', trainable=False)
session.run(var.assign(weights_dict[op_name][data]))
else:
var = tf.get_variable('weights', trainable=False)
session.run(var.assign(weights_dict[op_name][data]))
def conv(x, filter_height, filter_width, num_filters, stride_y, stride_x, name, padding='SAME', groups=1):
input_channels = int(x.get_shape()[-1])
convolve = lambda i, k: tf.nn.conv2d(i, k, strides = [1, stride_y, stride_x, 1],
padding = padding)
with tf.variable_scope(name) as scope:
weights = tf.get_variable('weights', shape = [filter_height, filter_width, input_channels/groups, num_filters])
biases = tf.get_variable('biases', shape = [num_filters])
if groups == 1:
conv = convolve(x, weights)
else:
# Split input and weights and convolve them separately
input_groups = tf.split(axis=3, num_or_size_splits=groups, value=x)
weight_groups = tf.split(axis=3, num_or_size_splits=groups,
value=weights)
output_groups = [convolve(i, k) for i, k in zip(input_groups, weight_groups)]
# Concat the convolved output together again
conv = tf.concat(axis=3, values=output_groups)
# Add biases
bias = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape().as_list())
# Apply relu function
relu = tf.nn.relu(bias, name = scope.name)
return relu
def fc(x, num_in, num_out, name, relu=True):
with tf.variable_scope(name) as scope:
weights = tf.get_variable('weights', shape=[num_in, num_out], trainable=True)
biases = tf.get_variable('biases', [num_out], trainable=True)
act = tf.nn.xw_plus_b(x, weights, biases, name=scope.name)
if relu == True:
relu = tf.nn.relu(act)
return relu
else:
return act
def max_pool(x, filter_height, filter_width, stride_y, stride_x, name, padding='SAME'):
return tf.nn.max_pool(x, ksize=[1, filter_height, filter_width, 1], strides=[1, stride_y, stride_x, 1], padding=padding, name=name)
def lrn(x, radius, alpha, beta, name, bias=1.0):
return tf.nn.local_response_normalization(x, depth_radius=radius, alpha=alpha, beta=beta, bias=bias, name=name)
def dropout(x, keep_prob):
return tf.nn.dropout(x, keep_prob)
from tensorflow.python.framework import dtypes
from tensorflow.python.framework.ops import convert_to_tensor
IMAGENET_MEAN = tf.constant([123.68, 116.779, 103.939], dtype=tf.float32)
class ImageDataGenerator(object):
def __init__(self, phase, batch_size, buffer_size=1000):
self.phase = phase
self._get_filenames()
# number of smaples in dataset
self.data_size = len(self.lbls)
self.img_paths = convert_to_tensor(self.img_paths, dtype=dtypes.string)
self.lbls = convert_to_tensor(self.lbls, dtype=dtypes.int32)
# create dataset
data = tf.data.Dataset.from_tensor_slices((self.img_paths, self.lbls))
data = data.map(self._my_input_fn)
data = data.batch(batch_size)
self.data = data
def _get_filenames(self):
# root = '/data6/lekevin/cayman'
root = '/data4/plankton_wi17/mpl/source_domain/spcbench/bench_data'
dataset_version = 'V1b'
#img_dir = root + '/bench_data/'
img_dir = '/data5/Plankton_wi18/rawcolor_db2/images/'
csv_filename = os.path.join(str(root), str(dataset_version), '{}.csv')
dataset = PlanktonDataset(csv_filename=csv_filename.format(self.phase), img_dir=img_dir, phase=self.phase)
print(dataset)
self.num_classes = 2
self.img_paths, self.lbls = dataset.get_fns()
def _my_input_fn(self, filename, label):
one_hot = tf.one_hot(label, self.num_classes)
img_string = tf.read_file(filename)
img_decoded = tf.image.decode_png(img_string, channels=3)
img_resized = tf.image.resize_images(img_decoded, [227, 227])
img_centered = tf.subtract(img_resized, IMAGENET_MEAN)
img_bgr = img_centered[:, :, ::-1]
return img_bgr, one_hot
# Learning Params
batch_size = 256
learning_rate = 0.001
num_epochs = 10
# Network params
dropout_rate = 0.5
train_layers = ['fc8', 'fc7', 'fc6']
num_classes = 2
# How often to write tf.summary data to disk
display_step = 20
"""
Main Part of the Finetuning Script
"""
# Place data loading and processing on the gpu
with tf.device('/device:GPU:1'):
data = {phase: ImageDataGenerator(phase=phase, batch_size=batch_size) for phase in ['train', 'valid']}
# Create reinitializable iterator given the dataset structure
iterator = tf.data.Iterator.from_structure(data['train'].data.output_types,
data['train'].data.output_shapes)
next_batch = iterator.get_next()
# Ops for intializing the iterator
init_op = {phase: iterator.make_initializer(data[phase].data) for phase in ['train','valid']}
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize Model
model = CaffeNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
score = model.fc8
# List of trainable variables of the layers to train
var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers]
# Operation for calculating the loss
with tf.name_scope('cross_ent'):
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
# Training optimization
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name.replace(':', '_') + '/gradient', gradient)
# Add variables to train to the summary
for var in var_list:
tf.summary.histogram(var.name.replace(':', '_') , var)
# Add loss to summary
tf.summary.scalar('cross_entropy', loss)
# Evaluation operation: accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = "/data6/lekevin/cayman/tensorflow_code/tmp/finetune_alexnet/tensorboard"
checkpoint_path = "/data6/lekevin/cayman/tensorflow_code/tmp/finetune_alexnet/checkpoints"
if not os.path.exists(checkpoint_path):
os.makedirs(checkpoint_path)
merged_summary = tf.summary.merge_all() # Merge all summaries together
writer = tf.summary.FileWriter(filewriter_path) # Initialize filewriter
saver = tf.train.Saver() # Initialize saver
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(data['train'].data_size/batch_size))
val_batches_per_epoch = int(np.floor(data['valid'].data_size/batch_size))
# Start Tensorflow session
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load pretrained weights into non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch+1))
# Initialize iterator with the training dataset
sess.run(init_op['train'])
for step in range(train_batches_per_epoch):
# Get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training operation
sess.run(train_op, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
# Generate summary with the current batch of data and write to file
if step%display_step==0:
s = sess.run(merged_summary, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
writer.add_summary(s, epoch*train_batches_per_epoch + step)
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
sess.run(init_op['valid'])
test_acc = 0.
test_count = 0
for _ in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
acc = sess.run(accuracy, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
test_acc += acc
test_count += 1
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(datetime.now(), test_acc))
print("Saving checkpoint of model...".format(datetime.now()))
# Save checkpoint of the model
checkpoint_name = os.path.join(checkpoint_path, 'model_epoch' + str(epoch+1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(datetime.now(), checkpoint_name))
def _activation_summary(x):
"""Helper to create summaries for activations.
Creates a summary that provides a histogram of activations.
Creates a summary that measures the sparsity of activations.
Args:
x: Tensor
Returns:
nothing
"""
# Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
# session. This helps the clarity of presentation on tensorboard.
tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name)
tf.summary.histogram(tensor_name + '/activations', x)
tf.summary.scalar(tensor_name + '/sparsity',
tf.nn.zero_fraction(x))
def inference(images):
"""Build the CIFAR-10 model.
Args:
images: Images returned from distorted_inputs() or inputs().
Returns:
Logits.
"""
pass
def _variable_on_cpu(name, shape, initializer):
"""Helper to create a Variable stored on CPU memory.
Args:
name: name of the variable
shape: list of ints
initializer: initializer for Variable
Returns:
Variable Tensor
"""
with tf.device('/cpu:0'):
dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype)
return var
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment