Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Sentiment classification using transfer learning
import collections, math, random, numpy
import tensorflow as tf
from sklearn.cross_validation import train_test_split
sentences = """hated the movie it was stupid;\ni hated it so boring;\nawesome the movie was inspiring;\nhated it what a disaster;\nwe hated the movie they were idiotic;\nhe was stupid, hated her;\nstupid movie is boring;\ninspiring ourselves, awesome;\ninspiring me, brilliant;\nwe hated it they were rubbish;\nany inspiring movie is amazing;\nit was stupid what a disaster;\nits stupid, rubbish;\nstupid, idiotic!;\nawesome great movie;\nboring, must be hated;\nhe was boring the movie was stupid;\nboring movie was a disaster;\nboth boring and rubbish;\nso boring and idiotic;\ngreat to amazing;\ndisaster, more than hated;\nbetween disaster and stupid;\ndisaster, so boring;\nawesome movie, brilliant;\ntoo awesome she was amazing;\nhe was brilliant loved it;\ndisaster, only idiotic;\nrubbish movie hated him;\nit was rubbish, why so stupid?;\nrubbish, too boring;\nrubbish, disaster!;\nrubbish, very idiotic;\nidiotic movie hated it;\nshe was idiotic, so stupid;\nidiotic movie, it was boring;\nit was idiotic, movie was a disaster;\nidiotic and rubbish;\nI loved it, it was awesome;\nhe was stupid, hated her;\nbrilliant, loved it;\nloved the movie amazing;\nit was great loved the movie;\nmovie was great, inspiring;\ngreat movie, awesome;\nthey were great, brilliant;\ngreat amazing!;\nThey were inspiring loved it;\ninspiring movie great;\nawesome loved it;\nthey were brilliant great movie;\nshe was brilliant the movie was inspiring;\nhe was brilliant between them they were awesome;\nbrilliant, above amazing;\njust amazing loved it;\ndisaster, o what rubbish;\nabove amazing beyond inspiring;\nso amazing movie was awesome;\namazing brilliant movie;"""
words = [word for word in sentences.replace(';', '').replace('!', '').replace('?', '').replace(',', '').lower().split() if word.lower() not in ['it', 'movie', 'the' 'was', 'were', 'so', 'a', 'i', 'he', 'her', 'me', 'any', 'its', 'be', 'they', 'and']]
vocabulary_size = 15
def build_dataset(words):
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
dictionary = dict()
for word, _ in count:
dictionary[word] = len(dictionary)
data = list()
for word in words:
if word in dictionary:
index = dictionary[word]
else:
index = 0
data.append(index)
return data, dictionary
data, dictionary = build_dataset(words)
data_index = 0
def generate_batch(batch_size, num_skips, skip_window):
global data_index
batch = numpy.ndarray(shape=(batch_size), dtype=numpy.int32)
labels = numpy.ndarray(shape=(batch_size, 1), dtype=numpy.int32)
span = 2 * skip_window + 1
buffer = collections.deque(maxlen=span)
for _ in range(span):
buffer.append(data[data_index])
data_index = (data_index + 1) % len(data)
for i in range(batch_size // num_skips):
target = skip_window
targets_to_avoid = [skip_window]
for j in range(num_skips):
while target in targets_to_avoid:
target = random.randint(0, span - 1)
targets_to_avoid.append(target)
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[target]
buffer.append(data[data_index])
data_index = (data_index + 1) % len(data)
return batch, labels
batch_size = 20
embedding_size = 4
skip_window = 3
num_skips = 2
valid_size = 4
valid_window = 10
valid_examples = numpy.random.choice(valid_window, valid_size, replace=False)
num_sampled = 6
graph = tf.Graph()
with graph.as_default():
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
with tf.device('/cpu:0'):
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
nce_weights = tf.Variable(
tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
loss = tf.reduce_mean(
tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels,
num_sampled, vocabulary_size))
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(
normalized_embeddings, valid_dataset)
similarity = tf.matmul(
valid_embeddings, normalized_embeddings, transpose_b=True)
init = tf.global_variables_initializer()
with tf.Session(graph=graph) as session:
init.run()
average_loss = 0
for step in range(10001):
batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
_, loss_val, normalized_embeddings_np = session.run([optimizer, loss, normalized_embeddings], feed_dict=feed_dict)
average_loss += loss_val
final_embeddings = normalized_embeddings.eval()
neg_sent="""hated the movie it was stupid;\ni hated it so boring;\nhated it what a disaster;\nwe hated it they were rubbish;\nwe hated the movie they were idiotic;\nhe was stupid, hated her;\nstupid movie is boring;\nit was stupid what a disaster;\nits stupid, rubbish;\nstupid, idiotic!;""".replace(';', '').replace('!', '').replace('?', '').replace(',', '').lower().split('\n')
pos_sent="""I loved it, it was awesome;\nbrilliant, loved it;\nloved the movie amazing;\nit was great loved the movie;\nmovie was great, inspiring;\ngreat movie, awesome;\nthey were great, brilliant;\ngreat amazing!;\nThey were inspiring loved it;\ninspiring movie great;""".replace(';', '').replace('!', '').replace('?', '').replace(',', '').lower().split('\n')
x_sent = numpy.zeros((len(pos_sent + neg_sent), embedding_size), dtype=numpy.float)
y_sent = numpy.zeros((len(pos_sent + neg_sent), 2), dtype=numpy.int)
for i, sent in enumerate(pos_sent):
y_sent[i, 0] = 1
x_sent[i] = numpy.average([final_embeddings[dictionary[word]] for word in sent.split() if word in dictionary], axis=0)
for i, sent in enumerate(neg_sent):
y_sent[len(pos_sent) + i, 1] = 1
x_sent[len(pos_sent) + i] = numpy.average([final_embeddings[dictionary[word]] for word in sent.split() if word in dictionary], axis=0)
train_X, test_X, train_y, test_y = train_test_split(x_sent, y_sent, test_size=0.10, random_state=42)
x_size = embedding_size
y_size = 2
h_size = 4
X = tf.placeholder("float", shape=[None, x_size])
y = tf.placeholder("float", shape=[None, y_size])
w_1 = tf.Variable(tf.random_normal((x_size, h_size), stddev=0.1))
w_2 = tf.Variable(tf.random_normal((h_size, y_size), stddev=0.1))
h = tf.nn.sigmoid(tf.matmul(X, w_1))
yhat = tf.matmul(h, w_2)
predict = tf.argmax(yhat, dimension=1)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(yhat, y))
updates = tf.train.GradientDescentOptimizer(0.01).minimize(cost)
sess = tf.InteractiveSession()
init = tf.initialize_all_variables()
sess.run(init)
for epoch in range(1000):
for i in range(len(train_X)):
sess.run(updates, feed_dict={X: train_X[i: i + 1], y: train_y[i: i + 1]})
train_accuracy = numpy.mean(numpy.argmax(train_y, axis=1) == sess.run(predict, feed_dict={X: train_X, y: train_y}))
test_accuracy = numpy.mean(numpy.argmax(test_y, axis=1) == sess.run(predict, feed_dict={X: test_X, y: test_y}))
print("Epoch = %d, train accuracy = %.2f%%, test accuracy = %.2f%%" % (epoch + 1, 100. * train_accuracy, 100. * test_accuracy))
sess.close()
@YiningTeng
Copy link

Hi prats226, while trying to run your code, I got a strange error, I wonder if you have any clue for it? Thank you very much!

Traceback (most recent call last):
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 510, in _apply_op_helper
preferred_dtype=default_dtype)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1040, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 883, in _TensorTensorConversionFunction
(dtype.name, t.dtype.name, str(t)))
ValueError: Tensor conversion requested dtype int32 for Tensor with dtype float32: 'Tensor("nce_loss/Slice_1:0", shape=(?, 4), dtype=float32, device=/device:CPU:0)'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/yining/Downloads/pycharm-2018.1/helpers/pydev/pydev_run_in_console.py", line 53, in run_file
pydev_imports.execfile(file, globals, locals) # execute the script
File "/home/yining/Downloads/pycharm-2018.1/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home/yining/PycharmProjects/CNIT_NLP/TransferLearningHElloWorld.py", line 77, in
num_sampled, vocabulary_size))
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/nn_impl.py", line 1241, in nce_loss
name=name)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/nn_impl.py", line 1069, in _compute_sampled_logits
sampled_logits = math_ops.matmul(inputs, sampled_w, transpose_b=True)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/math_ops.py", line 2108, in matmul
a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/gen_math_ops.py", line 4209, in mat_mul
name=name)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 546, in _apply_op_helper
inferred_from[input_arg.type_attr]))
TypeError: Input 'b' of 'MatMul' Op has type float32 that does not match type int32 of argument 'a'.

@ClaudeCoulombe
Copy link

Greetings YiningTeng,

Versions always changed, so change those lines:

loss = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels, num_sampled, vocabulary_size))
=>
loss = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, train_labels, embed, num_sampled, vocabulary_size))
and

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(yhat, y))
=>
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=yhat, labels=y))

similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
=>
similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))

Maybe also...

X = tf.placeholder("float", shape=[None, x_size])
y = tf.placeholder("float", shape=[None, y_size])
=>
X = tf.placeholder(tf.float32, shape=[None, x_size])
y = tf.placeholder(tf.float32, shape=[None, y_size])

and

init = tf.initialize_all_variables()
=>
init = tf.global_variables_initializer()

@jizefeng0810
Copy link

Can author update the code?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment