import collections, math, random, numpy | |
import tensorflow as tf | |
from sklearn.cross_validation import train_test_split | |
sentences = """hated the movie it was stupid;\ni hated it so boring;\nawesome the movie was inspiring;\nhated it what a disaster;\nwe hated the movie they were idiotic;\nhe was stupid, hated her;\nstupid movie is boring;\ninspiring ourselves, awesome;\ninspiring me, brilliant;\nwe hated it they were rubbish;\nany inspiring movie is amazing;\nit was stupid what a disaster;\nits stupid, rubbish;\nstupid, idiotic!;\nawesome great movie;\nboring, must be hated;\nhe was boring the movie was stupid;\nboring movie was a disaster;\nboth boring and rubbish;\nso boring and idiotic;\ngreat to amazing;\ndisaster, more than hated;\nbetween disaster and stupid;\ndisaster, so boring;\nawesome movie, brilliant;\ntoo awesome she was amazing;\nhe was brilliant loved it;\ndisaster, only idiotic;\nrubbish movie hated him;\nit was rubbish, why so stupid?;\nrubbish, too boring;\nrubbish, disaster!;\nrubbish, very idiotic;\nidiotic movie hated it;\nshe was idiotic, so stupid;\nidiotic movie, it was boring;\nit was idiotic, movie was a disaster;\nidiotic and rubbish;\nI loved it, it was awesome;\nhe was stupid, hated her;\nbrilliant, loved it;\nloved the movie amazing;\nit was great loved the movie;\nmovie was great, inspiring;\ngreat movie, awesome;\nthey were great, brilliant;\ngreat amazing!;\nThey were inspiring loved it;\ninspiring movie great;\nawesome loved it;\nthey were brilliant great movie;\nshe was brilliant the movie was inspiring;\nhe was brilliant between them they were awesome;\nbrilliant, above amazing;\njust amazing loved it;\ndisaster, o what rubbish;\nabove amazing beyond inspiring;\nso amazing movie was awesome;\namazing brilliant movie;""" | |
words = [word for word in sentences.replace(';', '').replace('!', '').replace('?', '').replace(',', '').lower().split() if word.lower() not in ['it', 'movie', 'the' 'was', 'were', 'so', 'a', 'i', 'he', 'her', 'me', 'any', 'its', 'be', 'they', 'and']] | |
vocabulary_size = 15 | |
def build_dataset(words): | |
count = [['UNK', -1]] | |
count.extend(collections.Counter(words).most_common(vocabulary_size - 1)) | |
dictionary = dict() | |
for word, _ in count: | |
dictionary[word] = len(dictionary) | |
data = list() | |
for word in words: | |
if word in dictionary: | |
index = dictionary[word] | |
else: | |
index = 0 | |
data.append(index) | |
return data, dictionary | |
data, dictionary = build_dataset(words) | |
data_index = 0 | |
def generate_batch(batch_size, num_skips, skip_window): | |
global data_index | |
batch = numpy.ndarray(shape=(batch_size), dtype=numpy.int32) | |
labels = numpy.ndarray(shape=(batch_size, 1), dtype=numpy.int32) | |
span = 2 * skip_window + 1 | |
buffer = collections.deque(maxlen=span) | |
for _ in range(span): | |
buffer.append(data[data_index]) | |
data_index = (data_index + 1) % len(data) | |
for i in range(batch_size // num_skips): | |
target = skip_window | |
targets_to_avoid = [skip_window] | |
for j in range(num_skips): | |
while target in targets_to_avoid: | |
target = random.randint(0, span - 1) | |
targets_to_avoid.append(target) | |
batch[i * num_skips + j] = buffer[skip_window] | |
labels[i * num_skips + j, 0] = buffer[target] | |
buffer.append(data[data_index]) | |
data_index = (data_index + 1) % len(data) | |
return batch, labels | |
batch_size = 20 | |
embedding_size = 4 | |
skip_window = 3 | |
num_skips = 2 | |
valid_size = 4 | |
valid_window = 10 | |
valid_examples = numpy.random.choice(valid_window, valid_size, replace=False) | |
num_sampled = 6 | |
graph = tf.Graph() | |
with graph.as_default(): | |
train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) | |
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) | |
valid_dataset = tf.constant(valid_examples, dtype=tf.int32) | |
with tf.device('/cpu:0'): | |
embeddings = tf.Variable( | |
tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) | |
embed = tf.nn.embedding_lookup(embeddings, train_inputs) | |
nce_weights = tf.Variable( | |
tf.truncated_normal([vocabulary_size, embedding_size], | |
stddev=1.0 / math.sqrt(embedding_size))) | |
nce_biases = tf.Variable(tf.zeros([vocabulary_size])) | |
loss = tf.reduce_mean( | |
tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels, | |
num_sampled, vocabulary_size)) | |
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss) | |
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) | |
normalized_embeddings = embeddings / norm | |
valid_embeddings = tf.nn.embedding_lookup( | |
normalized_embeddings, valid_dataset) | |
similarity = tf.matmul( | |
valid_embeddings, normalized_embeddings, transpose_b=True) | |
init = tf.global_variables_initializer() | |
with tf.Session(graph=graph) as session: | |
init.run() | |
average_loss = 0 | |
for step in range(10001): | |
batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window) | |
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels} | |
_, loss_val, normalized_embeddings_np = session.run([optimizer, loss, normalized_embeddings], feed_dict=feed_dict) | |
average_loss += loss_val | |
final_embeddings = normalized_embeddings.eval() | |
neg_sent="""hated the movie it was stupid;\ni hated it so boring;\nhated it what a disaster;\nwe hated it they were rubbish;\nwe hated the movie they were idiotic;\nhe was stupid, hated her;\nstupid movie is boring;\nit was stupid what a disaster;\nits stupid, rubbish;\nstupid, idiotic!;""".replace(';', '').replace('!', '').replace('?', '').replace(',', '').lower().split('\n') | |
pos_sent="""I loved it, it was awesome;\nbrilliant, loved it;\nloved the movie amazing;\nit was great loved the movie;\nmovie was great, inspiring;\ngreat movie, awesome;\nthey were great, brilliant;\ngreat amazing!;\nThey were inspiring loved it;\ninspiring movie great;""".replace(';', '').replace('!', '').replace('?', '').replace(',', '').lower().split('\n') | |
x_sent = numpy.zeros((len(pos_sent + neg_sent), embedding_size), dtype=numpy.float) | |
y_sent = numpy.zeros((len(pos_sent + neg_sent), 2), dtype=numpy.int) | |
for i, sent in enumerate(pos_sent): | |
y_sent[i, 0] = 1 | |
x_sent[i] = numpy.average([final_embeddings[dictionary[word]] for word in sent.split() if word in dictionary], axis=0) | |
for i, sent in enumerate(neg_sent): | |
y_sent[len(pos_sent) + i, 1] = 1 | |
x_sent[len(pos_sent) + i] = numpy.average([final_embeddings[dictionary[word]] for word in sent.split() if word in dictionary], axis=0) | |
train_X, test_X, train_y, test_y = train_test_split(x_sent, y_sent, test_size=0.10, random_state=42) | |
x_size = embedding_size | |
y_size = 2 | |
h_size = 4 | |
X = tf.placeholder("float", shape=[None, x_size]) | |
y = tf.placeholder("float", shape=[None, y_size]) | |
w_1 = tf.Variable(tf.random_normal((x_size, h_size), stddev=0.1)) | |
w_2 = tf.Variable(tf.random_normal((h_size, y_size), stddev=0.1)) | |
h = tf.nn.sigmoid(tf.matmul(X, w_1)) | |
yhat = tf.matmul(h, w_2) | |
predict = tf.argmax(yhat, dimension=1) | |
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(yhat, y)) | |
updates = tf.train.GradientDescentOptimizer(0.01).minimize(cost) | |
sess = tf.InteractiveSession() | |
init = tf.initialize_all_variables() | |
sess.run(init) | |
for epoch in range(1000): | |
for i in range(len(train_X)): | |
sess.run(updates, feed_dict={X: train_X[i: i + 1], y: train_y[i: i + 1]}) | |
train_accuracy = numpy.mean(numpy.argmax(train_y, axis=1) == sess.run(predict, feed_dict={X: train_X, y: train_y})) | |
test_accuracy = numpy.mean(numpy.argmax(test_y, axis=1) == sess.run(predict, feed_dict={X: test_X, y: test_y})) | |
print("Epoch = %d, train accuracy = %.2f%%, test accuracy = %.2f%%" % (epoch + 1, 100. * train_accuracy, 100. * test_accuracy)) | |
sess.close() |
Greetings YiningTeng,
Versions always changed, so change those lines:
loss = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels, num_sampled, vocabulary_size))
=>
loss = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, train_labels, embed, num_sampled, vocabulary_size))
and
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(yhat, y))
=>
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=yhat, labels=y))
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
=>
similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))
Maybe also...
X = tf.placeholder("float", shape=[None, x_size])
y = tf.placeholder("float", shape=[None, y_size])
=>
X = tf.placeholder(tf.float32, shape=[None, x_size])
y = tf.placeholder(tf.float32, shape=[None, y_size])
and
init = tf.initialize_all_variables()
=>
init = tf.global_variables_initializer()
Can author update the code?
Hi prats226, while trying to run your code, I got a strange error, I wonder if you have any clue for it? Thank you very much!
Traceback (most recent call last):
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 510, in _apply_op_helper
preferred_dtype=default_dtype)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1040, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 883, in _TensorTensorConversionFunction
(dtype.name, t.dtype.name, str(t)))
ValueError: Tensor conversion requested dtype int32 for Tensor with dtype float32: 'Tensor("nce_loss/Slice_1:0", shape=(?, 4), dtype=float32, device=/device:CPU:0)'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/yining/Downloads/pycharm-2018.1/helpers/pydev/pydev_run_in_console.py", line 53, in run_file
pydev_imports.execfile(file, globals, locals) # execute the script
File "/home/yining/Downloads/pycharm-2018.1/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home/yining/PycharmProjects/CNIT_NLP/TransferLearningHElloWorld.py", line 77, in
num_sampled, vocabulary_size))
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/nn_impl.py", line 1241, in nce_loss
name=name)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/nn_impl.py", line 1069, in _compute_sampled_logits
sampled_logits = math_ops.matmul(inputs, sampled_w, transpose_b=True)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/math_ops.py", line 2108, in matmul
a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/ops/gen_math_ops.py", line 4209, in mat_mul
name=name)
File "/home/yining/PycharmProjects/CNIT_NLP/venv/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 546, in _apply_op_helper
inferred_from[input_arg.type_attr]))
TypeError: Input 'b' of 'MatMul' Op has type float32 that does not match type int32 of argument 'a'.