Analogy Evaluation for Word Embeddings

analogyeval.py

import pandas
import numpy as np
from sklearn.preprocessing import normalize

# load any set of word embeddings like that
vecs = {"word": np.array([0, 1, 2, 3, 4, 5]}
# load the word2vec analogies
analogies = pandas.read_csv("questions-words.txt", names=["a", "b", "c", "d"], sep=" ")


def analogy(vecs, analogies):
    keys = list(vecs.keys())
    X = normalize(np.vstack((vecs[key] for key in keys)), axis=1)
    for col in analogies.columns:
        analogies[col] = analogies[col].str.lower()
        analogies["vec" + col] = [vecs.get(word, None) for word in analogies[col]]
    analogies = analogies.dropna()
    analogies["3ca"] = (analogies["vecb"] - analogies["veca"] + analogies["vecc"]).apply(normalize)
    Y = np.vstack(analogies["3ca"].values)
    bestids = np.hstack((np.argmax(X.dot(Y[i - 1000:min(i, len(analogies))].T), axis=0).reshape(1, -1) for i in list(range(1000, 20001, 1000))))
    revind = dict(zip(keys, range(len(keys))))
    trueids = np.array([revind[word] for word in analogies["d"]])
    print("Accuracy:\t" + str(len(trueids[(trueids == bestids)[0]]) * 1.0 / len(trueids)))

Not clean, not really beautiful, but should work.