word2vec in keras

document-embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import sys\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import keras\n",
    "from keras.layers import Dense, Activation, Dropout, Flatten, Merge, Layer, RepeatVector\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.models import Sequential\n",
    "from keras.preprocessing import text as keras_text\n",
    "import keras.preprocessing.sequence as keras_sequence\n",
    "\n",
    "import keras.backend as K\n",
    "\n",
    "import deeplearn\n",
    "\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "files = sorted(glob.glob('/data/filtered-papers/joined-json/part-r-*'))\n",
    "\n",
    "def all_paper_text():\n",
    "    for file in files[:1]:\n",
    "        print('Loading....', file)\n",
    "        with open(file) as f:\n",
    "            for line in f:\n",
    "                yield json.loads(line)['abstract']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "VOCAB_SIZE = 50000\n",
    "BATCH_SIZE = 32\n",
    "CONTEXT = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading.... /data/filtered-papers/joined-json/part-r-00000-4a18d38d-7ee1-4190-8c9a-f453040bea01\n",
      "Processing... 7.45MB."
     ]
    }
   ],
   "source": [
    "from deeplearn import preprocessing, display\n",
    "from importlib import reload\n",
    "reload(preprocessing)\n",
    "reload(display)\n",
    "tokenizer = preprocessing.Tokenizer(VOCAB_SIZE)\n",
    "tokenizer.fit(all_paper_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sampling_table = keras_sequence.make_sampling_table(tokenizer.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting epoch...\n",
      "Loading.... /data/filtered-papers/joined-json/part-r-00000-4a18d38d-7ee1-4190-8c9a-f453040bea01\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('foster', 'awareness', 1),\n",
       " ('within', 'materializable', 0),\n",
       " ('choosing', 'sibling', 0),\n",
       " ('robots', 'exhausted', 0),\n",
       " ('robots', 'uniprocessor', 0),\n",
       " ('predictors', 'takes', 0),\n",
       " ('tailored', 'can', 1),\n",
       " ('distinguishing', 'sdaz', 0),\n",
       " ('predictors', 'bias', 0),\n",
       " ('environments', 'in', 1)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HIDDEN_SIZE = 200\n",
    "\n",
    "def training_data(generator, sampling_table):\n",
    "    print('Starting epoch...')\n",
    "    for batch in tokenizer.terms(generator, 200):\n",
    "        ctx, labels = keras_sequence.skipgrams(batch,\n",
    "                                              tokenizer.size(), \n",
    "                                              window_size=4,\n",
    "                                              negative_samples=2.0,\n",
    "                                              sampling_table=sampling_table)\n",
    "\n",
    "        ctx = np.asarray(ctx)\n",
    "        labels = np.asarray(labels)\n",
    "        yield [ctx[:, 0], ctx[:, 1]], labels\n",
    "\n",
    "def words(lst):\n",
    "    return [tokenizer.index_to_term[t] for t in lst]\n",
    "\n",
    "_gen = training_data(all_paper_text(), sampling_table)\n",
    "couples, labels = next(_gen)\n",
    "list(zip(words(couples[0][:10]), words(couples[1][:10]), labels[:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "embedding_3 (Embedding)          (None, 1, 200)        10000000                                     \n",
      "____________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)          (None, 1, 200)        10000000                                     \n",
      "____________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)              (None, 1)             0           merge_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "activation_2 (Activation)        (None, 1)             0           flatten_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 20000000\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "from keras.regularizers import l2\n",
    "\n",
    "def logistic(y_true, y_pred):\n",
    "    return K.sum(y_true * -K.log(1e-5 + y_pred) +\n",
    "                 (1 - y_true) * -K.log(1e-5 + 1 - y_pred))\n",
    "\n",
    "def w2v_model():\n",
    "    with tf.device('/gpu:0'):\n",
    "        word = Sequential()\n",
    "        word.add(Embedding(\n",
    "                    input_dim=VOCAB_SIZE, output_dim=HIDDEN_SIZE, input_length=1, mask_zero=False,\n",
    "                    weights=None\n",
    "        ))\n",
    "\n",
    "        ctx = Sequential()\n",
    "        ctx.add(Embedding(\n",
    "                    input_dim=VOCAB_SIZE, output_dim=HIDDEN_SIZE, input_length=1, mask_zero=False,\n",
    "                    weights=None\n",
    "        ))\n",
    "\n",
    "        encoder = Sequential()\n",
    "        encoder.add(Merge([word, ctx], mode='dot', dot_axes=2))\n",
    "        encoder.add(Flatten())\n",
    "        encoder.add(Activation('sigmoid'))\n",
    "        encoder.compile(loss=logistic, optimizer='rmsprop')\n",
    "        return encoder\n",
    "\n",
    "training_model = w2v_model()\n",
    "training_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting epoch...Epoch 1/50\n",
      "\n",
      "Loading.... /data/filtered-papers/joined-json/part-r-00000-4a18d38d-7ee1-4190-8c9a-f453040bea01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/russellp/anaconda3/lib/python3.5/site-packages/Keras-1.0.4-py3.5.egg/keras/engine/training.py:1403: UserWarning: Epoch comprised more than `samples_per_epoch` samples, which might affect learning results. Set `samples_per_epoch` correctly to avoid this warning.\n",
      "  warnings.warn('Epoch comprised more than '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8s - loss: 588.3471\n",
      "Epoch 2/50\n",
      "5s - loss: 618.1930\n",
      "Epoch 3/50\n",
      "5s - loss: 599.8142\n",
      "Epoch 4/50\n",
      "5s - loss: 591.9885\n",
      "Epoch 5/50\n",
      "5s - loss: 629.2736\n",
      "Epoch 6/50\n",
      "5s - loss: 604.4747\n",
      "Epoch 7/50\n",
      "5s - loss: 613.7711\n",
      "Epoch 8/50\n",
      "5s - loss: 613.1408\n",
      "Epoch 9/50\n",
      "7s - loss: 586.3921\n",
      "Epoch 10/50\n",
      "4s - loss: 916.5526\n",
      "Epoch 11/50\n",
      "5s - loss: 599.5526\n",
      "Epoch 12/50\n",
      "6s - loss: 576.3741\n",
      "Epoch 13/50\n",
      "5s - loss: 629.2401\n",
      "Epoch 14/50\n",
      "4s - loss: 690.1956\n",
      "Epoch 15/50\n",
      "5s - loss: 584.9657\n",
      "Epoch 16/50\n",
      "8s - loss: 565.4906\n",
      "Epoch 17/50\n",
      "5s - loss: 559.3230\n",
      "Epoch 18/50\n",
      "5s - loss: 603.5484\n",
      "Epoch 19/50\n",
      "5s - loss: 581.4872\n",
      "Epoch 20/50\n",
      "5s - loss: 567.3234\n",
      "Epoch 21/50\n",
      "5s - loss: 580.4386\n",
      "Epoch 22/50\n",
      "5s - loss: 580.4246\n",
      "Epoch 23/50\n",
      "5s - loss: 568.5602\n",
      "Epoch 24/50\n",
      "5s - loss: 550.9526\n",
      "Epoch 25/50\n",
      "5s - loss: 561.8375\n",
      "Epoch 26/50\n",
      "5s - loss: 592.9327\n",
      "Epoch 27/50\n",
      "5s - loss: 557.0448\n",
      "Epoch 28/50\n",
      "5s - loss: 550.8710\n",
      "Epoch 29/50\n",
      "5s - loss: 546.0234\n",
      "Epoch 30/50\n",
      "5s - loss: 552.9884\n",
      "Epoch 31/50\n",
      "5s - loss: 548.4450\n",
      "Epoch 32/50\n",
      "5s - loss: 556.9434\n",
      "Epoch 33/50\n",
      "5s - loss: 545.6861\n",
      "Epoch 34/50\n",
      "5s - loss: 553.5572\n",
      "Epoch 35/50\n",
      "5s - loss: 494.1891\n",
      "Epoch 36/50\n",
      "5s - loss: 521.7431\n",
      "Epoch 37/50\n",
      "5s - loss: 538.1924\n",
      "Epoch 38/50\n",
      "5s - loss: 494.3068\n",
      "Epoch 39/50\n",
      "5s - loss: 546.7488\n",
      "Epoch 40/50\n",
      "5s - loss: 497.9315\n",
      "Epoch 41/50\n",
      "5s - loss: 516.8103\n",
      "Epoch 42/50\n",
      "5s - loss: 546.5882\n",
      "Epoch 43/50\n",
      "6s - loss: 489.9367\n",
      "Epoch 44/50\n",
      "5s - loss: 526.4753\n",
      "Epoch 45/50\n",
      "6s - loss: 490.0774\n",
      "Epoch 46/50\n",
      "7s - loss: 492.9176\n",
      "Epoch 47/50\n",
      "10s - loss: 498.3710\n",
      "Epoch 48/50\n",
      "7s - loss: 522.7182\n",
      "Epoch 49/50\n",
      "7s - loss: 471.5638\n",
      "Epoch 50/50\n",
      "7s - loss: 463.3277\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fa1880a5668>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_model.fit_generator(\n",
    "    training_data(all_paper_text(), sampling_table), \n",
    "    samples_per_epoch=40000,\n",
    "    verbose=2,\n",
    "    nb_epoch=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "position ['proposed', 'in', 'are', 'approach', 'position']\n",
      "dataset ['is', 'between', 'that', 'we', 'dataset']\n",
      "selected ['study', 'wide', 'in', 'problem', 'selected']\n",
      "outperforms ['to', 'design', 'statistical', 'for', 'outperforms']\n",
      "questions ['generating', 'particular', 'compared', 'improve', 'questions']\n",
      "evolutionary ['by', 'method', 'be', 'image', 'evolutionary']\n",
      "learn ['we', 'for', 'in', 'are', 'learn']\n",
      "external ['all', 'has', 'types', 'in', 'external']\n",
      "around ['is', 'at', 'on', 'and', 'around']\n",
      "precision ['in', 'approach', 'is', 'that', 'precision']\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import np_utils\n",
    "\n",
    "words = np_utils.normalize(training_model.layers[0].layers[0].get_weights()[0])\n",
    "ctx = np_utils.normalize(training_model.layers[0].layers[1].get_weights()[0])\n",
    "ctx[:200] = 0\n",
    "\n",
    "for i in range(1000, 1010):\n",
    "    dist = np.dot(words, words[i].T)\n",
    "    best = np.argsort(dist)\n",
    "    print(tokenizer.index_to_term[i], [tokenizer.index_to_term[t] for t in best[-5:]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.set_printoptions(precision=3, linewidth=100)\n",
    "\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "nn = NearestNeighbors(metric='cosine', algorithm='brute')\n",
    "nn = nn.fit(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('neural', -1.1920929e-07),\n",
       " ('be', 0.16951573),\n",
       " ('in', 0.17450088),\n",
       " ('these', 0.17672753),\n",
       " ('for', 0.1768989)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def neighbors(w):\n",
    "    dists, idxs = nn.kneighbors(w.reshape(1, -1))\n",
    "    idxs = idxs[0]\n",
    "    return list(zip([tokenizer.index_to_term[idx] for idx in idxs], dists[0]))\n",
    "\n",
    "def value(term):\n",
    "    return words[tokenizer.term_to_index[term]]\n",
    "\n",
    "neighbors(value('neural'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from IPython.core.display import HTML\n",
    "import urllib2\n",
    "def css_styling():\n",
    "    styles = urllib2.urlopen('https://raw.githubusercontent.com/barbagroup/CFDPython/master/styles/custom.css', \"r\").read()\n",
    "    return HTML(styles)\n",
    "css_styling()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}