C_Word2Vec_TF_NCE_Test_Freeze_Layer_C.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from Utils.FS import file\n",
    "from nltk.tokenize.punkt import PunktSentenceTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import brown\n",
    "from scipy.sparse import coo_matrix, dok_matrix\n",
    "from scipy.sparse.linalg import svds\n",
    "from sklearn.preprocessing import normalize\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.manifold import TSNE\n",
    "from wordcloud import WordCloud\n",
    "from nltk.corpus import stopwords\n",
    "import tensorflow as tf\n",
    "import math\n",
    "import TextPreprocess.words2dict as words2dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sents = brown.sents()\n",
    "sents = [[token.lower() for token in sent] for sent in sents]\n",
    "words = brown.words()\n",
    "words = [word.lower() for word in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens: 1161192\n",
      "Number of sentences: 57340\n",
      "Longest sentences length: 180\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of tokens: {}\".format(len(words)))\n",
    "print(\"Number of sentences: {}\".format(len(sents)))\n",
    "print(\"Longest sentences length: {}\".format(max([len(sent) for sent in sents])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique tokens: 49815\n"
     ]
    }
   ],
   "source": [
    "words_dict, inv_words_dict = words2dict.convert(words)\n",
    "words_size = len(words_dict)\n",
    "print(\"Number of unique tokens: {}\".format(words_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def singleSideWindow(sents, words_dict, window_size, reverse = False):\n",
    "    window = []\n",
    "    row = []\n",
    "    col = []\n",
    "    data = []\n",
    "        \n",
    "    for sent in reversed(sents) if reverse else sents:\n",
    "        for word in reversed(sent) if reverse else sent:\n",
    "            for w in window:\n",
    "                if w == word:\n",
    "                    continue\n",
    "                row.append(words_dict[word])\n",
    "                col.append(words_dict[w])\n",
    "                data.append(1)\n",
    "            if len(window) == window_size:\n",
    "                window.pop(0)\n",
    "            window.append(word)\n",
    "    return coo_matrix((data, (row, col)), shape=(len(words_dict), len(words_dict)), dtype='float64')\n",
    "    \n",
    "\n",
    "def sents2wordContextMatrix(sents, words_dict, window_size = 5):\n",
    "    m = coo_matrix((words_size, words_size), 'float64')\n",
    "    \n",
    "    print('Doing forward pass...')\n",
    "    m += singleSideWindow(sents, words_dict, window_size)\n",
    "    \n",
    "    print('Doing backward pass...')\n",
    "    m += singleSideWindow(sents, words_dict, window_size, True)\n",
    "    \n",
    "    return m\n",
    "\n",
    "def sents2wordCoocurrenceMatrix(sents, words_dict, window_size = 10):\n",
    "    #don't really care edge cases....\n",
    "\n",
    "    window = []\n",
    "    row = []\n",
    "    col = []\n",
    "    data = []\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            for i in range(len(window)- 1):\n",
    "                for j in range(i+1, len(window)):\n",
    "                    row += [words_dict[window[i]], words_dict[window[j]]]\n",
    "                    col += [words_dict[window[j]], words_dict[window[i]]]\n",
    "                    data += [1, 1]\n",
    "            if len(window) == window_size:\n",
    "                window.pop(0)\n",
    "            window.append(word)\n",
    "    print('Preparing sparse matrix...')\n",
    "    print('Length of data: {}'.format(len(data)))\n",
    "    return coo_matrix((data, (row,col)), shape=(words_size, words_size), dtype='float64').multiply(1/len(data))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def singleSideWindowGenerator(c, w, sents, words_dict, window_size = 5, batch_size = 32, reverse = False):\n",
    "    window = ['--' for i in range(window_size)]\n",
    "    for sent in reversed(sents) if reverse else sents:\n",
    "        for word in reversed(sent) if reverse else sent:\n",
    "            for context in window:\n",
    "                c.append(words_dict[context])\n",
    "                w.append([words_dict[word]])\n",
    "                if(len(c) == batch_size):\n",
    "                    yield c, w\n",
    "                    c = []\n",
    "                    w = []\n",
    "            window.pop(0)\n",
    "            window.append(word)\n",
    "\n",
    "def sents2batchGenerator(sents, words_dict, window_size = 5, batch_size=32):\n",
    "    c = []\n",
    "    w = []\n",
    "\n",
    "    while(True):\n",
    "        window = ['--' for i in range(window_size)]\n",
    "        for sent in sents:\n",
    "            for word in sent:\n",
    "                for context in window:\n",
    "                    c.append([words_dict[context]])\n",
    "                    w.append(words_dict[word])\n",
    "                    if(len(c) == batch_size):\n",
    "                        yield w, c\n",
    "                        c = []\n",
    "                        w = []\n",
    "                window.pop(0)\n",
    "                window.append(word)\n",
    "\n",
    "        window = ['--' for i in range(window_size)]\n",
    "        for sent in reversed(sents):\n",
    "            for word in reversed(sent):\n",
    "                for context in window:\n",
    "                    c.append([words_dict[context]])\n",
    "                    w.append(words_dict[word])\n",
    "                    if(len(c) == batch_size):\n",
    "                        yield w, c\n",
    "                        c = []\n",
    "                        w = []\n",
    "                window.pop(0)\n",
    "                window.append(word)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sents2freq(sents):\n",
    "    freq = {}\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            if word in freq.keys():\n",
    "                freq[word] += 1\n",
    "            else:\n",
    "                freq[word] = 1\n",
    "    return freq\n",
    "\n",
    "words_freq = sents2freq(sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "WINDOW_SIZE = 2\n",
    "BATCH_SIZE = 32\n",
    "generator = sents2batchGenerator(sents, words_dict, window_size = WINDOW_SIZE, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "DIMENSION = 50\n",
    "VOCABULAY_SIZE = len(words_dict)\n",
    "NEGATIVE_SAMPLE = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    \n",
    "    inputs = tf.placeholder(tf.int32, shape=[BATCH_SIZE])\n",
    "    labels = tf.placeholder(tf.int32, shape=[BATCH_SIZE, 1])\n",
    "\n",
    "    with tf.device('/cpu:0'):\n",
    "    \n",
    "        embeddings = tf.Variable(\n",
    "            tf.random_uniform([VOCABULAY_SIZE, DIMENSION], -1.0, 1.0))\n",
    "        embed = tf.nn.embedding_lookup(embeddings, inputs, max_norm=1)\n",
    "\n",
    "        nce_weights = tf.Variable(\n",
    "            tf.truncated_normal([VOCABULAY_SIZE, DIMENSION],\n",
    "                                stddev=1.0 / math.sqrt(DIMENSION)), trainable=False)\n",
    "\n",
    "        nce_biases = tf.Variable(tf.zeros([VOCABULAY_SIZE]), trainable=False)\n",
    "\n",
    "        loss = tf.reduce_mean(\n",
    "          tf.nn.nce_loss(weights=nce_weights,\n",
    "                     biases=nce_biases,\n",
    "                     labels=labels,\n",
    "                     inputs=embed,\n",
    "                     num_sampled=NEGATIVE_SAMPLE,\n",
    "                     num_classes=VOCABULAY_SIZE))\n",
    "\n",
    "        #optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "        optimizer = tf.train.MomentumOptimizer(1.0, 0.5).minimize(loss)\n",
    "        #optimizer = tf.train.AdamOptimizer().minimize(loss) # super slow\n",
    "        \n",
    "        init = tf.global_variables_initializer()\n",
    "        word2VecSaver = tf.train.Saver({'NCE_Weights': nce_weights, 'NCE_Biases': nce_biases})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def cloestWord(word, words_vec, count = 10, method=None):\n",
    "    if method == 'cos':\n",
    "        dist = np.array([ sum(words_vec[word] * words_vec[key]) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[::-1][:10]\n",
    "    else:\n",
    "        dist = np.array([ sum(np.square(np.array(words_vec[word]) - np.array(words_vec[key]))) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[:10]\n",
    "    return [list(words_vec.keys())[i] for i in top_ten]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/brown-Words2Vec-50.ckpt\n",
      "Average loss at step  2000 :  6.57848383498\n",
      "['two', 'efficient', 'other', 'real', 'for', \"'\", 'step', 'god', 'yale', 'is']\n",
      "Average loss at step  4000 :  6.03950827336\n",
      "['two', 'other', \"'\", 'real', 'efficient', 'step', 'time', 'attention', 'before', 'best']\n",
      "Average loss at step  6000 :  5.77292838049\n",
      "['two', 'other', 'political', 'first', 'men', 'best', 'western', 'time', 'step', 'day']\n",
      "Average loss at step  8000 :  5.70323363233\n",
      "['two', 'other', 'first', 'men', 'london', 'best', 'given', 'time', 'very', 'western']\n",
      "Average loss at step  10000 :  5.48292535138\n",
      "['two', 'other', 'many', 'first', 'years', 'political', 'step', 'real', 'london', 'western']\n",
      "Average loss at step  12000 :  5.33735982132\n",
      "['two', 'other', 'first', 'years', 'political', 'attention', 'cities', 'step', 'most', 'continued']\n",
      "Average loss at step  14000 :  5.35958235884\n",
      "['two', 'other', 'years', 'first', 'cities', 'attention', 'western', 'best', 'political', 'efficient']\n",
      "Average loss at step  16000 :  5.39006040812\n",
      "['two', 'other', 'first', 'three', 'cities', 'most', 'years', 'best', 'prime', 'emotional']\n",
      "Average loss at step  18000 :  5.29760877228\n",
      "['two', 'other', 'first', 'same', 'three', 'most', 'cities', 'best', 'place', 'these']\n",
      "Average loss at step  20000 :  5.25021736431\n",
      "['two', 'first', 'other', 'three', 'same', 'cities', 'most', 'emotional', 'best', 'days']\n",
      "Average loss at step  22000 :  5.29387792301\n",
      "['two', 'first', 'other', 'three', 'same', 'cities', 'most', 'best', 'days', 'emotional']\n",
      "Average loss at step  24000 :  5.34763185692\n",
      "['two', 'first', 'three', 'other', 'same', 'most', 'cities', 'emotional', 'london', 'following']\n",
      "Average loss at step  26000 :  5.37150803638\n",
      "['two', 'three', 'other', 'first', 'same', 'most', 'cities', 'days', 'many', 'emotional']\n",
      "Average loss at step  28000 :  5.45284001303\n",
      "['two', 'other', 'three', 'first', 'same', 'most', 'emotional', 'all', 'cities', 'next']\n",
      "Average loss at step  30000 :  5.2968873136\n",
      "['two', 'three', 'other', 'first', 'same', 'most', 'next', 'all', 'following', 'emotional']\n",
      "Average loss at step  32000 :  5.28134200048\n",
      "['two', 'other', 'three', 'first', 'same', 'most', 'prime', 'next', 'following', 'side']\n",
      "Average loss at step  34000 :  5.23364107347\n",
      "['two', 'three', 'other', 'first', 'most', 'same', 'following', 'next', 'minutes', 'prime']\n",
      "Average loss at step  36000 :  5.29432342362\n",
      "['two', 'three', 'first', 'other', 'most', 'before', 'several', 'begin', 'a', 'all']\n",
      "Average loss at step  38000 :  5.29249332643\n",
      "['two', 'three', 'first', 'other', 'most', 'same', 'several', 'next', 'last', 'sides']\n",
      "Average loss at step  40000 :  5.23743906879\n",
      "['two', 'three', 'first', 'other', 'most', 'same', 'before', 'several', 'next', 'only']\n",
      "Average loss at step  42000 :  5.18836245966\n",
      "['two', 'three', 'first', 'other', 'begin', 'before', 'some', 'ten', 'several', 'after']\n",
      "Average loss at step  44000 :  5.34686196613\n",
      "['two', 'three', 'other', 'before', 'begin', 'ten', 'years', 'first', 'cheek', 'card']\n",
      "Average loss at step  46000 :  5.18227294517\n",
      "['two', 'three', 'years', 'before', 'ten', 'other', 'minutes', 'days', 'card', 'begin']\n",
      "Average loss at step  48000 :  5.08706278133\n",
      "['two', 'three', 'years', 'days', 'ten', 'other', 'minutes', 'before', 'these', 'ago']\n",
      "Average loss at step  50000 :  5.08641372514\n",
      "['two', 'three', 'days', 'years', 'ten', 'these', 'minutes', 'other', 'ago', 'before']\n",
      "Average loss at step  52000 :  5.06834110117\n",
      "['two', 'three', 'minutes', 'days', 'ten', 'years', 'ago', 'other', 'these', 'four']\n",
      "Average loss at step  54000 :  5.00099687243\n",
      "['two', 'three', 'ten', 'minutes', 'days', 'other', 'ago', 'these', 'four', 'years']\n",
      "Average loss at step  56000 :  4.993351753\n",
      "['two', 'three', 'ten', 'four', 'other', 'minutes', 'days', 'these', 'ago', 'few']\n",
      "Average loss at step  58000 :  4.9656892513\n",
      "['two', 'three', 'ten', 'other', 'four', 'few', 'days', 'minutes', 'these', 'many']\n",
      "Average loss at step  60000 :  5.01449389172\n",
      "['two', 'three', 'ten', 'four', 'other', 'many', 'few', 'these', 'days', 'minutes']\n",
      "Average loss at step  62000 :  4.97930092144\n",
      "['two', 'three', 'ten', 'other', 'four', 'few', 'many', 'these', 'five', 'days']\n",
      "Average loss at step  64000 :  5.00204757535\n",
      "['two', 'three', 'ten', 'four', 'other', 'few', 'many', 'these', 'five', 'several']\n",
      "Average loss at step  66000 :  5.0213359127\n",
      "['two', 'three', 'ten', 'four', 'few', 'other', 'these', 'many', 'five', 'several']\n",
      "Average loss at step  68000 :  4.96242860961\n",
      "['two', 'three', 'ten', 'other', 'many', 'few', 'four', 'these', 'several', 'five']\n",
      "Average loss at step  70000 :  4.97196233869\n",
      "['two', 'three', 'many', 'few', 'these', 'other', 'ten', 'several', 'some', 'four']\n",
      "Average loss at step  72000 :  4.94542736959\n",
      "['two', 'three', 'many', 'some', 'several', 'other', 'these', 'few', 'four', 'ten']\n",
      "Average loss at step  74000 :  4.91748528957\n",
      "['two', 'three', 'several', 'many', 'some', 'four', 'few', 'other', 'ten', 'five']\n",
      "Average loss at step  76000 :  4.89626541996\n",
      "['two', 'three', 'several', 'many', 'some', 'few', 'four', 'ten', 'other', 'five']\n",
      "Average loss at step  78000 :  4.87346191752\n",
      "['two', 'three', 'several', 'many', 'some', 'few', 'these', 'four', 'other', 'ten']\n",
      "Average loss at step  80000 :  5.0261722033\n",
      "['two', 'three', 'several', 'many', 'ten', 'four', 'few', 'five', 'other', 'these']\n",
      "Average loss at step  82000 :  4.99116965175\n",
      "['two', 'three', 'several', 'few', 'many', 'ten', 'four', 'five', 'other', 'these']\n",
      "Average loss at step  84000 :  4.96531359553\n",
      "['two', 'three', 'several', 'few', 'many', 'ten', 'four', 'five', 'these', 'some']\n",
      "Average loss at step  86000 :  4.95895332122\n",
      "['two', 'three', 'several', 'many', 'few', 'four', 'ten', 'five', 'some', 'these']\n",
      "Average loss at step  88000 :  5.03150780416\n",
      "['two', 'three', 'several', 'many', 'few', 'four', 'ten', 'five', 'some', 'these']\n",
      "Average loss at step  90000 :  5.01271961308\n",
      "['two', 'three', 'several', 'few', 'many', 'four', 'five', 'ten', 'some', 'these']\n",
      "Average loss at step  92000 :  5.08355231977\n",
      "['two', 'three', 'several', 'few', 'many', 'four', 'five', 'ten', 'some', 'these']\n",
      "Average loss at step  94000 :  5.03624957633\n",
      "['two', 'three', 'several', 'few', 'many', 'four', 'five', 'ten', 'some', 'hundred']\n",
      "Average loss at step  96000 :  5.04594976544\n",
      "['two', 'three', 'few', 'several', 'four', 'five', 'ten', 'many', 'hundred', 'six']\n",
      "Average loss at step  98000 :  4.93645152295\n",
      "['two', 'three', 'several', 'few', 'four', 'five', 'many', 'ten', 'hundred', 'six']\n",
      "Average loss at step  100000 :  5.03599874663\n",
      "['two', 'three', 'several', 'few', 'four', 'five', 'ten', 'many', 'hundred', 'some']\n",
      "Average loss at step  102000 :  5.08460411167\n",
      "['two', 'three', 'several', 'few', 'four', 'five', 'ten', 'many', 'hundred', 'some']\n",
      "Average loss at step  104000 :  5.12525860381\n",
      "['two', 'three', 'several', 'few', 'four', 'five', 'ten', 'hundred', 'many', 'some']\n",
      "Average loss at step  106000 :  5.21065674353\n",
      "['two', 'three', 'several', 'few', 'four', 'five', 'many', 'ten', 'some', 'hundred']\n",
      "Average loss at step  108000 :  5.09805604935\n",
      "['two', 'three', 'several', 'few', 'four', 'five', 'many', 'ten', 'hundred', 'some']\n",
      "Average loss at step  110000 :  5.09977544093\n",
      "['two', 'three', 'several', 'few', 'four', 'five', 'many', 'ten', 'hundred', 'some']\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-12f0c83a28cc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0maverage_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mbatch_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-ee1be9ff3945>\u001b[0m in \u001b[0;36msents2batchGenerator\u001b[0;34m(sents, words_dict, window_size, batch_size)\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreversed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mcontext\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mwindow\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m                     \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwords_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m                     \u001b[0mw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwords_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m                     \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "num_steps = 200000\n",
    "WORDS2VEC_MODEL = './model/brown-Words2Vec-{}.ckpt'.format(DIMENSION)\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    init.run()\n",
    "    word2VecSaver.restore(session, WORDS2VEC_MODEL)   \n",
    "      \n",
    "    average_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batch_inputs, batch_labels = next(generator)\n",
    "        feed_dict = {inputs: batch_inputs, labels: batch_labels}\n",
    "\n",
    "        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "                print('Average loss at step ', step, ': ', average_loss)\n",
    "                average_loss = 0\n",
    "                \n",
    "                emb = embeddings.eval()\n",
    "                normalize(emb, norm='l2', axis=1, copy=False)\n",
    "                words_vec = {}\n",
    "                for i in range(emb.shape[0]):\n",
    "                    words_vec[inv_words_dict[i]] = emb[i]\n",
    "                print(cloestWord('two', words_vec, method='cos'))\n",
    "                \n",
    "                \n",
    "    #save_path = word2VecSaver.save(session, MODEL)\n",
    "    final_embeddings = embeddings.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "normalize(final_embeddings, norm='l2', axis=1, copy=False)\n",
    "print(final_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sum((final_embeddings[2] ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "words_vec = {}\n",
    "for i in range(final_embeddings.shape[0]):\n",
    "    words_vec[inv_words_dict[i]] = final_embeddings[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotData(vocabs, X, Y):\n",
    "    plt.clf()\n",
    "    plt.figure(figsize=(36, 36))\n",
    "    plt.scatter(X, Y)\n",
    "    plt.axis([min(X), max(X), min(Y), max(Y)])\n",
    "    for label, x, y in zip(vocabs, X, Y):\n",
    "        plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')\n",
    "    plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot(vocabs, words_vec):\n",
    "    X = [words_vec[vocab][0] for vocab in vocabs]\n",
    "    Y = [words_vec[vocab][1] for vocab in vocabs]\n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotTSNE(vocabs, vectors):\n",
    "    tsne = TSNE(perplexity=30, n_components=2, n_iter=5000, random_state = 7890, method='exact')\n",
    "    #np.set_printoptions(suppress=True)\n",
    "    data = np.array([vectors[vocab] for vocab in vocabs])    \n",
    "    DATA = tsne.fit_transform(data)\n",
    "    X = DATA[:, 0]\n",
    "    Y = DATA[:, 1]\n",
    "    \n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from DataLoader import GloVe\n",
    "\n",
    "glove = GloVe.load2('./data/GloVe/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "vocabs = ['man', 'woman', 'king', 'queen', 'male', 'female', 'boy', 'girl']\n",
    "np.random.seed(1234)\n",
    "\n",
    "random_vocabs = []\n",
    "for i in np.random.randint(0, len(words_dict), 2000):\n",
    "    if inv_words_dict[i] in glove.keys():\n",
    "        random_vocabs.append(inv_words_dict[i])\n",
    "        \n",
    "print(len(random_vocabs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#plotTSNE(random_vocabs, words_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#plotTSNE(random_vocabs, glove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('man', words_vec, method='cos'))\n",
    "print(cloestWord('man', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('woman', words_vec, method='cos'))\n",
    "print(cloestWord('woman', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('however', words_vec, method='cos'))\n",
    "print(cloestWord('however', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('his', words_vec, method='cos'))\n",
    "print(cloestWord('his', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('zero', words_vec, method='cos'))\n",
    "print(cloestWord('zero', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('one', words_vec, method='cos'))\n",
    "print(cloestWord('one', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('two', words_vec, method='cos'))\n",
    "print(cloestWord('two', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}