C_Word2Vec_RNN_NCE_C.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from Utils.FS import file\n",
    "from Corpus import brown\n",
    "from nltk.tokenize.punkt import PunktSentenceTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from scipy.sparse import coo_matrix, dok_matrix\n",
    "from scipy.sparse.linalg import svds\n",
    "from sklearn.preprocessing import normalize\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.manifold import TSNE\n",
    "from wordcloud import WordCloud\n",
    "from nltk.corpus import stopwords\n",
    "import tensorflow as tf\n",
    "import math\n",
    "import TextPreprocess.words2dict as words2dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import brown\n",
    "words = brown.words()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1161192\n",
      "83\n",
      "[('e', 589980), ('t', 423392), ('a', 371418), ('o', 357020), ('i', 333212), ('n', 332908), ('s', 300431), ('r', 287337), ('h', 249219), ('l', 192894), ('d', 184215), ('c', 139434), ('u', 127159), ('m', 113186), ('f', 106409), ('p', 90770), ('g', 89140), ('w', 83137), ('y', 80164), ('b', 66277), (',', 58982), ('.', 55578), ('v', 46206), ('k', 29685), (\"'\", 28683), ('`', 17674), ('T', 15568), ('-', 15401), ('I', 12543), ('A', 11385), ('S', 10322), ('x', 9379), ('H', 8015), ('C', 7776), ('M', 7455), ('B', 6527), ('W', 6003), (';', 5566), ('1', 5182), ('P', 5162), ('q', 4862), ('j', 4748), ('?', 4694), ('0', 4458), ('z', 4431), ('F', 4263), ('D', 4080), ('N', 3798), ('R', 3663), ('G', 3444)]\n"
     ]
    }
   ],
   "source": [
    "print(len(words))\n",
    "from nltk.probability import FreqDist\n",
    "\n",
    "chars = []\n",
    "for word in words:\n",
    "    chars += [char for char in word]\n",
    "        \n",
    "fdist = FreqDist(chars)\n",
    "print(len(fdist))\n",
    "print(fdist.most_common(50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def singleSideWindow(sents, words_dict, window_size, reverse = False):\n",
    "    window = []\n",
    "    row = []\n",
    "    col = []\n",
    "    data = []\n",
    "        \n",
    "    for sent in reversed(sents) if reverse else sents:\n",
    "        for word in reversed(sent) if reverse else sent:\n",
    "            for w in window:\n",
    "                if w == word:\n",
    "                    continue\n",
    "                row.append(words_dict[word])\n",
    "                col.append(words_dict[w])\n",
    "                data.append(1)\n",
    "            if len(window) == window_size:\n",
    "                window.pop(0)\n",
    "            window.append(word)\n",
    "    return coo_matrix((data, (row, col)), shape=(len(words_dict), len(words_dict)), dtype='float64')\n",
    "    \n",
    "\n",
    "def sents2wordContextMatrix(sents, words_dict, window_size = 5):\n",
    "    m = coo_matrix((words_size, words_size), 'float64')\n",
    "    \n",
    "    print('Doing forward pass...')\n",
    "    m += singleSideWindow(sents, words_dict, window_size)\n",
    "    \n",
    "    print('Doing backward pass...')\n",
    "    m += singleSideWindow(sents, words_dict, window_size, True)\n",
    "    \n",
    "    return m\n",
    "\n",
    "def sents2wordCoocurrenceMatrix(sents, words_dict, window_size = 10):\n",
    "    #don't really care edge cases....\n",
    "\n",
    "    window = []\n",
    "    row = []\n",
    "    col = []\n",
    "    data = []\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            for i in range(len(window)- 1):\n",
    "                for j in range(i+1, len(window)):\n",
    "                    row += [words_dict[window[i]], words_dict[window[j]]]\n",
    "                    col += [words_dict[window[j]], words_dict[window[i]]]\n",
    "                    data += [1, 1]\n",
    "            if len(window) == window_size:\n",
    "                window.pop(0)\n",
    "            window.append(word)\n",
    "    print('Preparing sparse matrix...')\n",
    "    print('Length of data: {}'.format(len(data)))\n",
    "    return coo_matrix((data, (row,col)), shape=(words_size, words_size), dtype='float64').multiply(1/len(data))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def singleSideWindowGenerator(c, w, sents, words_dict, window_size = 5, batch_size = 32, reverse = False):\n",
    "    window = ['--' for i in range(window_size)]\n",
    "    for sent in reversed(sents) if reverse else sents:\n",
    "        for word in reversed(sent) if reverse else sent:\n",
    "            for context in window:\n",
    "                c.append(words_dict[context])\n",
    "                w.append([words_dict[word]])\n",
    "                if(len(c) == batch_size):\n",
    "                    yield c, w\n",
    "                    c = []\n",
    "                    w = []\n",
    "            window.pop(0)\n",
    "            window.append(word)\n",
    "\n",
    "def sents2batchGenerator(sents, words_dict, window_size = 5, batch_size=32):\n",
    "    c = []\n",
    "    w = []\n",
    "\n",
    "    while(True):\n",
    "        window = ['--' for i in range(window_size)]\n",
    "        for sent in sents:\n",
    "            for word in sent:\n",
    "                for context in window:\n",
    "                    c.append([words_dict[context]])\n",
    "                    w.append(words_dict[word])\n",
    "                    if(len(c) == batch_size):\n",
    "                        yield w, c\n",
    "                        c = []\n",
    "                        w = []\n",
    "                window.pop(0)\n",
    "                window.append(word)\n",
    "\n",
    "        window = ['--' for i in range(window_size)]\n",
    "        for sent in reversed(sents):\n",
    "            for word in reversed(sent):\n",
    "                for context in window:\n",
    "                    c.append([words_dict[context]])\n",
    "                    w.append(words_dict[word])\n",
    "                    if(len(c) == batch_size):\n",
    "                        yield w, c\n",
    "                        c = []\n",
    "                        w = []\n",
    "                window.pop(0)\n",
    "                window.append(word)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sents2freq(sents):\n",
    "    freq = {}\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            if word in freq.keys():\n",
    "                freq[word] += 1\n",
    "            else:\n",
    "                freq[word] = 1\n",
    "    return freq\n",
    "\n",
    "words_freq = sents2freq(sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "WINDOW_SIZE = 2\n",
    "BATCH_SIZE = 32\n",
    "generator = sents2batchGenerator(sents, words_dict, window_size = WINDOW_SIZE, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "DIMENSION = 50\n",
    "VOCABULAY_SIZE = len(words_dict)\n",
    "NEGATIVE_SAMPLE = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    \n",
    "    inputs = tf.placeholder(tf.int32, shape=[BATCH_SIZE])\n",
    "    labels = tf.placeholder(tf.int32, shape=[BATCH_SIZE, 1])\n",
    "\n",
    "    with tf.device('/cpu:0'):\n",
    "    \n",
    "        embeddings = tf.Variable(\n",
    "            tf.random_uniform([VOCABULAY_SIZE, DIMENSION], -1.0, 1.0))\n",
    "        embed = tf.nn.embedding_lookup(embeddings, inputs, max_norm=1)\n",
    "\n",
    "        nce_weights = tf.Variable(\n",
    "            tf.truncated_normal([VOCABULAY_SIZE, DIMENSION],\n",
    "                                stddev=1.0 / math.sqrt(DIMENSION)))\n",
    "\n",
    "        nce_biases = tf.Variable(tf.zeros([VOCABULAY_SIZE]))\n",
    "\n",
    "        loss = tf.reduce_mean(\n",
    "          tf.nn.nce_loss(weights=nce_weights,\n",
    "                     biases=nce_biases,\n",
    "                     labels=labels,\n",
    "                     inputs=embed,\n",
    "                     num_sampled=NEGATIVE_SAMPLE,\n",
    "                     num_classes=VOCABULAY_SIZE))\n",
    "\n",
    "        #optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "        optimizer = tf.train.MomentumOptimizer(1.0, 0.5).minimize(loss)\n",
    "        #optimizer = tf.train.AdamOptimizer().minimize(loss) # super slow\n",
    "        \n",
    "        init = tf.global_variables_initializer()\n",
    "        word2VecSaver = tf.train.Saver({'Words2Vec': embeddings, 'NCE_Weights': nce_weights, 'NCE_Biases': nce_biases})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def cloestWord(word, words_vec, count = 10, method=None):\n",
    "    if method == 'cos':\n",
    "        dist = np.array([ sum(words_vec[word] * words_vec[key]) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[::-1][:10]\n",
    "    else:\n",
    "        dist = np.array([ sum(np.square(np.array(words_vec[word]) - np.array(words_vec[key]))) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[:10]\n",
    "    return [list(words_vec.keys())[i] for i in top_ten]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "num_steps = 200000\n",
    "MODEL = './model/brown-Words2Vec-{}.ckpt'.format(DIMENSION)\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    init.run()\n",
    "      \n",
    "    average_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batch_inputs, batch_labels = next(generator)\n",
    "        feed_dict = {inputs: batch_inputs, labels: batch_labels}\n",
    "\n",
    "        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "                print('Average loss at step ', step, ': ', average_loss)\n",
    "                average_loss = 0\n",
    "                \n",
    "                emb = embeddings.eval()\n",
    "                normalize(emb, norm='l2', axis=1, copy=False)\n",
    "                words_vec = {}\n",
    "                for i in range(emb.shape[0]):\n",
    "                    words_vec[inv_words_dict[i]] = emb[i]\n",
    "                print(cloestWord('two', words_vec, method='cos'))\n",
    "                \n",
    "                \n",
    "    save_path = word2VecSaver.save(session, MODEL)\n",
    "    final_embeddings = embeddings.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "normalize(final_embeddings, norm='l2', axis=1, copy=False)\n",
    "print(final_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sum((final_embeddings[2] ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "words_vec = {}\n",
    "for i in range(final_embeddings.shape[0]):\n",
    "    words_vec[inv_words_dict[i]] = final_embeddings[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotData(vocabs, X, Y):\n",
    "    plt.clf()\n",
    "    plt.figure(figsize=(36, 36))\n",
    "    plt.scatter(X, Y)\n",
    "    plt.axis([min(X), max(X), min(Y), max(Y)])\n",
    "    for label, x, y in zip(vocabs, X, Y):\n",
    "        plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')\n",
    "    plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot(vocabs, words_vec):\n",
    "    X = [words_vec[vocab][0] for vocab in vocabs]\n",
    "    Y = [words_vec[vocab][1] for vocab in vocabs]\n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotTSNE(vocabs, vectors):\n",
    "    tsne = TSNE(perplexity=30, n_components=2, n_iter=5000, random_state = 7890, method='exact')\n",
    "    #np.set_printoptions(suppress=True)\n",
    "    data = np.array([vectors[vocab] for vocab in vocabs])    \n",
    "    DATA = tsne.fit_transform(data)\n",
    "    X = DATA[:, 0]\n",
    "    Y = DATA[:, 1]\n",
    "    \n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from DataLoader import GloVe\n",
    "\n",
    "glove = GloVe.load2('./data/GloVe/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "vocabs = ['man', 'woman', 'king', 'queen', 'male', 'female', 'boy', 'girl']\n",
    "np.random.seed(1234)\n",
    "\n",
    "random_vocabs = []\n",
    "for i in np.random.randint(0, len(words_dict), 2000):\n",
    "    if inv_words_dict[i] in glove.keys():\n",
    "        random_vocabs.append(inv_words_dict[i])\n",
    "        \n",
    "print(len(random_vocabs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#plotTSNE(random_vocabs, words_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#plotTSNE(random_vocabs, glove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('man', words_vec, method='cos'))\n",
    "print(cloestWord('man', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('woman', words_vec, method='cos'))\n",
    "print(cloestWord('woman', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('however', words_vec, method='cos'))\n",
    "print(cloestWord('however', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('his', words_vec, method='cos'))\n",
    "print(cloestWord('his', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('zero', words_vec, method='cos'))\n",
    "print(cloestWord('zero', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('one', words_vec, method='cos'))\n",
    "print(cloestWord('one', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(cloestWord('two', words_vec, method='cos'))\n",
    "print(cloestWord('two', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}