C_RNN_GRU_NCE_Autoencoder_C.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from Utils.FS import file\n",
    "from Utils.tensorflow_helper import show_graph\n",
    "from nltk.tokenize.punkt import PunktSentenceTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import brown\n",
    "from scipy.sparse import coo_matrix, dok_matrix\n",
    "from scipy.sparse.linalg import svds\n",
    "from sklearn.preprocessing import normalize\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.manifold import TSNE\n",
    "from wordcloud import WordCloud\n",
    "from nltk.corpus import stopwords\n",
    "import tensorflow as tf\n",
    "import math\n",
    "import TextPreprocess.words2dict as words2dict\n",
    "from tensorflow.python.layers import core as layers_core\n",
    "from tensorflow.python.client import timeline\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sents = brown.sents()\n",
    "sents = [[token.lower() for token in sent] for sent in sents]\n",
    "words = brown.words()\n",
    "words = [word.lower() for word in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens: 1161192\n",
      "Number of sentences: 57340\n",
      "Longest sentences length: 180\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of tokens: {}\".format(len(words)))\n",
    "print(\"Number of sentences: {}\".format(len(sents)))\n",
    "print(\"Longest sentences length: {}\".format(max([len(sent) for sent in sents])))\n",
    "MAX_SENTENCE_LENGTH = max([len(sent) for sent in sents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "334\n",
      "five\n",
      "Number of unique tokens: 49815\n"
     ]
    }
   ],
   "source": [
    "words_dict, inv_words_dict = words2dict.convert(words)\n",
    "print(words_dict['five'])\n",
    "print(inv_words_dict[334])\n",
    "\n",
    "words_size = len(words_dict)\n",
    "print(\"Number of unique tokens: {}\".format(words_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "VOCABULAY_SIZE = len(words_dict)\n",
    "GO_SYMBOL = VOCABULAY_SIZE - 1\n",
    "PADDING_SYMBOL = VOCABULAY_SIZE - 2\n",
    "UNK_SYMBOL = VOCABULAY_SIZE - 3\n",
    "\n",
    "PAD = inv_words_dict[PADDING_SYMBOL]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def batchPadding(batch, padding_symbol=PADDING_SYMBOL):\n",
    "    size = max([len(record) for record in batch])\n",
    "    result = np.full((len(batch), size), padding_symbol)\n",
    "    for i in range(len(batch)):\n",
    "        result[i][:len(batch[i])] = batch[i]\n",
    "    return result\n",
    "\n",
    "def batchMask(batch):\n",
    "    size = max([len(record) for record in batch])\n",
    "    result = np.full((len(batch), size), 0.0)\n",
    "    for i in range(len(batch)):\n",
    "        result[i][:len(batch[i])] = 1.0\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ndef dataGenerator(sents, words_dict, window_size = 2, batch_size=32, train_length=2, epouch=-1, padding_symbol=PADDING_SYMBOL):\\n    train = []\\n    label = []\\n    length = []\\n    while(epouch < 0 or epouch > 0):\\n        left_window = [padding_symbol for i in range(window_size)]\\n        target = [padding_symbol for i in range(train_length)]\\n        right_window = [padding_symbol for i in range(window_size)]\\n        for sent in sents:\\n            for word in sent:\\n                right_window.append(words_dict[word])\\n                target.append(right_window.pop(0))\\n                left_window.append(target.pop(0))\\n                left_window.pop(0)\\n                \\n                for context in left_window + right_window:\\n                    train.append(list(target))\\n                    label.append(list([context]))\\n                    length.append(len(target))\\n                    if(len(train) == batch_size):\\n                        yield train, label, length\\n                        train = []\\n                        label = []\\n                        length = []\\n        epouch -= 1\\n        print('epouch done...')\\n\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "def dataGenerator(sents, words_dict, window_size = 2, batch_size=32, train_length=2, epouch=-1, padding_symbol=PADDING_SYMBOL):\n",
    "    train = []\n",
    "    label = []\n",
    "    length = []\n",
    "    while(epouch < 0 or epouch > 0):\n",
    "        left_window = [padding_symbol for i in range(window_size)]\n",
    "        target = [padding_symbol for i in range(train_length)]\n",
    "        right_window = [padding_symbol for i in range(window_size)]\n",
    "        for sent in sents:\n",
    "            for word in sent:\n",
    "                right_window.append(words_dict[word])\n",
    "                target.append(right_window.pop(0))\n",
    "                left_window.append(target.pop(0))\n",
    "                left_window.pop(0)\n",
    "                \n",
    "                for context in left_window + right_window:\n",
    "                    train.append(list(target))\n",
    "                    label.append(list([context]))\n",
    "                    length.append(len(target))\n",
    "                    if(len(train) == batch_size):\n",
    "                        yield train, label, length\n",
    "                        train = []\n",
    "                        label = []\n",
    "                        length = []\n",
    "        epouch -= 1\n",
    "        print('epouch done...')\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sentenceGenerator(sents, words_dict, batch_size=32, epouch=-1, padding_symbol=PADDING_SYMBOL):\n",
    "    train = []\n",
    "    length = []\n",
    "    while(epouch < 0 or epouch > 0):\n",
    "        for sent in sents:\n",
    "            train.append([words_dict[word] for word in sent])\n",
    "            length.append(len(sent))\n",
    "            if(len(train) == batch_size):\n",
    "                yield batchPadding(train), length, batchMask(train)\n",
    "                train = []\n",
    "                length = []\n",
    "        epouch -= 1\n",
    "        print('epouch done...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#TRAIN_LENGTH = 4\n",
    "#WINDOW_SIZE = 2\n",
    "BATCH_SIZE = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#generator = dataGenerator(sents, words_dict, window_size = WINDOW_SIZE, batch_size=BATCH_SIZE, train_length=TRAIN_LENGTH, epouch=1)\n",
    "#bigram_list = []\n",
    "#for batch_inputs, _, _ in generator:\n",
    "#    bigram_list += ['_'.join([inv_words_dict[idx] for idx in batch_input]) for batch_input in batch_inputs]\n",
    "        \n",
    "#bigrams_dict, inv_bigrams_dict = words2dict.convert(bigram_list)\n",
    "#g = sentenceGenerator(sents, words_dict, batch_size=1, epouch=1)\n",
    "#count = 0\n",
    "#for a,b,c in g:\n",
    "#    if len(a[0]) < 10:\n",
    "#        count = count + 1\n",
    "#print('total ', count)   \n",
    "#batch, lengths, mask = next(generator)\n",
    "#print(batch[0])\n",
    "#print(lengths)\n",
    "#print(mask[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ndef visualizeData(generator):\\n    train, label, length = next(generator)\\n    for i in range(len(train)):\\n        print([inv_words_dict[word] for word in train[i]], [inv_words_dict[word] for word in label[i]], length[i])\\n\\ngenerator = dataGenerator(sents[:1], words_dict, window_size = 1, batch_size=64, train_length=2)\\n#print(sents[0])\\n#visualizeData(generator)\\n'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "def visualizeData(generator):\n",
    "    train, label, length = next(generator)\n",
    "    for i in range(len(train)):\n",
    "        print([inv_words_dict[word] for word in train[i]], [inv_words_dict[word] for word in label[i]], length[i])\n",
    "\n",
    "generator = dataGenerator(sents[:1], words_dict, window_size = 1, batch_size=64, train_length=2)\n",
    "#print(sents[0])\n",
    "#visualizeData(generator)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#generator = dataGenerator(sents, words_dict, window_size = WINDOW_SIZE, batch_size=BATCH_SIZE, train_length=TRAIN_LENGTH)\n",
    "generator = sentenceGenerator(sents, words_dict, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79\n"
     ]
    }
   ],
   "source": [
    "TRAIN_EMBEDDING = False\n",
    "TRAIN_DISEMBEDDING = True\n",
    "DIMENSION = 50\n",
    "ENCODER_DIMENSION = [DIMENSION, 1000]\n",
    "ENCODER_LAYERS = len(ENCODER_DIMENSION)\n",
    "\n",
    "RNN_DIMENSION = [1000, DIMENSION]\n",
    "RNN_LAYERS = len(RNN_DIMENSION)\n",
    "\n",
    "\n",
    "NEGATIVE_SAMPLE = 256\n",
    "SOFTMAX_SAMPLE = int(np.log2(VOCABULAY_SIZE)) + 64\n",
    "MODE = 'train'\n",
    "\n",
    "print(SOFTMAX_SAMPLE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Decoder/Reshape:0\", shape=(?, ?, 1000), dtype=float32)\n",
      "Tensor(\"Decoder/concat:0\", shape=(?, ?, 1050), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Hoiy/anaconda/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/ops/gradients_impl.py:93: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    \n",
    "        #IN\n",
    "        inputs = tf.placeholder(tf.int32, (None, None), name = \"Input_Sentence_Word_Index\")\n",
    "        #OUT: (batch, time) int32\n",
    "        \n",
    "        batch_size = tf.shape(inputs)[0]\n",
    "        steps = tf.shape(inputs)[1]\n",
    "        \n",
    "        #IN\n",
    "        input_lengths = tf.placeholder(tf.int32, (None), name = \"Input_Sentence_Length\")\n",
    "        #OUT: (batch) int32\n",
    "                \n",
    "        batch_size = tf.shape(inputs)[0]\n",
    "        steps = tf.shape(inputs)[1]\n",
    "        \n",
    "        encoder_inputs = inputs\n",
    "        #OUT: (batch, time) int32\n",
    "        \n",
    "        #decoder_inputs = tf.pad(tf.slice(encoder_inputs, [0,1], [batch_size, steps-1]), [[1, 0], [0, 0]])\n",
    "        decoder_inputs = tf.pad(\n",
    "            tf.slice(encoder_inputs, [0,0], [batch_size, steps-1]) - GO_SYMBOL, \n",
    "            [[0, 0], [1, 0]]\n",
    "        ) + GO_SYMBOL\n",
    "        #OUT: (batch, time) int32\n",
    "        \n",
    "        decoder_input_lengths = tf.reshape(input_lengths, [batch_size])\n",
    "        #OUT: (batch) int32\n",
    "\n",
    "        # assume same input length\n",
    "        decoder_masks = tf.placeholder(tf.float32, (None, None), name = \"Input_Sentence_Mask\")\n",
    "        #OUT: (batch, time)\n",
    "        \n",
    "        #labels = tf.placeholder(tf.int32, (None, 1), name = \"Context_Word_Index\")\n",
    "        #OUT: (batch, 1) int32\n",
    "        \n",
    "\n",
    "        embeddings = tf.Variable(tf.random_uniform([VOCABULAY_SIZE, DIMENSION], -1.0, 1.0), trainable=TRAIN_EMBEDDING, name=\"Word2Vec\")\n",
    "        \n",
    "        #IN: (batch, time) int32\n",
    "        inputs_embed = tf.nn.embedding_lookup(embeddings, inputs, max_norm=1)\n",
    "        #OUT: (batch, time, dim) float32\n",
    "        decoder_embed = tf.nn.embedding_lookup(embeddings, decoder_inputs, max_norm=1)\n",
    "\n",
    "        \"\"\"\n",
    "        #IN: (batch, time, dim) \n",
    "        #weights = [tf.Variable(tf.random_uniform([DIMENSION * 2, DIMENSION], minval=-1, maxval=1)) for i in range(RNN_LAYERS)]\n",
    "        #bias = [tf.Variable(tf.random_uniform([DIMENSION], minval=-1, maxval=1)) for i in range(RNN_LAYERS)]\n",
    "        \n",
    "        #nn_input = tf.reshape(inputs_embed, (batch_size, DIMENSION))\n",
    "        #nn_output = tf.ones((batch_size, DIMENSION))\n",
    "        #for i in range(RNN_LAYERS):\n",
    "        #    nn_output = tf.concat([nn_output, nn_input], 1)\n",
    "        #    nn_output = tf.tanh(tf.matmul(nn_output, weights[i]) + bias[i])\n",
    "        #out: (batch, DIMENSION)\n",
    "\n",
    "        #IN: (batch, time, dim) float32\n",
    "        #rnn_inputs = tf.transpose(inputs_embed, [1, 0, 2])\n",
    "        #rnn_inputs = inputs_embed\n",
    "        #OUT: (batch, time, dim) float32\n",
    "        \n",
    "        #cell = tf.contrib.rnn.LSTMCell(DIMENSION)\n",
    "        cell = tf.contrib.rnn.GRUCell(DIMENSION)\n",
    "        \n",
    "        outputs, states = tf.contrib.legacy_seq2seq.embedding_rnn_seq2seq(\n",
    "            encoder_inputs = encoder_inputs,\n",
    "            decoder_inputs = decoder_inputs,\n",
    "            cell=cell,\n",
    "            num_encoder_symbols = VOCABULAY_SIZE,\n",
    "            num_decoder_symbols = VOCABULAY_SIZE,\n",
    "            embedding_size = DIMENSION,\n",
    "            output_projection=(\n",
    "                tf.Variable(tf.random_iniform([DIMENSION, VOCABULAY_SIZE], minval=-0.1 , maxval=0.1)),\n",
    "                tf.Variable(tf.random_iniform([VOCABULAY_SIZE], minval=-0.1 , maxval=0.1))\n",
    "            ),\n",
    "            feed_previous=False,\n",
    "        )\n",
    "        \n",
    "        loss = tf.contrib.legacy_seq2seq.sequence_loss(\n",
    "            logits=outputs,\n",
    "            targets=labels,\n",
    "            weights,\n",
    "            average_across_timesteps=True,\n",
    "            average_across_batch=True,\n",
    "            softmax_loss_function=None,\n",
    "            name=None\n",
    "        )\n",
    "        \"\"\"\n",
    "        # OUT: [time [batch_size, DIMENSION]]\n",
    "        \n",
    "        \n",
    "        ###### IN: (batch, time, DIMENSION) float32 ######\n",
    "        \n",
    "        with tf.variable_scope(\"Encoder\") as encoder_scope:\n",
    "\n",
    "            #cell = tf.contrib.rnn.LSTMCell(RNN_DIMENSION[-1])\n",
    "            #cell = tf.contrib.rnn.GRUCell(ENCODER_DIMENSION) \n",
    "            cells = [tf.contrib.rnn.GRUCell(ENCODER_DIMENSION[i]) for i in range(ENCODER_LAYERS)]\n",
    "            stack = tf.contrib.rnn.MultiRNNCell(cells)\n",
    "\n",
    "            #rnn_tuple_state = tuple([tf.nn.rnn_cell.LSTMStateTuple(l[i][0], l[i][1]) for i in range(RNN_LAYERS)])\n",
    "\n",
    "            #cell = tf.contrib.rnn.LSTMCell(DIMENSION, state_is_tuple=True)        \n",
    "            #cell = tf.contrib.rnn.GRUCell(DIMENSION)\n",
    "            #initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "\n",
    "            rnn_outputs, rnn_states = tf.nn.dynamic_rnn(stack, inputs_embed, dtype=tf.float32, sequence_length=input_lengths)\n",
    "            \n",
    "            #IN: (batch, time, RNN_DIMENSION[-1]) float32\n",
    "            index = tf.range(0, batch_size) * tf.shape(inputs)[1] + (input_lengths - 1)\n",
    "            rnn_final_state = tf.gather(tf.reshape(rnn_outputs, [-1, ENCODER_DIMENSION[-1]]), index)\n",
    "            rnn_final_state = tf.reshape( rnn_final_state, [-1, ENCODER_DIMENSION[-1]])\n",
    "            #rnn_final_state = tf.clip_by_norm(rnn_final_state, 1, axes=[1])\n",
    "            ###rnn_final_state = tf.gather(tf.reshape(rnn_outputs, [-1, DIMENSION]), index)\n",
    "            #OUT: (batch, RNN_DIMENSION[-1])\n",
    "            \n",
    "        ###### OUT: (batch, time, RNN_DIMENSION[-1]) float32 ######\n",
    "        \n",
    "        \n",
    "        ###### IN: (batch, time, RNN_DIMENSION[-1]) float32 ######\n",
    "        \n",
    "        with tf.variable_scope(\"Decoder\") as decoder_scope:\n",
    "            #cell = tf.contrib.rnn.LSTMCell(RNN_DIMENSION[-1])\n",
    "            cells = [tf.contrib.rnn.GRUCell(RNN_DIMENSION[i]) for i in range(RNN_LAYERS)]\n",
    "            stack = tf.contrib.rnn.MultiRNNCell(cells)\n",
    "            \n",
    "            decoder_init_state = [cell.zero_state(batch_size, tf.float32) for cell in cells]\n",
    "            decoder_init_state[0] = rnn_final_state\n",
    "            decoder_init_state = tuple(decoder_init_state)\n",
    "            \n",
    "            print(tf.reshape(tf.tile(rnn_final_state, [1, steps]), [-1, steps, ENCODER_DIMENSION[-1]]))\n",
    "            \n",
    "            decoder_inputs = tf.concat([decoder_embed, tf.reshape(tf.tile(rnn_final_state, [1, steps]), [-1, steps, ENCODER_DIMENSION[-1]])], 2)\n",
    "            \n",
    "            print(decoder_inputs)\n",
    "            \n",
    "            if MODE == \"train\":\n",
    "                helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                    inputs=decoder_inputs,\n",
    "                    sequence_length=decoder_input_lengths)\n",
    "                \n",
    "            elif MODE == \"infer\":\n",
    "                helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                    embedding=embedding,\n",
    "                    start_tokens=tf.tile([GO_SYMBOL], [batch_size]),\n",
    "                    end_token=END_SYMBOL)\n",
    "                \n",
    "           \n",
    "            \n",
    "            decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell=stack,\n",
    "                helper=helper,\n",
    "                initial_state=decoder_init_state,\n",
    "                #output_layer=layers_core.Dense(VOCABULAY_SIZE, use_bias=True, activation=None))\n",
    "                output_layer=None)\n",
    "            #sequence_loss has softmax already\n",
    "\n",
    "            decoder_outputs, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder=decoder,\n",
    "                output_time_major=False,\n",
    "                impute_finished=False,\n",
    "                maximum_iterations=None,\n",
    "                parallel_iterations=32,\n",
    "                swap_memory=False,\n",
    "                scope=None\n",
    "            )\n",
    "            \n",
    "            nce_weights = tf.Variable(\n",
    "            tf.truncated_normal([VOCABULAY_SIZE, RNN_DIMENSION[-1]],\n",
    "                                stddev=1.0 / math.sqrt(RNN_DIMENSION[-1])), trainable=TRAIN_DISEMBEDDING)\n",
    "\n",
    "            nce_biases = tf.Variable(tf.zeros([VOCABULAY_SIZE]), trainable=TRAIN_DISEMBEDDING)\n",
    "        \n",
    "            def sample_softmax_loss_fn(labels, inputs):\n",
    "                return tf.nn.sampled_softmax_loss(\n",
    "                        weights = nce_weights,\n",
    "                        biases = nce_biases,\n",
    "                        labels = tf.reshape(labels, [-1, 1]),\n",
    "                        inputs = inputs,\n",
    "                        num_sampled=SOFTMAX_SAMPLE,\n",
    "                        num_classes=VOCABULAY_SIZE,\n",
    "                        num_true=1,\n",
    "                        sampled_values=None,\n",
    "                        remove_accidental_hits=True,\n",
    "                    )\n",
    "        \n",
    "            def nce_loss_fn(labels, inputs):\n",
    "                nce_loss = tf.nn.nce_loss(\n",
    "                    weights=nce_weights,\n",
    "                    biases=nce_biases,\n",
    "                    labels=tf.reshape(labels, [-1, 1]),\n",
    "                    inputs=inputs,\n",
    "                    num_sampled=NEGATIVE_SAMPLE,\n",
    "                    num_classes=VOCABULAY_SIZE)\n",
    "                return nce_loss\n",
    "        \n",
    "            seq_loss = tf.contrib.seq2seq.sequence_loss(\n",
    "                logits=decoder_outputs.rnn_output,\n",
    "                targets=inputs,\n",
    "                weights=decoder_masks,\n",
    "                average_across_timesteps=True, # make sure you handled decoder_masks and optimizer and average loss\n",
    "                average_across_batch=False,\n",
    "                #softmax_loss_function=nce_loss_fn,\n",
    "                softmax_loss_function=sample_softmax_loss_fn,\n",
    "                name=None\n",
    "            )\n",
    "            \n",
    "            max_loss = tf.reduce_max(seq_loss)\n",
    "            loss = tf.reduce_mean(seq_loss)\n",
    "            \n",
    "            \n",
    "            project_input = tf.placeholder(tf.float32, (None, DIMENSION), name = \"Project_Inputs\")\n",
    "            project = tf.nn.softmax(project_input @ tf.transpose(nce_weights) + nce_biases)\n",
    "            \n",
    "\n",
    "            \"\"\"\n",
    "            helper = tf.contrib.seq2seq.TrainingHelper(decoder_inputs, decoder_input_lengths)\n",
    "            decoder = tf.contrib.seq2seq.BasicDecoder(cell, helper, rnn_final_state)\n",
    "        \n",
    "            decoder_outputs, decoder_states = tf.contrib.seq2seq.dynamic_decode(decoder=decoder)\n",
    "            print(decoder_outputs)\n",
    "            #rnn_outputs, rnn_states = tf.nn.dynamic_rnn(cell, inputs_embed, initial_state=rnn_final_state, sequence_length=input_lengths)\n",
    "            \n",
    "            #weights: A 2D Tensor of shape [batch_size x sequence_length] and dtype float. Weights constitutes the weighting of each prediction in the sequence. When using weights as masking set all valid timesteps to 1 and all padded timesteps to 0.\n",
    "            loss = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(\n",
    "                logits=decoder_outputs,\n",
    "                targets=labels,\n",
    "                weights=decoder_masks,\n",
    "                average_across_timesteps=True,\n",
    "                average_across_batch=True,\n",
    "                softmax_loss_function=None,\n",
    "                name=None\n",
    "            ))\n",
    "            \"\"\"\n",
    "\n",
    "        #optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "        #optimizer = tf.train.MomentumOptimizer(1.0, 0.5).minimize(loss)\n",
    "        optimizer = tf.train.AdamOptimizer().minimize(seq_loss )\n",
    "        \n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()\n",
    "        #embeddings_saver = tf.train.Saver({'Words2Vec': embeddings})\n",
    "        word2VecSaver = tf.train.Saver({'Words2Vec': embeddings, 'NCE_Weights': nce_weights, 'NCE_Biases': nce_biases})\n",
    "        #context = tf.nn.softmax(tf.matmul(rnn_final_state, tf.transpose(nce_weights)) + nce_biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#show_graph(graph.as_graph_def())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "num_steps = 2000000\n",
    "MODEL = './model/seq2seq-autoencoder.ckpt'\n",
    "WORDS2VEC_MODEL = './model/brown-Words2Vec-{}.ckpt'.format(DIMENSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def cloestWord(vec, words_vec, count=10, method='cos'):\n",
    "    if method == 'cos':\n",
    "        dist = np.array([ sum(vec * words_vec[key]) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[::-1][:10]\n",
    "    else:\n",
    "        dist = np.array([ sum(np.square(np.array(vec) - np.array(words_vec[key]))) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[:10]\n",
    "    return [list(words_vec.keys())[i] for i in top_ten]\n",
    "\n",
    "def cloestWord2(word, emb, count=10, method='cos'):\n",
    "    return cloestWord3(emb[words_dict[word]], emb, count, method)\n",
    "\n",
    "def cloestWord3(vec, emb, count=10, method='cos'):\n",
    "    if method == 'cos':\n",
    "        dist = np.array([ sum(vec * emb[i]) for i in range(emb.shape[0])])\n",
    "        # dist: word index -> dist\n",
    "        \n",
    "        top = dist.argsort()[::-1][:count]\n",
    "        # top: ranking -> word index\n",
    "        \n",
    "    return [(inv_words_dict[i], \"%.2f\" % dist[i])  for i in top]\n",
    "\n",
    "def to_word_indices(words):\n",
    "    return [words_dict[word] for word in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/seq2seq-autoencoder.ckpt\n",
      "Restored training...\n",
      "Model saved in file: ./model/seq2seq-autoencoder.ckpt\n"
     ]
    }
   ],
   "source": [
    "DEBUG_SIZE = 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    try:\n",
    "        saver.restore(session, MODEL)\n",
    "        print('Restored training...')\n",
    "    except:\n",
    "        init.run()\n",
    "        try:\n",
    "            word2VecSaver.restore(session, WORDS2VEC_MODEL)\n",
    "        except:\n",
    "            print('Failed to restore words2vec')\n",
    "            \n",
    "        print('Restarting training...')\n",
    "    \n",
    "    run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "    run_metadata = tf.RunMetadata()\n",
    "\n",
    "      \n",
    "    average_loss = 0\n",
    "    average_max_loss = 0\n",
    "    start = time.time()\n",
    "    \n",
    "    for step in range(num_steps):\n",
    "        batch_inputs, batch_input_lengths, batch_masks = next(generator)            \n",
    "        feed_dict = {inputs: batch_inputs, input_lengths: batch_input_lengths, decoder_masks: batch_masks}\n",
    "\n",
    "        #_, loss_val = session.run([optimizer, loss], feed_dict=feed_dict, options=run_options, run_metadata=run_metadata)\n",
    "        _, loss_val, max_loss_val = session.run([optimizer, loss, max_loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "        average_max_loss += max_loss_val\n",
    "        \n",
    "        if step % DEBUG_SIZE == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= DEBUG_SIZE\n",
    "                average_max_loss /= DEBUG_SIZE\n",
    "                print('Total time for {0} steps: {1:.2f}s, each step: {2:.2f}s'.format(DEBUG_SIZE, time.time()-start, (time.time()-start) / DEBUG_SIZE))\n",
    "                print('Average mean loss at step ', step, ': ', average_loss)\n",
    "                print('Average max loss at step ', step, ': ', average_max_loss)\n",
    "                average_loss = 0\n",
    "                average_max_loss = 0\n",
    "                start = time.time()\n",
    "                \n",
    "                feed_dict = {inputs: [\n",
    "                    to_word_indices(['once', 'upon', 'a', 'time', PAD, PAD, PAD]), \n",
    "                    to_word_indices(['this', 'cat', 'is', 'cute', PAD, PAD, PAD]),\n",
    "                    to_word_indices(['the', 'weather', 'today', 'is', 'pretty', 'great', '.']),\n",
    "                ], input_lengths:[4, 4, 7], decoder_masks: [[1,1,1,1,0,0,0], [1,1,1,1,0,0,0], [1,1,1,1,1,1,1]]}\n",
    "\n",
    "                a, b, c, d = session.run([decoder_outputs.rnn_output, seq_loss, loss, max_loss], feed_dict)\n",
    "                print('encoder_input: ', [inv_words_dict[out] for out in feed_dict[inputs][0]])\n",
    "                print('decoder_output: ', [inv_words_dict[word.argmax()] for word in project.eval({\n",
    "                    project_input: a[0]\n",
    "                })])\n",
    "                print('encoder_input: ', [inv_words_dict[out] for out in feed_dict[inputs][1]])\n",
    "                print('decoder_output: ', [inv_words_dict[word.argmax()] for word in project.eval({\n",
    "                    project_input: a[1]\n",
    "                })])\n",
    "                print('encoder_input: ', [inv_words_dict[out] for out in feed_dict[inputs][2]])\n",
    "                print('decoder_output: ', [inv_words_dict[word.argmax()] for word in project.eval({\n",
    "                    project_input: a[2]\n",
    "                })])\n",
    "                print(b)\n",
    "                print(c)\n",
    "                print(d)\n",
    "\n",
    "                \n",
    "        if step % DEBUG_SIZE == 0:\n",
    "            save_path = saver.save(session, MODEL)\n",
    "            print(\"Model saved in file: %s\" % save_path)\n",
    "            \n",
    "            # Create the Timeline object, and write it to a json\n",
    "            tl = timeline.Timeline(run_metadata.step_stats)\n",
    "            ctf = tl.generate_chrome_trace_format()\n",
    "            with open('timeline.json', 'w') as f:\n",
    "                f.write(ctf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph) as session:\n",
    "    saver.restore(session, MODEL)\n",
    "    generator = sentenceGenerator(sents, words_dict, batch_size=2)\n",
    "    batch_inputs, batch_input_lengths, batch_masks = next(generator)\n",
    "    \n",
    "    feed_dict = {\n",
    "        inputs: batch_inputs,\n",
    "        input_lengths: batch_input_lengths,\n",
    "        decoder_masks: batch_masks\n",
    "    }\n",
    "    \n",
    "    feed_dict = {inputs: [\n",
    "        to_word_indices(['once', 'upon', 'a', 'time']), \n",
    "        to_word_indices(['this', 'cat', 'is', 'cute'])\n",
    "    ], input_lengths:[4, 4], decoder_masks: [[1, 1, 1, 1], [1,1,1,1]]}\n",
    "    \n",
    "    a, b, c = session.run([decoder_outputs.rnn_output, seq_loss, loss], feed_dict)\n",
    "    print('encoder_input: ', [inv_words_dict[out] for out in feed_dict[inputs][0]])\n",
    "    print('decoder_output: ', [inv_words_dict[word.argmax()] for word in project.eval({\n",
    "        project_input: a[0]\n",
    "    })])\n",
    "    print('encoder_input: ', [inv_words_dict[out] for out in feed_dict[inputs][1]])\n",
    "    print('decoder_output: ', [inv_words_dict[word.argmax()] for word in project.eval({\n",
    "        project_input: a[1]\n",
    "    })])\n",
    "    print(b)\n",
    "    print(c)\n",
    "    \n",
    "    #print(a[0][0].argsort())\n",
    "    #print('decoder_outputs: ', [inv_words_dict[out] for out in a[0]])\n",
    "    #print(b)\n",
    "    #print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}