5_seq2seq_classification_solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Seq2Seq Classification Tutorial\n",
    "\n",
    "### This tutorial is adapted from the PyTorch tutorial on ['Translation with a Sequence to Sequence Network and Attention'](https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SOS_token = 0\n",
    "EOS_token = 1\n",
    "\n",
    "\n",
    "class Lang:\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        self.word2index = {}\n",
    "        self.word2count = {}\n",
    "        self.index2word = {0: \"SOS\", 1: \"EOS\"}\n",
    "        self.n_words = 2  # Count SOS and EOS\n",
    "\n",
    "    def addSentence(self, sentence):\n",
    "        for word in sentence.split(' '):\n",
    "            self.addWord(word)\n",
    "\n",
    "    def addWord(self, word):\n",
    "        if word not in self.word2index:\n",
    "            self.word2index[word] = self.n_words\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.n_words] = word\n",
    "            self.n_words += 1\n",
    "        else:\n",
    "            self.word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Turn a Unicode string to plain ASCII, thanks to\n",
    "# https://stackoverflow.com/a/518232/2809427\n",
    "def unicodeToAscii(s):\n",
    "    return ''.join(\n",
    "        c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn'\n",
    "    )\n",
    "\n",
    "# Lowercase, trim, and remove non-letter characters\n",
    "\n",
    "\n",
    "def normalizeString(s):\n",
    "    s = unicodeToAscii(s.lower().strip())\n",
    "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
    "    s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def readLines(filename):\n",
    "    lines = open(filename, encoding='utf-8').read().strip().split('\\n')\n",
    "    return [unicodeToAscii(line) for line in lines]\n",
    "\n",
    "def normalizeWindowsString(s):\n",
    "    s = unicodeToAscii(s.lower().strip())\n",
    "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
    "    s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n",
    "    return s\n",
    "\n",
    "def randomTrainingExample(pairs, labels):\n",
    "    rand_key = random.choice(list(pairs))\n",
    "    return pairs[rand_key], labels[rand_key]\n",
    "\n",
    "\n",
    "MAX_RAW_LENGTH = 30\n",
    "MAX_RNN_LENGTH = 10\n",
    "all_categories = []\n",
    "\n",
    "\n",
    "def stripWords2MaxLen(sent):\n",
    "    allwords = sent.split(' ')\n",
    "    if(len(allwords) < MAX_RNN_LENGTH):\n",
    "        return sent\n",
    "    sent = ' '.join(allwords[i] for i in range(MAX_RNN_LENGTH-1))\n",
    "    #print(sent)\n",
    "    return sent\n",
    "\n",
    "def filterSentencePair(sent0, sent1):\n",
    "    return len(sent0.split(' ')) < MAX_RAW_LENGTH and \\\n",
    "        len(sent1.split(' ')) < MAX_RAW_LENGTH \n",
    "\n",
    "def readClassificationData(lang1):\n",
    "    print(\"Reading Lines...\")\n",
    "\n",
    "    # Read the file and split into lines\n",
    "    #with open('data/dev_w_id.txt', 'rb') as f:\n",
    "    #  lines = f.readlines()#.strip().split('\\n')\n",
    "    #print(lines[0])\n",
    "    #lines = open('data/dev_w_id.txt').read().strip().split('\\n')\n",
    "    lines = open('data/train_w_id.txt', encoding='windows-1252').readlines()\n",
    "    print(len(lines))\n",
    "    \n",
    "    input_lang = Lang(lang1)\n",
    "    pairs = {}\n",
    "    labels = {}\n",
    "    # Split every line into id, query, passage, label\n",
    "    for l in lines:\n",
    "        [id, sent0, sent1, label] = l.strip().replace('\\n','').split('\\t')\n",
    "        \n",
    "        if(filterSentencePair(sent0, sent1)):\n",
    "            sent0 = stripWords2MaxLen(normalizeString(sent0))\n",
    "            sent1 = stripWords2MaxLen(normalizeString(sent1))\n",
    "            #print([id, normalizeWindowsString(sent0), normalizeWindowsString(sent1), label])\n",
    "            pairs[id] = ([(sent0), (sent1)])\n",
    "            labels[id] = (label)\n",
    "            if label not in all_categories:\n",
    "                all_categories.append((label))\n",
    "            input_lang.addSentence((sent0))\n",
    "            input_lang.addSentence((sent1))\n",
    "    return input_lang, pairs, labels\n",
    "\n",
    "def prepareClassificationData(lang1):\n",
    "    input_lang, pairs, labels = readClassificationData(lang1)\n",
    "    if( len(pairs) != len(labels)):\n",
    "        print(\"Number of sentence pairs is not equal to label count\")\n",
    "        # exit(0)\n",
    "    print(\"Read %s sentence pairs and labels\" % len(pairs))\n",
    "\n",
    "    print(\"Counted words:\")\n",
    "    print(input_lang.name, input_lang.n_words)\n",
    "    return input_lang, pairs, labels\n",
    "\n",
    "input_lang, pairs, labels = prepareClassificationData('eng')\n",
    "print(randomTrainingExample(pairs, labels))\n",
    "print(randomTrainingExample(pairs, labels))\n",
    "print(all_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        output = embedded\n",
    "        output, hidden = self.gru(output, hidden)\n",
    "        return output, hidden\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderRNN(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size):\n",
    "        super(DecoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        output = self.embedding(input).view(1, 1, -1)\n",
    "        output = F.relu(output)\n",
    "        output, hidden = self.gru(output, hidden)\n",
    "        output = self.softmax(self.out(output[0]))\n",
    "        return output, hidden\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class AttnDecoderRNN(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size, class_size, dropout_p=0.1, max_length=MAX_RNN_LENGTH):\n",
    "        super(AttnDecoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.dropout_p = dropout_p\n",
    "        self.max_length = max_length\n",
    "        self.class_size = class_size\n",
    "\n",
    "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
    "        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n",
    "        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
    "        self.dropout = nn.Dropout(self.dropout_p)\n",
    "        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n",
    "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
    "        self.hidden2class = nn.Linear(self.hidden_size, self.class_size)\n",
    "\n",
    "    def forward(self, input, hidden, encoder_outputs):\n",
    "        #print(input)\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        #print(embedded.size())\n",
    "        embedded = self.dropout(embedded)\n",
    "\n",
    "        attn_weights = F.softmax(\n",
    "            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)\n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
    "                                 encoder_outputs.unsqueeze(0))\n",
    "\n",
    "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
    "        output = self.attn_combine(output).unsqueeze(0)\n",
    "\n",
    "        output = F.relu(output)\n",
    "        output, hidden = self.gru(output, hidden)\n",
    "\n",
    "        output = F.log_softmax(self.out(output[0]), dim=1)\n",
    "        \n",
    "        class_out = F.log_softmax(self.hidden2class(hidden[0]), dim=1)\n",
    "        return output, hidden, attn_weights, class_out\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexesFromSentence(lang, sentence):\n",
    "    return [lang.word2index[word] for word in sentence.split(' ')]\n",
    "\n",
    "\n",
    "def tensorFromSentence(lang, sentence):\n",
    "    indexes = indexesFromSentence(lang, sentence)\n",
    "    indexes.append(EOS_token)\n",
    "    #print(indexes)\n",
    "    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)\n",
    "\n",
    "\n",
    "def tensorsFromPair(pair):\n",
    "    #print(pair)\n",
    "    input_tensor = tensorFromSentence(input_lang, pair[0])\n",
    "    target_tensor = tensorFromSentence(input_lang, pair[1])\n",
    "    return (input_tensor, target_tensor)\n",
    "\n",
    "def tensorFromLabel(category):\n",
    "    category_tensor = torch.tensor([all_categories.index(category)], dtype=torch.long)\n",
    "    return category_tensor\n",
    "\n",
    "def categoryFromOutput(output):\n",
    "    top_n, top_i = output.topk(1)\n",
    "    category_i = top_i[0].item()\n",
    "    return all_categories[category_i], category_i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#teacher_forcing_ratio = 1.0\n",
    "\n",
    "\n",
    "def train(input_tensor, target_tensor, category_tensor, encoder, decoder, encoder_optimizer, \\\n",
    "          decoder_optimizer, criterion, max_length=MAX_RNN_LENGTH):\n",
    "    encoder_hidden = encoder.initHidden()\n",
    "\n",
    "    encoder_optimizer.zero_grad()\n",
    "    decoder_optimizer.zero_grad()\n",
    "\n",
    "    input_length = input_tensor.size(0)\n",
    "    target_length = target_tensor.size(0)\n",
    "\n",
    "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "    loss = 0\n",
    "\n",
    "    for ei in range(input_length):\n",
    "        encoder_output, encoder_hidden = encoder(\n",
    "            input_tensor[ei], encoder_hidden)\n",
    "        encoder_outputs[ei] = encoder_output[0, 0]\n",
    "\n",
    "    decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "\n",
    "    decoder_hidden = encoder_hidden\n",
    "\n",
    "    #use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
    "\n",
    "    # Teacher forcing: Feed the target as the next input\n",
    "    final_decode_class = []\n",
    "    for di in range(target_length):\n",
    "        decoder_output, decoder_hidden, decoder_attention, decoder_class = decoder(\n",
    "            decoder_input, decoder_hidden, encoder_outputs)\n",
    "        #final_decode_class = decoder_class\n",
    "        #loss += criterion(decoder_output, target_tensor[di])\n",
    "        decoder_input = target_tensor[di]  # Teacher forcing\n",
    "    \n",
    "    loss = criterion(decoder_class, category_tensor.cuda())\n",
    "    loss.backward()\n",
    "\n",
    "    encoder_optimizer.step()\n",
    "    decoder_optimizer.step()\n",
    "\n",
    "    return loss.item()# / target_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import math\n",
    "\n",
    "\n",
    "def asMinutes(s):\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)\n",
    "\n",
    "\n",
    "def timeSince(since, percent):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    es = s / (percent)\n",
    "    rs = es - s\n",
    "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.switch_backend('agg')\n",
    "import matplotlib.ticker as ticker\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def showPlot(points):\n",
    "    plt.figure()\n",
    "    fig, ax = plt.subplots()\n",
    "    # this locator puts ticks at regular intervals\n",
    "    loc = ticker.MultipleLocator(base=0.2)\n",
    "    ax.yaxis.set_major_locator(loc)\n",
    "    plt.plot(points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainIters(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):\n",
    "    start = time.time()\n",
    "    plot_losses = []\n",
    "    print_loss_total = 0  # Reset every print_every\n",
    "    plot_loss_total = 0  # Reset every plot_every\n",
    "\n",
    "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
    "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
    "    #[training_pairs,training_labels] = [randomTrainingExample(pairs, labels)\n",
    "    #                  for i in range(n_iters)]\n",
    "    training_pairs = []\n",
    "    training_labels = []\n",
    "    for i in range(n_iters):\n",
    "        training_pair, training_label = randomTrainingExample(pairs, labels)\n",
    "        training_pairs.append(training_pair)\n",
    "        training_labels.append(training_label)\n",
    "    \n",
    "    criterion = nn.NLLLoss()\n",
    "\n",
    "    for iter in range(1, n_iters + 1):\n",
    "        training_pair = training_pairs[iter - 1]\n",
    "        input_tensor, target_tensor = tensorsFromPair(training_pair)\n",
    "        category_tensor = tensorFromLabel(training_labels[iter-1]).cuda()\n",
    "        \n",
    "        #print(input_tensor.size())\n",
    "        #print(target_tensor.size())\n",
    "        #print(category_tensor.size())\n",
    "        #print()\n",
    "\n",
    "        loss = train(input_tensor, target_tensor, category_tensor, encoder,\n",
    "                     decoder, encoder_optimizer, decoder_optimizer, criterion)\n",
    "        print_loss_total += loss\n",
    "        plot_loss_total += loss\n",
    "\n",
    "        if iter % print_every == 0:\n",
    "            print_loss_avg = print_loss_total / print_every\n",
    "            print_loss_total = 0\n",
    "            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),\n",
    "                                         iter, iter / n_iters * 100, print_loss_avg))\n",
    "\n",
    "        if iter % plot_every == 0:\n",
    "            plot_loss_avg = plot_loss_total / plot_every\n",
    "            plot_losses.append(plot_loss_avg)\n",
    "            plot_loss_total = 0\n",
    "\n",
    "    showPlot(plot_losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training and Evaluating\n",
    "======================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = 256\n",
    "encoder1 = EncoderRNN(input_lang.n_words, hidden_size).cuda()\n",
    "attn_decoder1 = AttnDecoderRNN(hidden_size, input_lang.n_words, len(all_categories), dropout_p=0.1).cuda()\n",
    "\n",
    "trainIters(encoder1, attn_decoder1.cuda(), 75000, print_every=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(input_tensor, target_tensor, encoder, decoder, max_length=MAX_RNN_LENGTH):\n",
    "    encoder_hidden = encoder.initHidden()\n",
    "\n",
    "    input_length = input_tensor.size(0)\n",
    "    target_length = target_tensor.size(0)\n",
    "\n",
    "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "    for ei in range(input_length):\n",
    "        encoder_output, encoder_hidden = encoder(\n",
    "            input_tensor[ei], encoder_hidden)\n",
    "        encoder_outputs[ei] = encoder_output[0, 0]\n",
    "\n",
    "    decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "\n",
    "    decoder_hidden = encoder_hidden\n",
    "\n",
    "    #final_decode_class = []\n",
    "    for di in range(target_length):\n",
    "        decoder_output, decoder_hidden, decoder_attention, decoder_class = decoder(\n",
    "            decoder_input, decoder_hidden, encoder_outputs)\n",
    "        #final_decode_class = decoder_class\n",
    "        #loss += criterion(decoder_output, target_tensor[di])\n",
    "        decoder_input = target_tensor[di]  # Teacher forcing\n",
    "    \n",
    "    #loss = criterion(decoder_class, category_tensor)\n",
    "    #loss.backward()\n",
    "\n",
    "    #encoder_optimizer.step()\n",
    "    #decoder_optimizer.step()\n",
    "\n",
    "    return decoder_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateRandomly(encoder, decoder, n=10):\n",
    "    for i in range(n):\n",
    "        training_pair, training_label = randomTrainingExample(pairs, labels)\n",
    "        input_tensor, target_tensor = tensorsFromPair(training_pair)\n",
    "        decoded_class = evaluate(input_tensor, target_tensor, encoder, decoder)\n",
    "        guess, guess_i = categoryFromOutput(decoded_class)\n",
    "        \n",
    "        print(training_label, guess)\n",
    "        print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluateRandomly(encoder1, attn_decoder1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}