15_transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "15_transformer",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPOoTMyxU0/FaNzYOPkBjFn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lunaB/Pytorch-Study/blob/master/15_transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s-_rjfkVRCmP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2c839d0c-8d90-47af-fcde-8859557ecb77"
      },
      "source": [
        "import sys\n",
        "import random\n",
        "import datetime as dt\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "torch.manual_seed(0)\n",
        "torch.cuda.manual_seed(0)\n",
        "np.random.seed(0)\n",
        "random.seed(0)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "# Batch Size를 작게 설정하는 경우 GPU가 CPU보다 학습 시간이 느린것을 확인. (이 상황에선 cpu가 더 빠름)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cpu')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 334
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0AqsQ2aJoRCh"
      },
      "source": [
        "raw = [\"안녕 난 영채야\\thi my name is youngchae\",\n",
        "       \"안녕 넌 참 신기하다\\thi you are so amazing\",\n",
        "       \"와 정말 신기하다\\twow so amazing\",\n",
        "       \"영채야 이거 봐봐\\tlook at this youngchae\"]\n",
        "\n",
        "SOS_token = 0\n",
        "EOS_token = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2orRw-Yfr8J2"
      },
      "source": [
        "class Vocab:\n",
        "    def __init__(self):\n",
        "        self.vocab2index = {\"<SOS>\": SOS_token, \"<EOS>\": EOS_token}\n",
        "        self.index2vocab = {SOS_token: \"<SOS>\", EOS_token: \"<EOS>\"}\n",
        "        self.vocab_count = {}\n",
        "        self.n_vocab = len(self.vocab2index)\n",
        "\n",
        "    def add_vocab(self, sentence):\n",
        "        for word in sentence.split(\" \"):\n",
        "            if word not in self.vocab2index:\n",
        "                self.vocab2index[word] = self.n_vocab\n",
        "                self.vocab_count[word] = 1\n",
        "                self.index2vocab[self.n_vocab] = word\n",
        "                self.n_vocab += 1\n",
        "            else:\n",
        "                self.vocab_count[word] += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sr8nPfwVsDmU"
      },
      "source": [
        "# 하나라도 max length를 초과하면 false 리턴 preprocess에서 초과하는 데이터를 pair셋에 포함시키지 않기위해 사용\n",
        "def filter_pair(pair, source_max_length, target_max_length):\n",
        "    return len(pair[0].split(\" \")) < source_max_length and len(pair[1].split(\" \")) < target_max_length"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-64Ax0PaqpvB"
      },
      "source": [
        "def preprocess(corpus, source_max_length, target_max_length):\n",
        "    print(\"reading corpus...\")\n",
        "    pairs = []\n",
        "    for line in corpus:\n",
        "        pairs.append([s for s in line.strip().lower().split(\"\\t\")])\n",
        "    print(\"Read {} sentence pairs\".format(len(pairs)))\n",
        "\n",
        "    pairs = [pair for pair in pairs if filter_pair(pair, source_max_length, target_max_length)]\n",
        "    print(\"Trimmed to {} sentence pairs\".format(len(pairs)))\n",
        "\n",
        "    source_vocab = Vocab()\n",
        "    target_vocab = Vocab()\n",
        "\n",
        "    print(\"Counting words...\")\n",
        "    for pair in pairs:\n",
        "        source_vocab.add_vocab(pair[0])\n",
        "        target_vocab.add_vocab(pair[1])\n",
        "    print(\"source vocab size =\", source_vocab.n_vocab)\n",
        "    print(\"target vocab size =\", target_vocab.n_vocab)\n",
        "\n",
        "    return pairs, source_vocab, target_vocab"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6EB8lqEvsLgJ"
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
        "\n",
        "    def forward(self, x, hidden):\n",
        "        x = self.embedding(x).view(1, 1, -1)\n",
        "        x, hidden = self.gru(x, hidden)\n",
        "        return x, hidden"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9TegstZ6sNfD"
      },
      "source": [
        "# attention 부분을 직접 짜서 조금 지저분함\n",
        "# dot product attention으로 만듬\n",
        "class Decoder(nn.Module):\n",
        "    def __init__(self, hidden_size, output_size):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
        "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
        "        self.out = nn.Linear(hidden_size * 2, output_size)\n",
        "        self.softmax = nn.Softmax(dim=0)\n",
        "        self.logsoftmax = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    def forward(self, x, hidden, encoder_hiddens):\n",
        "        x = self.embedding(x).view(1, 1, -1)\n",
        "        x, hidden = self.gru(x, hidden)\n",
        "\n",
        "        att = torch.Tensor([])\n",
        "        for i in range(len(encoder_hiddens)):\n",
        "            # print(encoder_hiddens[i])\n",
        "            # print(x[0].T)\n",
        "            att = torch.cat((att, encoder_hiddens[i].matmul(x[0].T)))\n",
        "\n",
        "        att = self.softmax(att)\n",
        "        \n",
        "        # print(att)\n",
        "        # print(encoder_hiddens)\n",
        "\n",
        "        att = encoder_hiddens * att.view(-1,1,1)\n",
        "        # print(att)\n",
        "\n",
        "        att_val = att.sum(0)\n",
        "        # print(att_val)\n",
        "        # print(x)\n",
        "        \n",
        "        x = torch.cat((att_val, x[0]), dim=1)\n",
        "        # print(x)\n",
        "\n",
        "        x = self.out(x)\n",
        "        # print(x)\n",
        "        x = self.logsoftmax(x)\n",
        "\n",
        "        return x, hidden"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yI2tFx5dsNu9"
      },
      "source": [
        "def tensorize(vocab, sentence):\n",
        "    indexes = [vocab.vocab2index[word] for word in sentence.split(\" \")]\n",
        "    indexes.append(vocab.vocab2index[\"<EOS>\"])\n",
        "    return torch.Tensor(indexes).long().to(device).view(-1, 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08OqIuMLsPs7"
      },
      "source": [
        "def train(pairs, source_vocab, target_vocab, encoder, decoder, n_iter, print_every=1000, learning_rate=0.01):\n",
        "    loss_total = 0\n",
        "\n",
        "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
        "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
        "\n",
        "    training_batch = [random.choice(pairs) for _ in range(n_iter)]\n",
        "    training_source = [tensorize(source_vocab, pair[0]) for pair in training_batch]\n",
        "    training_target = [tensorize(target_vocab, pair[1]) for pair in training_batch]\n",
        "\n",
        "    # softmax 한것을 nllloss로계산하면 crossentropy\n",
        "    criterion = nn.NLLLoss()\n",
        "\n",
        "    for i in range(1, n_iter + 1):\n",
        "        source_tensor = training_source[i - 1]\n",
        "        target_tensor = training_target[i - 1]\n",
        "\n",
        "        encoder_hidden = torch.zeros([1, 1, encoder.hidden_size]).to(device)\n",
        "\n",
        "        encoder_optimizer.zero_grad()\n",
        "        decoder_optimizer.zero_grad()\n",
        "\n",
        "        source_length = source_tensor.size(0)\n",
        "        target_length = target_tensor.size(0)\n",
        "\n",
        "        loss = 0\n",
        "\n",
        "        encoder_outputs = torch.Tensor([])\n",
        "\n",
        "        for enc_input in range(source_length):\n",
        "            encoder_output, encoder_hidden = encoder(source_tensor[enc_input], encoder_hidden)\n",
        "            encoder_outputs = torch.cat((encoder_outputs, encoder_output))\n",
        "            # encoder_outputs.append(encoder_output)\n",
        "\n",
        "        # print(encoder_outputs)\n",
        "\n",
        "        decoder_input = torch.Tensor([[SOS_token]]).long().to(device)\n",
        "        decoder_hidden = encoder_hidden # connect encoder output to decoder input\n",
        "\n",
        "        for di in range(target_length):\n",
        "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
        "\n",
        "            # print(decoder_output)\n",
        "            # print(target_tensor[di])\n",
        "\n",
        "            loss += criterion(decoder_output, target_tensor[di])\n",
        "            decoder_input = target_tensor[di]  # teacher forcing\n",
        "\n",
        "        loss.backward()\n",
        "\n",
        "        encoder_optimizer.step()\n",
        "        decoder_optimizer.step()\n",
        "\n",
        "        loss_iter = loss.item() / target_length\n",
        "        loss_total += loss_iter\n",
        "\n",
        "        if i % print_every == 0:\n",
        "            loss_avg = loss_total / print_every\n",
        "            loss_total = 0\n",
        "            print(\"[{} - {}%] loss = {:05.4f}\".format(i, i / n_iter * 100, loss_avg))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8EPJsqo6sYjZ"
      },
      "source": [
        "def evaluate(pairs, source_vocab, target_vocab, encoder, decoder, target_max_length):\n",
        "    for pair in pairs:\n",
        "        print(\">\", pair[0])\n",
        "        print(\"=\", pair[1])\n",
        "        source_tensor = tensorize(source_vocab, pair[0])\n",
        "        source_length = source_tensor.size()[0]\n",
        "        encoder_hidden = torch.zeros([1, 1, encoder.hidden_size]).to(device)\n",
        "\n",
        "        \n",
        "        encoder_outputs = torch.Tensor([])\n",
        "        for ei in range(source_length):\n",
        "            encoder_output, encoder_hidden = encoder(source_tensor[ei], encoder_hidden)\n",
        "            encoder_outputs = torch.cat((encoder_outputs, encoder_output))\n",
        "\n",
        "        decoder_input = torch.Tensor([[SOS_token]], device=device).long()\n",
        "        decoder_hidden = encoder_hidden\n",
        "        decoded_words = []\n",
        "\n",
        "        for di in range(target_max_length):\n",
        "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
        "            _, top_index = decoder_output.data.topk(1)\n",
        "            if top_index.item() == EOS_token:\n",
        "                decoded_words.append(\"<EOS>\")\n",
        "                break\n",
        "            else:\n",
        "                decoded_words.append(target_vocab.index2vocab[top_index.item()])\n",
        "\n",
        "            decoder_input = top_index.squeeze().detach()\n",
        "\n",
        "        predict_words = decoded_words\n",
        "        predict_sentence = \" \".join(predict_words)\n",
        "        print(\"<\", predict_sentence)\n",
        "        print(\"\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovq136DLsaxA"
      },
      "source": [
        "SOURCE_MAX_LENGTH = 10\n",
        "TARGET_MAX_LENGTH = 10"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aIcgapjtscdY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "7b7b4d7c-74ec-4a0a-9361-cea6f1c655a4"
      },
      "source": [
        "load_pairs, load_source_vocab, load_target_vocab = preprocess(raw, SOURCE_MAX_LENGTH, TARGET_MAX_LENGTH)\n",
        "print(random.choice(load_pairs))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "reading corpus...\n",
            "Read 4 sentence pairs\n",
            "Trimmed to 4 sentence pairs\n",
            "Counting words...\n",
            "source vocab size = 12\n",
            "target vocab size = 15\n",
            "['영채야 이거 봐봐', 'look at this youngchae']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Vu1EtWAsefV"
      },
      "source": [
        "enc_hidden_size = 16\n",
        "dec_hidden_size = enc_hidden_size\n",
        "enc = Encoder(load_source_vocab.n_vocab, enc_hidden_size).to(device)\n",
        "dec = Decoder(dec_hidden_size, load_target_vocab.n_vocab).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4YYzuNgsg4P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "07aaaa7a-cb3d-4a7b-fac4-d58e13a81603"
      },
      "source": [
        "train(load_pairs, load_source_vocab, load_target_vocab, enc, dec, 5000, print_every=1000)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1000 - 20.0%] loss = 0.6425\n",
            "[2000 - 40.0%] loss = 0.0351\n",
            "[3000 - 60.0%] loss = 0.0147\n",
            "[4000 - 80.0%] loss = 0.0090\n",
            "[5000 - 100.0%] loss = 0.0065\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WCiVXjAztPpf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "outputId": "ab3d0846-736e-4591-8b0b-fa09f8a76269"
      },
      "source": [
        "evaluate(load_pairs, load_source_vocab, load_target_vocab, enc, dec, TARGET_MAX_LENGTH)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "> 안녕 난 영채야\n",
            "= hi my name is youngchae\n",
            "< hi my name is youngchae <EOS>\n",
            "\n",
            "> 안녕 넌 참 신기하다\n",
            "= hi you are so amazing\n",
            "< hi you are so amazing <EOS>\n",
            "\n",
            "> 와 정말 신기하다\n",
            "= wow so amazing\n",
            "< wow so amazing <EOS>\n",
            "\n",
            "> 영채야 이거 봐봐\n",
            "= look at this youngchae\n",
            "< look at this youngchae <EOS>\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X3-ozMR-72Lg"
      },
      "source": [
        "# load_pairs, load_source_vocab, load_target_vocab = preprocess(raw, SOURCE_MAX_LENGTH, TARGET_MAX_LENGTH)\n",
        "# print(random.choice(load_pairs))\n",
        "\n",
        "# enc_hidden_size = 16\n",
        "# dec_hidden_size = enc_hidden_size\n",
        "# enc = Encoder(load_source_vocab.n_vocab, enc_hidden_size).to(device)\n",
        "# dec = Decoder(dec_hidden_size, load_target_vocab.n_vocab).to(device)\n",
        "\n",
        "# train(load_pairs, load_source_vocab, load_target_vocab, enc, dec, 1, print_every=1000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q0eWNPC3Jz2C",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "d8a84ba2-4697-40a8-9e31-1abac8088d7d"
      },
      "source": [
        "# test\n",
        "a = torch.Tensor([[[-0.2807, -0.0313, -0.2060, -0.5300, -0.0359, -0.0324,  0.1610,\n",
        "           0.0359,  0.1713, -0.0643,  0.1804,  0.1991, -0.0182,  0.0163,\n",
        "           0.1620, -0.0905]],\n",
        "\n",
        "        [[ 0.3218,  0.2815, -0.3996, -0.0347, -0.3250, -0.5432, -0.0847,\n",
        "           0.2177, -0.2206, -0.4593,  0.0988,  0.0595,  0.1314, -0.2499,\n",
        "          -0.0862, -0.2302]],\n",
        "\n",
        "        [[ 0.3090,  0.2358, -0.3276, -0.0723, -0.0751,  0.0968, -0.3588,\n",
        "           0.1281, -0.1979, -0.3368,  0.0836,  0.3466,  0.0074, -0.4496,\n",
        "           0.1435, -0.3111]],\n",
        "\n",
        "        [[ 0.3193,  0.3354,  0.2523,  0.0485, -0.2001, -0.5193, -0.2616,\n",
        "          -0.4318, -0.0128,  0.1569,  0.2528, -0.0342,  0.2377, -0.4790,\n",
        "          -0.3276, -0.0365]]])\n",
        "b = torch.Tensor([[[1]],[[2]],[[3]],[[4]]])\n",
        "a*b"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[-0.2807, -0.0313, -0.2060, -0.5300, -0.0359, -0.0324,  0.1610,\n",
              "           0.0359,  0.1713, -0.0643,  0.1804,  0.1991, -0.0182,  0.0163,\n",
              "           0.1620, -0.0905]],\n",
              "\n",
              "        [[ 0.6436,  0.5630, -0.7992, -0.0694, -0.6500, -1.0864, -0.1694,\n",
              "           0.4354, -0.4412, -0.9186,  0.1976,  0.1190,  0.2628, -0.4998,\n",
              "          -0.1724, -0.4604]],\n",
              "\n",
              "        [[ 0.9270,  0.7074, -0.9828, -0.2169, -0.2253,  0.2904, -1.0764,\n",
              "           0.3843, -0.5937, -1.0104,  0.2508,  1.0398,  0.0222, -1.3488,\n",
              "           0.4305, -0.9333]],\n",
              "\n",
              "        [[ 1.2772,  1.3416,  1.0092,  0.1940, -0.8004, -2.0772, -1.0464,\n",
              "          -1.7272, -0.0512,  0.6276,  1.0112, -0.1368,  0.9508, -1.9160,\n",
              "          -1.3104, -0.1460]]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 290
        }
      ]
    }
  ]
}