2 - Evaluate Seq2seq with Attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from tokenizers import Tokenizer\n",
    "\n",
    "import math\n",
    "import random\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "from seq2seq import generate_data_loader, Seq2seqConfig, Arguments, Seq2seq, train, evaluate, test, plot_loss, save_checkpoint, load_checkpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 - Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "SEED = 2021\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else : \n",
    "    device = torch.device('cpu')\n",
    "\n",
    "tokenizer = Tokenizer.from_file(\"./data/preprocess/vocab.json\")\n",
    "config = Seq2seqConfig('./data/preprocess/vocab.json', device, hidden_size=100, embedding_size=620, maxout_hidden_size = 50)\n",
    "args = Arguments(batch_size = 4, lr = 1e-05, clip = 1.0, epochs = 4, beam_size = 3)\n",
    "seq2seq = Seq2seq(config, device).to(device)\n",
    "\n",
    "if os.path.isfile(args.checkpoint_file) :\n",
    "    print('Loading existing checkpoint... \\n')\n",
    "    args, seq2seq = load_checkpoint(args.checkpoint_file, seq2seq)\n",
    "\n",
    "train_dataset, train_data_loader, dev_dataset, dev_data_loader, test_dataset, test_data_loader = generate_data_loader(args.batch_size, './data/preprocess/train.pkl', './data/preprocess/dev.pkl', './data/preprocess/test.pkl', tokenizer)\n",
    "loss_fn = nn.CrossEntropyLoss(ignore_index = config.padding_idx)\n",
    "optimizer = torch.optim.Adam(seq2seq.parameters(), lr = args.lr)\n",
    "\n",
    "# base_lr, max_lr, train_dataset_len = 0.0001, 0.001, train_dataset.__len__()\n",
    "# step_size = math.ceil(train_dataset_len / batch_size) * 2\n",
    "# scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=base_lr, max_lr = max_lr, step_size_up=step_size, mode='triangular', cycle_momentum= False)\n",
    "\n",
    "\n",
    "for epoch in range(args.state['state_epoch'], args.epochs):\n",
    "\n",
    "    print(f\"Epoch {epoch+1}\\n-------------------------------\")\n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss = train(seq2seq, device, train_data_loader, optimizer, config, loss_fn, args.clip)\n",
    "    dev_loss, dev_rouge = evaluate(seq2seq, device, tokenizer, dev_data_loader, config, loss_fn, args.beam_size, args.batch_size)\n",
    "    args.update_state(train_loss, dev_loss, dev_rouge, epoch+1)\n",
    "    save_checkpoint(args, seq2seq)\n",
    "    end_time = time.time()\n",
    "    \n",
    "    print(f'Epoch took : {end_time-start_time}')\n",
    "        \n",
    "plot_loss(args.state['train_loss_set'], args.state['dev_loss_set'], args.state['dev_rouge_set'], args.state['state_epoch'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2- Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test(seq2seq, device, args, tokenizer, test_data_loader, config, args.beam_size, args.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}