HW1P2_Starter_Student.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Kelton8Z/speech_frame_classifier/blob/main/HW1P2_Starter_Student.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F9ERgBpbcMmB"
      },
      "source": [
        "# HW1: Frame-Level Speech Recognition"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CLkH6GMGcWcE"
      },
      "source": [
        "In this homework, you will be working with MFCC data consisting of 15 features at each time step/frame. Your model should be able to recognize the phoneme occured in that frame."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z4vZbDmJvMp1"
      },
      "source": [
        "# Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "rwYu9sSUnSho",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a1f40cbc-b50e-4d46-cdb5-429af8069712"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.8 MB 32.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 181 kB 26.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 158 kB 54.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 63 kB 1.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 157 kB 47.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 157 kB 61.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 157 kB 48.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 157 kB 56.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 157 kB 53.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 157 kB 6.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 157 kB 77.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 156 kB 57.6 MB/s \n",
            "\u001b[?25h  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "!pip install torchsummaryX wandb --quiet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qI4qfx7tiBZt",
        "outputId": "f1a2d1f3-bd25-4b56-b105-ac8608f7ca72"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Device:  cuda\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "from torchsummaryX import summary\n",
        "import sklearn\n",
        "import sklearn.metrics\n",
        "import gc\n",
        "import zipfile\n",
        "import pandas as pd\n",
        "from tqdm.auto import tqdm\n",
        "import os\n",
        "import datetime\n",
        "import wandb\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "print(\"Device: \", device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "8yBgXjKV1O0Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a7804948-498f-4a66-c84a-75061ba90b06"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "### If you are using colab, you can import google drive to save model checkpoints in a folder\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "N-9qE20hmCgQ"
      },
      "outputs": [],
      "source": [
        "### PHONEME LIST\n",
        "PHONEMES = [\n",
        "            'SIL',   'AA',    'AE',    'AH',    'AO',    'AW',    'AY',  \n",
        "            'B',     'CH',    'D',     'DH',    'EH',    'ER',    'EY',\n",
        "            'F',     'G',     'HH',    'IH',    'IY',    'JH',    'K',\n",
        "            'L',     'M',     'N',     'NG',    'OW',    'OY',    'P',\n",
        "            'R',     'S',     'SH',    'T',     'TH',    'UH',    'UW',\n",
        "            'V',     'W',     'Y',     'Z',     'ZH',    '<sos>', '<eos>']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZIi0Big7vPa9"
      },
      "source": [
        "# Kaggle"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBCbeRhixGM7"
      },
      "source": [
        "This section contains code that helps you install kaggle's API, creating kaggle.json with you username and API key details. Make sure to input those in the given code to ensure you can download data from the competition successfully."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "TPBUd7Cnl-Rx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "495ac7d0-05d6-428f-9020-e1de1381ea48"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting kaggle==1.5.8\n",
            "  Downloading kaggle-1.5.8.tar.gz (59 kB)\n",
            "\u001b[K     |████████████████████████████████| 59 kB 6.5 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: kaggle\n",
            "  Building wheel for kaggle (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kaggle: filename=kaggle-1.5.8-py3-none-any.whl size=73275 sha256=fcd485893d836785a33d610e9be08b84dcc81e53fbf56852f98981b9af7c7778\n",
            "  Stored in directory: /root/.cache/pip/wheels/de/f7/d8/c3902cacb7e62cb611b1ad343d7cc07f42f7eb76ae3a52f3d1\n",
            "Successfully built kaggle\n",
            "Installing collected packages: kaggle\n",
            "  Attempting uninstall: kaggle\n",
            "    Found existing installation: kaggle 1.5.12\n",
            "    Uninstalling kaggle-1.5.12:\n",
            "      Successfully uninstalled kaggle-1.5.12\n",
            "Successfully installed kaggle-1.5.8\n"
          ]
        }
      ],
      "source": [
        "!pip install --upgrade --force-reinstall --no-deps kaggle==1.5.8\n",
        "!mkdir /root/.kaggle\n",
        "\n",
        "with open(\"/root/.kaggle/kaggle.json\", \"w+\") as f:\n",
        "    f.write('{\"username\":\"keltonzhang\",\"key\":\"0f5b3ef7c93101b2a2b00d4445a4cdec\"}') \n",
        "    # Put your kaggle username & key here\n",
        "\n",
        "!chmod 600 /root/.kaggle/kaggle.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "if2Somqfbje1",
        "outputId": "1d6c7d1c-f554-4812-83b1-cfee2b5cea99"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading 11-785-f22-hw1p2.zip to /content\n",
            "100% 2.12G/2.13G [00:12<00:00, 112MB/s]\n",
            "100% 2.13G/2.13G [00:12<00:00, 188MB/s]\n"
          ]
        }
      ],
      "source": [
        "# commands to download data from kaggle\n",
        "\n",
        "!kaggle competitions download -c 11-785-f22-hw1p2\n",
        "!mkdir '/content/data'\n",
        "\n",
        "!unzip -qo '11-785-f22-hw1p2.zip' -d '/content/data'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vuzce0_TdcaR"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2_7QgMbBdgPp"
      },
      "source": [
        "This section covers the dataset/dataloader class for speech data. You will have to spend time writing code to create this class successfully. We have given you a lot of comments guiding you on what code to write at each stage, from top to bottom of the class. Please try and take your time figuring this out, as it will immensely help in creating dataset/dataloader classes for future homeworks.\n",
        "\n",
        "Before running the following cells, please take some time to analyse the structure of data. Try loading a single MFCC and its transcipt, print out the shapes and print out the values. Do the transcripts look like phonemes?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "YpLCvi3AJC5z"
      },
      "outputs": [],
      "source": [
        "# Dataset class to load train and validation data\n",
        "\n",
        "import enum\n",
        "\n",
        "\n",
        "class AudioDataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, data_path, context, offset=0, partition= \"train\", limit=-1): # Feel free to add more arguments\n",
        "\n",
        "        self.context = context\n",
        "        self.offset = offset\n",
        "        self.data_path = data_path\n",
        "\n",
        "        self.mfcc_dir = data_path + \"/mfcc/\" #\"train-clean-100/mfcc\" if partition == \"train\" else \"dev-clean/mfcc\" # TODO: MFCC directory - use partition to acces train/dev directories from kaggle data\n",
        "        self.transcript_dir = data_path + \"/transcript/\" #\"train-clean/transcript\" if partition == \"train\" else \"dev-clean/transcript\" # TODO: Transcripts directory - use partition to acces train/dev directories from kaggle data\n",
        "\n",
        "        mfcc_names = [self.mfcc_dir+f for f in sorted(os.listdir(self.mfcc_dir))] # TODO: List files in sefl.mfcc_dir_dir using os.listdir in sorted order, optionally subset using limit to slice the number of files you load\n",
        "        transcript_names = [self.transcript_dir+f for f in sorted(os.listdir(self.transcript_dir))] # TODO: List files in self.transcript_dir using os.listdir in sorted order, optionally subset using limit to slice the number of files you load\n",
        "\n",
        "        assert len(mfcc_names) == len(transcript_names) # Making sure that we have the same no. of mfcc and transcripts\n",
        "\n",
        "        self.mfccs, self.transcripts = [], []\n",
        "\n",
        "        # TODO:\n",
        "        # Iterate through mfccs and transcripts\n",
        "        for i in range(0, len(mfcc_names)):\n",
        "        #   Load a single mfcc\n",
        "            mfcc = np.load(mfcc_names[i])\n",
        "        #   Optionally do Cepstral Normalization of mfcc\n",
        "        #   Load the corresponding transcript\n",
        "            transcript = np.load(transcript_names[i])[1:-1] # Remove [SOS] and [EOS] from the transcript (Is there an efficient way to do this \n",
        "            # without traversing through the transcript?)\n",
        "        #   Append each mfcc to self.mfcc, transcript to self.transcript\n",
        "            self.mfccs.append(mfcc)\n",
        "            self.transcripts.append(transcript)\n",
        "\n",
        "        \n",
        "\n",
        "        # NOTE:\n",
        "        # Each mfcc is of shape T1 x 15, T2 x 15, ...\n",
        "        # Each transcript is of shape (T1+2) x 15, (T2+2) x 15 before removing [SOS] and [EOS]\n",
        "\n",
        "        # TODO: Concatenate all mfccs in self.mfccs such that the final shape is T x 15 (Where T = T1 + T2 + ...) \n",
        "        self.mfccs = np.concatenate(self.mfccs)\n",
        "\n",
        "        # TODO: Concatenate all transcripts in self.transcripts such that the final shape is (T,) meaning, each time step has one phoneme output\n",
        "        self.transcripts = np.concatenate(self.transcripts, axis=None)\n",
        "        # Hint: Use numpy to concatenate\n",
        "\n",
        "        # Take some time to think about what we have done. self.mfcc is an array of the format (Frames x Features). Our goal is to recognize phonemes of each frame\n",
        "        # From hw0, you will be knowing what context is. We can introduce context by padding zeros on top and bottom of self.mfcc\n",
        "        # padding_width = [(context, context), (0, 0)]\n",
        "        # for i in range(len(self.mfccs)):\n",
        "        #     self.mfccs[i] = np.pad(self.mfccs[i], padding_width, mode='constant', constant_values=0) # TODO\n",
        "\n",
        "        # These are the available phonemes in the transcript\n",
        "        self.phonemes = [\n",
        "            'SIL',   'AA',    'AE',    'AH',    'AO',    'AW',    'AY',  \n",
        "            'B',     'CH',    'D',     'DH',    'EH',    'ER',    'EY',\n",
        "            'F',     'G',     'HH',    'IH',    'IY',    'JH',    'K',\n",
        "            'L',     'M',     'N',     'NG',    'OW',    'OY',    'P',\n",
        "            'R',     'S',     'SH',    'T',     'TH',    'UH',    'UW',\n",
        "            'V',     'W',     'Y',     'Z',     'ZH',    '<sos>', '<eos>']\n",
        "        # But the neural network cannot predict strings as such. Instead we map these phonemes to integers\n",
        "        d = {phone:idx for idx, phone in enumerate(self.phonemes[:-2])}\n",
        "        self.transcripts = [d[phone] for phone in self.transcripts] # TODO: Map the phonemes to their corresponding list indexes in self.phonemes\n",
        "        # Now, if an element in self.transcript is 0, it means that it is 'SIL' (as per the above example)\n",
        "\n",
        "        # Length of the dataset is now the length of concatenated mfccs/transcripts\n",
        "        self.length = len(self.transcripts) #len(self.mfccs) # without the padding\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.length\n",
        "\n",
        "    def __getitem__(self, ind):\n",
        "        ### Calculate starting timestep using offset and context (1 line)\n",
        "        start_ind = ind + self.offset - self.context\n",
        "        \n",
        "        ### Calculate ending timestep using offset and context (1 line)\n",
        "        end_ind = ind + self.offset + self.context + 1\n",
        "        \n",
        "        frames = self.mfccs[start_ind:end_ind] # TODO: Based on context and offset, return a frame at given index with context frames to the left, and right.\n",
        "        # After slicing, you get an array of shape 2*context+1 x 15. But our MLP needs 1d data and not 2d.\n",
        "        frames = frames.flatten() # TODO: Flatten to get 1d data\n",
        "\n",
        "        frames = torch.FloatTensor(frames) # Convert to tensors\n",
        "        phoneme = torch.tensor(self.transcripts[ind])       \n",
        "\n",
        "        return frames, phoneme"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "e8KfVP39S6o7"
      },
      "outputs": [],
      "source": [
        "class AudioTestDataset(torch.utils.data.Dataset):\n",
        "\n",
        "    # TODO: Create a test dataset class similar to the previous class but you dont have transcripts for this\n",
        "    # Imp: Read the mfccs in sorted order, do NOT shuffle the data here or in your dataloader.\n",
        "    # Dataset class to load train and validation data\n",
        "    def __init__(self, data_path, context, offset=0, limit=-1): # Feel free to add more arguments\n",
        "\n",
        "        self.context = context\n",
        "        self.offset = offset\n",
        "        self.data_path = data_path\n",
        "\n",
        "        self.mfcc_dir = data_path + \"/mfcc/\" #\"train-clean-100/mfcc\" if partition == \"train\" else \"dev-clean/mfcc\" # TODO: MFCC directory - use partition to acces train/dev directories from kaggle data\n",
        "\n",
        "        mfcc_names = [self.mfcc_dir+f for f in sorted(os.listdir(self.mfcc_dir))] # TODO: List files in sefl.mfcc_dir_dir using os.listdir in sorted order, optionally subset using limit to slice the number of files you load\n",
        "\n",
        "        self.mfccs = []\n",
        "\n",
        "        # TODO:\n",
        "        # Iterate through mfccs\n",
        "        for i in range(0, len(mfcc_names)):\n",
        "        #   Load a single mfcc\n",
        "            mfcc = np.load(mfcc_names[i])\n",
        "        #   Optionally do Cepstral Normalization of mfcc\n",
        "\n",
        "        #   Append each mfcc to self.mfcc\n",
        "            self.mfccs.append(mfcc)\n",
        "        \n",
        "\n",
        "        # NOTE:\n",
        "        # Each mfcc is of shape T1 x 15, T2 x 15, ...\n",
        "        # Each transcript is of shape (T1+2) x 15, (T2+2) x 15 before removing [SOS] and [EOS]\n",
        "\n",
        "        # TODO: Concatenate all mfccs in self.mfccs such that the final shape is T x 15 (Where T = T1 + T2 + ...) \n",
        "        self.mfccs = np.concatenate(self.mfccs)\n",
        "\n",
        "        # Take some time to think about what we have done. self.mfcc is an array of the format (Frames x Features). Our goal is to recognize phonemes of each frame\n",
        "        # From hw0, you will be knowing what context is. We can introduce context by padding zeros on top and bottom of self.mfcc\n",
        "        # self.mfccs = NotImplemented # TODO\n",
        "\n",
        "        # Length of the dataset is now the length of concatenated mfccs/transcripts\n",
        "        self.length = len(self.mfccs) # without the padding\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.length\n",
        "\n",
        "    def __getitem__(self, ind):\n",
        "        ### Calculate starting timestep using offset and context (1 line)\n",
        "        start_ind = ind + self.offset - self.context\n",
        "        \n",
        "        ### Calculate ending timestep using offset and context (1 line)\n",
        "        end_ind = ind + self.offset + self.context + 1\n",
        "        \n",
        "        frames = self.mfccs[start_ind:end_ind] # TODO: Based on context and offset, return a frame at given index with context frames to the left, and right.\n",
        "        # After slicing, you get an array of shape 2*context+1 x 15. But our MLP needs 1d data and not 2d.\n",
        "        frames = frames.flatten() # TODO: Flatten to get 1d data\n",
        "\n",
        "        frames = torch.FloatTensor(frames) # Convert to tensors\n",
        "\n",
        "        return frames"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qNacQ8bpt9nw"
      },
      "source": [
        "# Parameters Configuration"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WE7tsinAuLNy"
      },
      "source": [
        "Storing your parameters and hyperparameters in a single configuration dictionary makes it easier to keep track of them during each experiment. It can also be used with weights and biases to log your parameters for each experiment and keep track of them across multiple experiments. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "PmKwlFqgt_Zq"
      },
      "outputs": [],
      "source": [
        "config = {\n",
        "    'epochs': 10,\n",
        "    'batch_size' : 1024,\n",
        "    'context' : 0,\n",
        "    'learning_rate' : 0.001,\n",
        "    'architecture' : 'very-low-cutoff'\n",
        "    # Add more as you need them - e.g dropout values, weight decay, scheduler parameters\n",
        "}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2mlwaKlDt_2c"
      },
      "source": [
        "# Create Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "7xi7V8x8W9z4"
      },
      "outputs": [],
      "source": [
        "context = 0\n",
        "dir = 'data/'\n",
        "train_data = AudioDataset(dir+\"train-clean-100\", context) #TODO: Create a dataset object using the AudioDataset class for the training data \n",
        "\n",
        "val_data = AudioDataset(dir+\"dev-clean\", context) # TODO: Create a dataset object using the AudioDataset class for the validation data \n",
        "\n",
        "test_data = AudioTestDataset(dir+\"test-clean\", context) # TODO: Create a dataset object using the AudioTestDataset class for the test data "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "4mzoYfTKu14s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3d645a3c-6fcd-434b-ac12-eb345e9923bd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch size:  1024\n",
            "Context:  0\n",
            "Input size:  15\n",
            "Output symbols:  42\n",
            "Train dataset samples = 36191134, batches = 35343\n",
            "Validation dataset samples = 1937496, batches = 1893\n",
            "Test dataset samples = 1943253, batches = 1898\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:566: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        }
      ],
      "source": [
        "# Define dataloaders for train, val and test datasets\n",
        "# Dataloaders will yield a batch of frames and phonemes of given batch_size at every iteration\n",
        "train_loader = torch.utils.data.DataLoader(train_data, num_workers= 4,\n",
        "                                           batch_size=config['batch_size'], pin_memory= True,\n",
        "                                           shuffle= True)\n",
        "\n",
        "val_loader = torch.utils.data.DataLoader(val_data, num_workers= 2,\n",
        "                                         batch_size=config['batch_size'], pin_memory= True,\n",
        "                                         shuffle= False)\n",
        "\n",
        "test_loader = torch.utils.data.DataLoader(test_data, num_workers= 2, \n",
        "                                          batch_size=config['batch_size'], pin_memory= True, \n",
        "                                          shuffle= False)\n",
        "\n",
        "\n",
        "print(\"Batch size: \", config['batch_size'])\n",
        "print(\"Context: \", config['context'])\n",
        "print(\"Input size: \", (2*config['context']+1)*15)\n",
        "print(\"Output symbols: \", len(PHONEMES))\n",
        "\n",
        "print(\"Train dataset samples = {}, batches = {}\".format(train_data.__len__(), len(train_loader)))\n",
        "print(\"Validation dataset samples = {}, batches = {}\".format(val_data.__len__(), len(val_loader)))\n",
        "print(\"Test dataset samples = {}, batches = {}\".format(test_data.__len__(), len(test_loader)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "n-GV3UvgLSoF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e6b585b-2a42-45a0-f31e-360ee5fedd8c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([1024, 15]) torch.Size([1024])\n"
          ]
        }
      ],
      "source": [
        "# Testing code to check if your data loaders are working\n",
        "for i, data in enumerate(train_loader):\n",
        "    frames, phoneme = data\n",
        "    print(frames.shape, phoneme.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nxjwve20JRJ2"
      },
      "source": [
        "# Network Architecture\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3NJzT-mRw6iy"
      },
      "source": [
        "This section defines your network architecture for the homework. We have given you a sample architecture that can easily clear the very low cutoff for the early submission deadline."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "OvcpontXQq9j"
      },
      "outputs": [],
      "source": [
        "# This architecture will make you cross the very low cutoff\n",
        "# However, you need to run a lot of experiments to cross the medium or high cutoff\n",
        "class Network(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, context):\n",
        "\n",
        "        super(Network, self).__init__()\n",
        "\n",
        "        input_size = (2*context + 1) * 15 #Why is this the case?\n",
        "        output_size = 40 #Why?\n",
        "\n",
        "        self.model = torch.nn.Sequential(\n",
        "            torch.nn.Linear(input_size, output_size)\n",
        "        )      \n",
        "\n",
        "    def forward(self, x):\n",
        "        out = self.model(x)\n",
        "\n",
        "        return out"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HejoSXe3vMVU"
      },
      "source": [
        "# Define Model, Loss Function and Optimizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xAhGBH7-xxth"
      },
      "source": [
        "Here we define the model, loss function, optimizer and optionally a learning rate scheduler. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "_qtrEM1ZvLje"
      },
      "outputs": [],
      "source": [
        "input_size = 15*(2*config['context'] + 1)\n",
        "model = Network(config['context']).to(device)\n",
        "frames, phoneme = next(iter(train_loader))\n",
        "# Check number of parameters of your network - Remember, you are limited to 20 million parameters for HW1 (including ensembles)\n",
        "summary(model, frames.to(device))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "UROGEVJevKD-"
      },
      "outputs": [],
      "source": [
        "criterion = torch.nn.CrossEntropyLoss() #Defining Loss function \n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=config['learning_rate']) #Defining Optimizer\n",
        "# Recommended : Define Scheduler for Learning Rate, including but not limited to StepLR, MultiStepLR, CosineAnnealingLR, ReduceLROnPlateau, etc. \n",
        "# You can refer to Pytorch documentation for more information on how to use them."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IBwunYpyugFg"
      },
      "source": [
        "# Training and Validation Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1JgeNhx4x2-P"
      },
      "source": [
        "This section covers the training, and validation functions for each epoch of running your experiment with a given model architecture. The code has been provided to you, but we recommend going through the comments to understand the workflow to enable you to write these loops for future HWs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XblOHEVtKab2",
        "outputId": "f8581e77-69aa-4351-b274-bc1c9d96133c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "104"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "torch.cuda.empty_cache()\n",
        "gc.collect()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "8wjPz7DHqKcL"
      },
      "outputs": [],
      "source": [
        "def train(model, optimizer, criterion, dataloader):\n",
        "\n",
        "    model.train()\n",
        "    train_loss = 0.0 #Monitoring Loss\n",
        "    \n",
        "    for iter, (mfccs, phonemes) in enumerate(dataloader):\n",
        "\n",
        "        ### Move Data to Device (Ideally GPU)\n",
        "        mfccs = mfccs.to(device)\n",
        "        phonemes = phonemes.to(device)\n",
        "\n",
        "        ### Forward Propagation\n",
        "        logits = model(mfccs)\n",
        "\n",
        "        ### Loss Calculation\n",
        "        loss = criterion(logits, phonemes)\n",
        "        train_loss += loss.item()\n",
        "\n",
        "        ### Initialize Gradients\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        ### Backward Propagation\n",
        "        loss.backward()\n",
        "\n",
        "        ### Gradient Descent\n",
        "        optimizer.step()\n",
        "  \n",
        "    train_loss /= len(dataloader)\n",
        "    return train_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "Q5npQNFH315V"
      },
      "outputs": [],
      "source": [
        "def eval(model, dataloader):\n",
        "\n",
        "    model.eval() # set model in evaluation mode\n",
        "\n",
        "    phone_true_list = []\n",
        "    phone_pred_list = []\n",
        "\n",
        "    for i, data in enumerate(dataloader):\n",
        "\n",
        "        frames, phonemes = data\n",
        "        ### Move data to device (ideally GPU)\n",
        "        frames, phonemes = frames.to(device), phonemes.to(device) \n",
        "\n",
        "        with torch.inference_mode(): # makes sure that there are no gradients computed as we are not training the model now\n",
        "            ### Forward Propagation\n",
        "            logits = model(frames)\n",
        "\n",
        "        ### Get Predictions\n",
        "        predicted_phonemes = torch.argmax(logits, dim=1)\n",
        "        \n",
        "        ### Store Pred and True Labels\n",
        "        phone_pred_list.extend(predicted_phonemes.cpu().tolist())\n",
        "        phone_true_list.extend(phonemes.cpu().tolist())\n",
        "        \n",
        "        # Do you think we need loss.backward() and optimizer.step() here?\n",
        "    \n",
        "        del frames, phonemes, logits\n",
        "        torch.cuda.empty_cache()\n",
        "\n",
        "    ### Calculate Accuracy\n",
        "    accuracy = sklearn.metrics.accuracy_score(phone_pred_list, phone_true_list) \n",
        "    return accuracy*100"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yMd_XxPku5qp"
      },
      "source": [
        "# Weights and Biases Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tjIbhR1wwbgI"
      },
      "source": [
        "This section is to enable logging metrics and files with Weights and Biases. Please refer to wandb documentationa and recitation 0 that covers the use of weights and biases for logging, hyperparameter tuning and monitoring your runs for your homeworks. Using this tool makes it very easy to show results when submitting your code and models for homeworks, and also extremely useful for study groups to organize and run ablations under a single team in wandb. \n",
        "\n",
        "We have written code for you to make use of it out of the box, so that you start using wandb for all your HWs from the beginning."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "SCDYx5VEu6qI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "846f6d71-5c86-47c9-c803-f5a79bce8115"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:wandb.jupyter:Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "wandb.login(key=\"a1cf9ce16a0573089f92e793a07818b900c1c8e1\") #API Key is in your wandb account, under settings (wandb.ai/settings)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "xvUnYd3Bw2up",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 462
        },
        "outputId": "2fb28b61-af43-4605-bc87-cceae240fc8b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mkeltonzhang\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Tracking run with wandb version 0.13.3"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Run data is saved locally in <code>/content/wandb/run-20220917_195709-vi8d8xso</code>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Syncing run <strong><a href=\"https://wandb.ai/keltonzhang/hw1p2/runs/vi8d8xso\" target=\"_blank\">first-run</a></strong> to <a href=\"https://wandb.ai/keltonzhang/hw1p2\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Create your wandb run\n",
        "run = wandb.init(\n",
        "    name = \"first-run\", ### Wandb creates random run names if you skip this field, we recommend you give useful names\n",
        "    reinit=True, ### Allows reinitalizing runs when you re-run this cell\n",
        "    project=\"hw1p2\", ### Project should be created in your wandb account \n",
        "    config=config ### Wandb Config for your run\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "wft15E_IxYFi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f49c3f5-42b6-4fcc-a244-251e59172efe"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/content/wandb/run-20220917_195709-vi8d8xso/files/model_arch.txt']"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "### Save your model architecture as a string with str(model) \n",
        "model_arch = str(model)\n",
        "\n",
        "### Save it in a txt file \n",
        "arch_file = open(\"model_arch.txt\", \"w\")\n",
        "file_write = arch_file.write(model_arch)\n",
        "arch_file.close()\n",
        "\n",
        "### log it in your wandb run with wandb.save()\n",
        "wandb.save('model_arch.txt')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nclx_04fu7Dd"
      },
      "source": [
        "# Experiment"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MdLMWfEpyGOB"
      },
      "source": [
        "Now, it is time to finally run your ablations! Have fun!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "MG4F77Nm0Am9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 599
        },
        "outputId": "7e5f06da-b9bb-4099-f55b-b60ed6786570"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Epoch 1/10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:566: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\tTrain Loss: 2.0386\n",
            "\tValidation Accuracy: 42.74%\n",
            "\n",
            "Epoch 2/10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:566: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-6ee8a07efa19>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\nEpoch {}/{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'epochs'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-17-563a2b19260d>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, optimizer, criterion, dataloader)\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.0\u001b[0m \u001b[0;31m#Monitoring Loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmfccs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphonemes\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0;31m### Move Data to Device (Ideally GPU)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    679\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    680\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 681\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    682\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    683\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1357\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1359\u001b[0;31m             \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1360\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_get_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1313\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1314\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_alive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1315\u001b[0;31m                 \u001b[0msuccess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1316\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msuccess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1317\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_try_get_data\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1161\u001b[0m         \u001b[0;31m#   (bool: whether successfully get data, any: data if successful else None)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1162\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1163\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1164\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1165\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/queue.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m    177\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mremaining\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m                         \u001b[0;32mraise\u001b[0m \u001b[0mEmpty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 179\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_empty\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mremaining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    180\u001b[0m             \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_full\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnotify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    298\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m                     \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    301\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m                     \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "# Iterate over number of epochs to train and evaluate your model\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "best_acc = 0.0 ### Monitor best accuracy in your run\n",
        "\n",
        "for epoch in range(config['epochs']):\n",
        "    print(\"\\nEpoch {}/{}\".format(epoch+1, config['epochs']))\n",
        "\n",
        "    train_loss = train(model, optimizer, criterion, train_loader)\n",
        "    accuracy = eval(model, val_loader)\n",
        "\n",
        "    print(\"\\tTrain Loss: {:.4f}\".format(train_loss))\n",
        "    print(\"\\tValidation Accuracy: {:.2f}%\".format(accuracy))\n",
        "\n",
        "\n",
        "    ### Log metrics at each epoch in your run - Optionally, you can log at each batch inside train/eval functions (explore wandb documentation/wandb recitation)\n",
        "    wandb.log({\"train loss\": train_loss, \"validation accuracy\": accuracy})\n",
        "\n",
        "    ### Save checkpoint if accuracy is better than your current best\n",
        "    if accuracy >= best_acc:\n",
        "\n",
        "      ### Save checkpoint with information you want\n",
        "      torch.save({'epoch': epoch,\n",
        "              'model_state_dict': model.state_dict(),\n",
        "              'optimizer_state_dict': optimizer.state_dict(),\n",
        "              'loss': train_loss,\n",
        "              'acc': accuracy}, \n",
        "        './model_checkpoint.pth')\n",
        "      \n",
        "      ### Save checkpoint in wandb\n",
        "      wandb.save('checkpoint.pth')\n",
        "\n",
        "    # Is your training time very high? Look into mixed precision training if your GPU (Tesla T4, V100, etc) can make use of it \n",
        "    # Refer - https://pytorch.org/docs/stable/notes/amp_examples.html\n",
        "\n",
        "### Finish your wandb run\n",
        "run.finish()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_kXwf5YUo_4A"
      },
      "source": [
        "# Testing and submission to Kaggle"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WI1hSFYLpJvH"
      },
      "source": [
        "Before we get to the following code, make sure to see the format of submission given in *random_submission.csv*. Once you have done so, it is time to fill the following function to complete your inference on test data. Refer the eval function from previous cells to get an idea of how to go about completing this function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "R-SU9fZ3xHtk"
      },
      "outputs": [],
      "source": [
        "def test(model, test_loader):\n",
        "  ### What you call for model to perform inference?\n",
        "  model.eval()\n",
        "\n",
        "  ### List to store predicted phonemes of test data\n",
        "  test_predictions = []\n",
        "\n",
        "  ### Which mode do you need to avoid gradients?\n",
        "  with torch.inference_mode():\n",
        "\n",
        "      for i, frames in enumerate(tqdm(test_loader)):\n",
        "\n",
        "          frames = frames.float().to(device)             \n",
        "          \n",
        "          output = model(frames)\n",
        "\n",
        "          ### Get most likely predicted phoneme with argmax\n",
        "          predicted_phonemes = torch.argmax(output, dim=1)\n",
        "\n",
        "          ### How do you store predicted_phonemes with test_predictions? Hint, look at eval \n",
        "          test_predictions += predicted_phonemes\n",
        "  return test_predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "wG9v6Xmxu7wp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 392,
          "referenced_widgets": [
            "4f09d1aad0054aa581edb355c0996ae4",
            "e937d65d32784c78913b1d6e371758c2",
            "9985c360004a4f349bb115fb8f93e764",
            "ca5223c0d49d4bfca9da269643dfc4ef",
            "be5d12c299e545cfa30e2cb4df717bd8",
            "e331db2b2fe548aab8c96673291e6c47",
            "ab0cb4cfc9db49c7970980039076856e",
            "789c491926114b0495fd1bbee31642d4",
            "14f438f00cdc4d93971364d3d96dd0d4",
            "e87ebd7fecfb4a77b598c52c2c953dca",
            "e439a9f206f94c8fb2f744fb18edb563"
          ]
        },
        "outputId": "cc59dba6-6227-4085-c779-315b3f62dffb"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1898 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4f09d1aad0054aa581edb355c0996ae4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x7f5215c2b9e0>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\", line 1510, in __del__\n",
            "    self._shutdown_workers()\n",
            "Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x7f5215c2b9e0>  File \"/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\", line 1493, in _shutdown_workers\n",
            "    \n",
            "if w.is_alive():\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.7/multiprocessing/process.py\", line 151, in is_alive\n",
            "    assert self._parent_pid == os.getpid(), 'can only test a child process'\n",
            "AssertionError  File \"/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\", line 1510, in __del__\n",
            "    : can only test a child process\n",
            "self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\", line 1493, in _shutdown_workers\n",
            "    if w.is_alive():\n",
            "  File \"/usr/lib/python3.7/multiprocessing/process.py\", line 151, in is_alive\n",
            "    assert self._parent_pid == os.getpid(), 'can only test a child process'\n",
            "AssertionError: can only test a child process\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1943253\n"
          ]
        }
      ],
      "source": [
        "predictions = test(model, test_loader)\n",
        "print(len(predictions))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "ZE1hRnvf0bFz"
      },
      "outputs": [],
      "source": [
        "### Create CSV file with predictions\n",
        "with open(\"./submission.csv\", \"w+\") as f:\n",
        "    f.write(\"id,label\\n\")\n",
        "    for i in range(len(predictions)):\n",
        "        f.write(\"{},{}\\n\".format(i, predictions[i]))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "29wnvzKUKHKl",
        "outputId": "d26755b5-41df-4b4a-eb3d-63df4cb2ec4b"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "11-785-f22-hw1p2.zip  drive\t      model_checkpoint.pth  submission.csv\n",
            "data\t\t      model_arch.txt  sample_data\t    wandb\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "LjcammuCxMKN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e0775da-658b-4ab1-9ed3-1c8f34651a62"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.8)\n",
            "100% 18.5M/18.5M [00:05<00:00, 3.26MB/s]\n",
            "Successfully submitted to Frame-Level Speech Recognition"
          ]
        }
      ],
      "source": [
        "### Submit to kaggle competition using kaggle API\n",
        "!kaggle competitions submit -c 11-785-f22-hw1p2 -f ./submission.csv -m \"Test Submission\""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "z4vZbDmJvMp1",
        "ZIi0Big7vPa9",
        "yMd_XxPku5qp"
      ],
      "machine_shape": "hm",
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3.7.13 64-bit ('hw0p1')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.13"
    },
    "vscode": {
      "interpreter": {
        "hash": "1aa077e0fdae99ae001d6a0443850242fd71354508eabb98dff0dfbba672c0cc"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4f09d1aad0054aa581edb355c0996ae4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e937d65d32784c78913b1d6e371758c2",
              "IPY_MODEL_9985c360004a4f349bb115fb8f93e764",
              "IPY_MODEL_ca5223c0d49d4bfca9da269643dfc4ef"
            ],
            "layout": "IPY_MODEL_be5d12c299e545cfa30e2cb4df717bd8"
          }
        },
        "e937d65d32784c78913b1d6e371758c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e331db2b2fe548aab8c96673291e6c47",
            "placeholder": "​",
            "style": "IPY_MODEL_ab0cb4cfc9db49c7970980039076856e",
            "value": "100%"
          }
        },
        "9985c360004a4f349bb115fb8f93e764": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_789c491926114b0495fd1bbee31642d4",
            "max": 1898,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_14f438f00cdc4d93971364d3d96dd0d4",
            "value": 1898
          }
        },
        "ca5223c0d49d4bfca9da269643dfc4ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e87ebd7fecfb4a77b598c52c2c953dca",
            "placeholder": "​",
            "style": "IPY_MODEL_e439a9f206f94c8fb2f744fb18edb563",
            "value": " 1898/1898 [00:34&lt;00:00, 108.57it/s]"
          }
        },
        "be5d12c299e545cfa30e2cb4df717bd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e331db2b2fe548aab8c96673291e6c47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab0cb4cfc9db49c7970980039076856e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "789c491926114b0495fd1bbee31642d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "14f438f00cdc4d93971364d3d96dd0d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e87ebd7fecfb4a77b598c52c2c953dca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e439a9f206f94c8fb2f744fb18edb563": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}