mt5-base-squad-bn.ipynb

{"cells":[{"cell_type":"code","execution_count":34,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:04:42.512156Z","iopub.status.busy":"2022-10-08T17:04:42.511738Z","iopub.status.idle":"2022-10-08T17:04:42.556064Z","shell.execute_reply":"2022-10-08T17:04:42.554813Z","shell.execute_reply.started":"2022-10-08T17:04:42.512121Z"},"id":"YAyPA0luV6S4","outputId":"fd2734ad-f9c8-42d9-ed74-fcb01ed8428d","trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6468af1ab0fe4c56af34be92bba9a73c","version_major":2,"version_minor":0},"text/plain":["VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"]},"metadata":{},"output_type":"display_data"}],"source":["#!huggingface-cli login\n","#!pip install huggingface_hub\n","from huggingface_hub import notebook_login\n","notebook_login()"]},{"cell_type":"code","execution_count":5,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:47:28.174815Z","iopub.status.busy":"2022-10-08T16:47:28.174280Z","iopub.status.idle":"2022-10-08T16:47:56.021950Z","shell.execute_reply":"2022-10-08T16:47:56.020666Z","shell.execute_reply.started":"2022-10-08T16:47:28.174744Z"},"id":"plfqwa8p39Tn","outputId":"9e01f6d9-0889-4401-dca3-5881aee456fd","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Reading package lists... Done\n","Building dependency tree       \n","Reading state information... Done\n","The following NEW packages will be installed:\n","  git-lfs\n","0 upgraded, 1 newly installed, 0 to remove and 65 not upgraded.\n","Need to get 3316 kB of archives.\n","After this operation, 11.1 MB of additional disk space will be used.\n","Get:1 http://archive.ubuntu.com/ubuntu focal/universe amd64 git-lfs amd64 2.9.2-1 [3316 kB]\n","Fetched 3316 kB in 2s (1377 kB/s)  \n","Selecting previously unselected package git-lfs.\n","(Reading database ... 108827 files and directories currently installed.)\n","Preparing to unpack .../git-lfs_2.9.2-1_amd64.deb ...\n","Unpacking git-lfs (2.9.2-1) ...\n","Setting up git-lfs (2.9.2-1) ...\n","Processing triggers for man-db (2.9.1-1) ...\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: wandb in /opt/conda/lib/python3.7/site-packages (0.12.21)\n","Requirement already satisfied: protobuf<4.0dev,>=3.12.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (3.19.4)\n","Requirement already satisfied: shortuuid>=0.5.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (1.0.9)\n","Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from wandb) (59.8.0)\n","Requirement already satisfied: psutil>=5.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (5.9.1)\n","Requirement already satisfied: requests<3,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (2.28.1)\n","Requirement already satisfied: promise<3,>=2.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (2.3)\n","Requirement already satisfied: PyYAML in /opt/conda/lib/python3.7/site-packages (from wandb) (6.0)\n","Requirement already satisfied: GitPython>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (3.1.27)\n","Requirement already satisfied: Click!=8.0.0,>=7.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (8.0.4)\n","Requirement already satisfied: six>=1.13.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (1.15.0)\n","Requirement already satisfied: pathtools in /opt/conda/lib/python3.7/site-packages (from wandb) (0.1.2)\n","Requirement already satisfied: sentry-sdk>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (1.9.8)\n","Requirement already satisfied: setproctitle in /opt/conda/lib/python3.7/site-packages (from wandb) (1.3.2)\n","Requirement already satisfied: docker-pycreds>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from wandb) (0.4.0)\n","Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from Click!=8.0.0,>=7.0->wandb) (4.12.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from GitPython>=1.0.0->wandb) (4.3.0)\n","Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from GitPython>=1.0.0->wandb) (4.0.9)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (1.26.12)\n","Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (2.1.0)\n","Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (2022.6.15.2)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.0.0->wandb) (3.3)\n","Requirement already satisfied: smmap<6,>=3.0.1 in /opt/conda/lib/python3.7/site-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb) (3.0.5)\n","Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->Click!=8.0.0,>=7.0->wandb) (3.8.0)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0m"]}],"source":["# !pip install folium==0.2.1\n","# !pip install folium==0.9.1\n","!sudo apt-get install git-lfs\n","! pip install -q transformers datasets\n","!pip install wandb"]},{"cell_type":"code","execution_count":7,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:49:04.377253Z","iopub.status.busy":"2022-10-08T16:49:04.375998Z","iopub.status.idle":"2022-10-08T16:50:54.894976Z","shell.execute_reply":"2022-10-08T16:50:54.893459Z","shell.execute_reply.started":"2022-10-08T16:49:04.377200Z"},"id":"FSG9qgen3-Su","outputId":"c4b2b5b0-a2c4-44f6-ec8e-7be8754951c9","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Cloning into 'xl-sum'...\n","remote: Enumerating objects: 1156, done.\u001b[K\n","remote: Counting objects: 100% (47/47), done.\u001b[K\n","remote: Compressing objects: 100% (33/33), done.\u001b[K\n","remote: Total 1156 (delta 19), reused 30 (delta 12), pack-reused 1109\u001b[K\n","Receiving objects: 100% (1156/1156), 5.53 MiB | 3.91 MiB/s, done.\n","Resolving deltas: 100% (309/309), done.\n","Requirement already satisfied: tensorboard in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.10.0)\n","Collecting tensorboard\n","  Downloading tensorboard-2.10.1-py3-none-any.whl (5.9 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m0m\n","\u001b[?25hRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 2)) (1.0.2)\n","Collecting seqeval\n","  Downloading seqeval-1.2.2.tar.gz (43 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hRequirement already satisfied: psutil in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 4)) (5.9.1)\n","Collecting psutil\n","  Downloading psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (281 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.3/281.3 kB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting sacrebleu\n","  Downloading sacrebleu-2.2.1-py3-none-any.whl (116 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.9/116.9 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: sentencepiece!=0.1.92 in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 6)) (0.1.97)\n","Requirement already satisfied: protobuf in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 7)) (3.19.4)\n","Collecting protobuf\n","  Downloading protobuf-4.21.7-cp37-abi3-manylinux2014_x86_64.whl (408 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m408.4/408.4 kB\u001b[0m \u001b[31m35.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.6.1)\n","Requirement already satisfied: numpy>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.21.6)\n","Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.3.7)\n","Requirement already satisfied: setuptools>=41.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (59.8.0)\n","Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.8.1)\n","Requirement already satisfied: wheel>=0.26 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.37.1)\n","Requirement already satisfied: absl-py>=0.4 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.15.0)\n","Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.2.2)\n","Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.4.6)\n","Requirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.28.1)\n","Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.35.0)\n","  Downloading protobuf-3.19.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m40.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: grpcio>=1.24.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.43.0)\n","Requirement already satisfied: scipy>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->-r ./xl-sum/seq2seq/requirements.txt (line 2)) (1.7.3)\n","Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->-r ./xl-sum/seq2seq/requirements.txt (line 2)) (1.0.1)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->-r ./xl-sum/seq2seq/requirements.txt (line 2)) (3.1.0)\n","Requirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (0.8.10)\n","Requirement already satisfied: colorama in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (0.4.5)\n","Requirement already satisfied: lxml in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (4.9.1)\n","Requirement already satisfied: portalocker in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (2.5.1)\n","Requirement already satisfied: regex in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (2021.11.10)\n","Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from absl-py>=0.4->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.15.0)\n","Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.8)\n","Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.2.4)\n","Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.2.7)\n","Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.3.1)\n","Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.12.0)\n","Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2022.6.15.2)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.3)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.26.12)\n","Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.1.0)\n","Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.7/site-packages (from werkzeug>=1.0.1->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.1.1)\n","Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.8.0)\n","Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.3.0)\n","Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.4.8)\n","Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.2.0)\n","Building wheels for collected packages: seqeval\n","  Building wheel for seqeval (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16180 sha256=89eb7d94534e75882234f27ee8027d30f4588433be3ce7c800cb9531b3d3cb6c\n","  Stored in directory: /root/.cache/pip/wheels/05/96/ee/7cac4e74f3b19e3158dce26a20a1c86b3533c43ec72a549fd7\n","Successfully built seqeval\n","Installing collected packages: sacrebleu, psutil, protobuf, seqeval, tensorboard\n","  Attempting uninstall: psutil\n","    Found existing installation: psutil 5.9.1\n","    Uninstalling psutil-5.9.1:\n","      Successfully uninstalled psutil-5.9.1\n","  Attempting uninstall: protobuf\n","    Found existing installation: protobuf 3.19.4\n","    Uninstalling protobuf-3.19.4:\n","      Successfully uninstalled protobuf-3.19.4\n","  Attempting uninstall: tensorboard\n","    Found existing installation: tensorboard 2.10.0\n","    Uninstalling tensorboard-2.10.0:\n","      Successfully uninstalled tensorboard-2.10.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","tensorflow-io 0.21.0 requires tensorflow-io-gcs-filesystem==0.21.0, which is not installed.\n","dask-cudf 21.10.1 requires cupy-cuda114, which is not installed.\n","beatrix-jupyterlab 3.1.7 requires google-cloud-bigquery-storage, which is not installed.\n","tfx-bsl 1.9.0 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5, but you have tensorflow 2.6.4 which is incompatible.\n","tensorflow 2.6.4 requires h5py~=3.1.0, but you have h5py 3.7.0 which is incompatible.\n","tensorflow 2.6.4 requires numpy~=1.19.2, but you have numpy 1.21.6 which is incompatible.\n","tensorflow 2.6.4 requires tensorboard<2.7,>=2.6.0, but you have tensorboard 2.10.1 which is incompatible.\n","tensorflow 2.6.4 requires typing-extensions<3.11,>=3.7, but you have typing-extensions 4.3.0 which is incompatible.\n","tensorflow-transform 1.9.0 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<2.10,>=1.15.5, but you have tensorflow 2.6.4 which is incompatible.\n","tensorflow-serving-api 2.9.0 requires tensorflow<3,>=2.9.0, but you have tensorflow 2.6.4 which is incompatible.\n","pdpbox 0.2.1 requires matplotlib==3.1.1, but you have matplotlib 3.5.3 which is incompatible.\n","nnabla 1.30.0 requires protobuf<=3.19.4; platform_system != \"Windows\", but you have protobuf 3.19.6 which is incompatible.\n","grpcio-status 1.47.0 requires grpcio>=1.47.0, but you have grpcio 1.43.0 which is incompatible.\n","google-api-core 1.33.0 requires protobuf<4.0.0dev,>=3.20.1, but you have protobuf 3.19.6 which is incompatible.\n","gcsfs 2022.5.0 requires fsspec==2022.5.0, but you have fsspec 2022.8.2 which is incompatible.\n","dask-cudf 21.10.1 requires dask==2021.09.1, but you have dask 2022.2.0 which is incompatible.\n","dask-cudf 21.10.1 requires distributed==2021.09.1, but you have distributed 2022.2.0 which is incompatible.\n","apache-beam 2.40.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.5.1 which is incompatible.\n","allennlp 2.10.0 requires protobuf==3.20.0, but you have protobuf 3.19.6 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed protobuf-3.19.6 psutil-5.9.2 sacrebleu-2.2.1 seqeval-1.2.2 tensorboard-2.10.1\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0m\u001b[31mERROR: Invalid requirement: 'transformers/'\n","Hint: It looks like a path. File 'transformers/' does not exist.\u001b[0m\u001b[31m\n","\u001b[0mCollecting git+https://github.com/otuncelli/turkish-stemmer-python (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 1))\n","  Cloning https://github.com/otuncelli/turkish-stemmer-python to /tmp/pip-req-build-7hi5_z4a\n","  Running command git clone --filter=blob:none --quiet https://github.com/otuncelli/turkish-stemmer-python /tmp/pip-req-build-7hi5_z4a\n","  Resolved https://github.com/otuncelli/turkish-stemmer-python to commit 0c22380bf84a5ab1f219f4a905274c78afa04ed1\n","  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hCollecting git+https://github.com/abhik1505040/bengali-stemmer (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 2))\n","  Cloning https://github.com/abhik1505040/bengali-stemmer to /tmp/pip-req-build-l47v7d06\n","  Running command git clone --filter=blob:none --quiet https://github.com/abhik1505040/bengali-stemmer /tmp/pip-req-build-l47v7d06\n","  Resolved https://github.com/abhik1505040/bengali-stemmer to commit 375186caee8e50e3260dd6bc02d20d50277f3e39\n","  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hRequirement already satisfied: absl-py in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 3)) (0.15.0)\n","Requirement already satisfied: nltk in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (3.7)\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 5)) (1.21.6)\n","Requirement already satisfied: six>=1.14 in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 6)) (1.15.0)\n","Collecting pythainlp\n","  Downloading pythainlp-3.1.0-py3-none-any.whl (9.6 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m0m\n","\u001b[?25hCollecting pyonmttok\n","  Downloading pyonmttok-1.34.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.9 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.9/16.9 MB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n","\u001b[?25hRequirement already satisfied: jieba in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 9)) (0.42.1)\n","Collecting fugashi[unidic]\n","  Downloading fugashi-1.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (583 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m583.5/583.5 kB\u001b[0m \u001b[31m37.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (1.0.1)\n","Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (8.0.4)\n","Requirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (2021.11.10)\n","Requirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (4.64.0)\n","Requirement already satisfied: requests>=2.22.0 in /opt/conda/lib/python3.7/site-packages (from pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (2.28.1)\n","Collecting unidic\n","  Downloading unidic-1.1.0.tar.gz (7.7 kB)\n","  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (2022.6.15.2)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (3.3)\n","Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (2.1.0)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (1.26.12)\n","Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from click->nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (4.12.0)\n","Requirement already satisfied: wasabi<1.0.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from unidic->fugashi[unidic]->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 10)) (0.10.1)\n","Collecting plac<2.0.0,>=1.1.3\n","  Downloading plac-1.3.5-py2.py3-none-any.whl (22 kB)\n","Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (3.8.0)\n","Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (4.3.0)\n","Building wheels for collected packages: TurkishStemmer, bengali-stemmer, unidic\n","  Building wheel for TurkishStemmer (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for TurkishStemmer: filename=TurkishStemmer-1.3-py3-none-any.whl size=19871 sha256=197275a907938b014ece43d85f324ccdf281b309f451c302a331840634218a5e\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-w0p9nn89/wheels/ee/65/db/c127b9a272f949d5a421a9b7b43128aa8b1d143bafa52f10d1\n","  Building wheel for bengali-stemmer (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for bengali-stemmer: filename=bengali_stemmer-0.0.1-py2.py3-none-any.whl size=6408 sha256=e14e369878c48eff2ace9c6b404d052eb5bc930d82bed1b088f84e67c3c8ee07\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-w0p9nn89/wheels/6a/f4/ee/9298bdab6928b70071ca2876ff3950a7d4adb36ed489be77d2\n","  Building wheel for unidic (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for unidic: filename=unidic-1.1.0-py3-none-any.whl size=7426 sha256=efd4009cc6437bb8abb18fc0f5ce8c3c2cd9cffbaa4efeab9bf588abe85fb219\n","  Stored in directory: /root/.cache/pip/wheels/ce/4d/f1/170bb74b559ca338113c0315c9805e16dfd0a12411ec6b1122\n","Successfully built TurkishStemmer bengali-stemmer unidic\n","Installing collected packages: TurkishStemmer, plac, bengali-stemmer, pyonmttok, fugashi, unidic, pythainlp\n","Successfully installed TurkishStemmer-1.3 bengali-stemmer-0.0.1 fugashi-1.2.0 plac-1.3.5 pyonmttok-1.34.0 pythainlp-3.1.0 unidic-1.1.0\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mdownload url: https://cotonoha-dic.s3-ap-northeast-1.amazonaws.com/unidic-3.1.0.zip\n","Dictionary version: 3.1.0+2021-08-31\n","Downloading UniDic v3.1.0+2021-08-31...\n","unidic-3.1.0.zip: 100%|██████████████████████| 526M/526M [00:27<00:00, 19.3MB/s]\n","Finished download.\n","Downloaded UniDic v3.1.0+2021-08-31 to /opt/conda/lib/python3.7/site-packages/unidic/dicdir\n","Processing ./xl-sum/multilingual_rouge_scoring\n","  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hRequirement already satisfied: absl-py in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (0.15.0)\n","Requirement already satisfied: nltk in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (3.7)\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (1.21.6)\n","Requirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (1.15.0)\n","Requirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (4.64.0)\n","Requirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (1.0.1)\n","Requirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (2021.11.10)\n","Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (8.0.4)\n","Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from click->nltk->rouge-score==0.0.0) (4.12.0)\n","Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->rouge-score==0.0.0) (4.3.0)\n","Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->rouge-score==0.0.0) (3.8.0)\n","Building wheels for collected packages: rouge-score\n","  Building wheel for rouge-score (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for rouge-score: filename=rouge_score-0.0.0-py3-none-any.whl size=18182 sha256=f4ad3eb36d50a1fedddd7a43f0e832988c66cc1aa952799bd536bff95fdcf860\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-7l9arqtu/wheels/4f/07/35/339fd97dbcdebcdd6421137bf0930ce61a48e307e357213aa0\n","Successfully built rouge-score\n","Installing collected packages: rouge-score\n","Successfully installed rouge-score-0.0.0\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0m/opt/conda/lib/python3.7/runpy.py:125: RuntimeWarning: 'nltk.downloader' found in sys.modules after import of package 'nltk', but prior to execution of 'nltk.downloader'; this may result in unpredictable behaviour\n","  warn(RuntimeWarning(msg))\n","[nltk_data] Downloading package punkt to /usr/share/nltk_data...\n","[nltk_data]   Package punkt is already up-to-date!\n"]}],"source":["! git clone https://github.com/csebuetnlp/xl-sum\n","! pip install --upgrade -r ./xl-sum/seq2seq/requirements.txt\n","! pip install --upgrade transformers/\n","\n","# install rouge module and dependecies\n","! pip install -r ./xl-sum/multilingual_rouge_scoring/requirements.txt\n","! python -m unidic download # for japanese segmentation\n","! pip install --upgrade ./xl-sum/multilingual_rouge_scoring\n","! python -m nltk.downloader punkt"]},{"cell_type":"code","execution_count":10,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:51:39.182302Z","iopub.status.busy":"2022-10-08T16:51:39.180889Z","iopub.status.idle":"2022-10-08T16:51:39.188058Z","shell.execute_reply":"2022-10-08T16:51:39.186896Z","shell.execute_reply.started":"2022-10-08T16:51:39.182261Z"},"id":"oKOMJghk3-TY","trusted":true},"outputs":[],"source":["import os\n","os.environ[\"WANDB_DISABLED\"] = \"true\"\n","os.environ[\"TOKENIZERS_PARALLELISM\"]='false'"]},{"cell_type":"code","execution_count":11,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:51:40.234585Z","iopub.status.busy":"2022-10-08T16:51:40.234208Z","iopub.status.idle":"2022-10-08T16:51:40.239568Z","shell.execute_reply":"2022-10-08T16:51:40.238541Z","shell.execute_reply.started":"2022-10-08T16:51:40.234554Z"},"id":"AzE5b8of6zFF","trusted":true},"outputs":[],"source":["import datasets\n","import nltk\n","import numpy as np\n","\n","import logging\n","\n","logger = logging.getLogger(__name__)"]},{"cell_type":"markdown","metadata":{"id":"agUs01Ms12b_"},"source":["## HF auth"]},{"cell_type":"code","execution_count":12,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:51:43.693683Z","iopub.status.busy":"2022-10-08T16:51:43.693313Z","iopub.status.idle":"2022-10-08T16:51:44.653340Z","shell.execute_reply":"2022-10-08T16:51:44.651963Z","shell.execute_reply.started":"2022-10-08T16:51:43.693651Z"},"id":"IADibkaEs79C","trusted":true},"outputs":[],"source":["!git config --global credential.helper store"]},{"cell_type":"markdown","metadata":{"id":"qZQ0lpecFdiF"},"source":["## Model and tokenizer"]},{"cell_type":"code","execution_count":13,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:51:46.369941Z","iopub.status.busy":"2022-10-08T16:51:46.369235Z","iopub.status.idle":"2022-10-08T16:51:56.661101Z","shell.execute_reply":"2022-10-08T16:51:56.659843Z","shell.execute_reply.started":"2022-10-08T16:51:46.369902Z"},"id":"3MUOs2W6AtrZ","outputId":"e942c8cf-dcaf-4975-b9bf-e28907831397","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.7/site-packages (0.1.97)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0m"]}],"source":["# !pip install git+https://github.com/csebuetnlp/normalizer\n","!pip install sentencepiece\n","# !pip install transformers"]},{"cell_type":"code","execution_count":14,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:51:56.665206Z","iopub.status.busy":"2022-10-08T16:51:56.664804Z","iopub.status.idle":"2022-10-08T16:54:57.482229Z","shell.execute_reply":"2022-10-08T16:54:57.481112Z","shell.execute_reply.started":"2022-10-08T16:51:56.665171Z"},"id":"NiuzkIia_i3K","outputId":"943a0679-23dd-4433-80b6-2238121a23ce","trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"afb165fdbcdd4bdbaab1bea7c18a736d","version_major":2,"version_minor":0},"text/plain":["Downloading:   0%|          | 0.00/376 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"23a3cf8990a24953b4ead700a19cf838","version_major":2,"version_minor":0},"text/plain":["Downloading:   0%|          | 0.00/702 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"acd5f4cbb7b74dabbd117de5184a56a9","version_major":2,"version_minor":0},"text/plain":["Downloading:   0%|          | 0.00/4.11M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f0c416649d1d4d9aa16137712d1cc228","version_major":2,"version_minor":0},"text/plain":["Downloading:   0%|          | 0.00/65.0 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.7/site-packages/transformers/convert_slow_tokenizer.py:435: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n","  \"The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option\"\n","You are using a model of type mt5 to instantiate a model of type t5. This is not supported for all configurations of models and can yield errors.\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"9eb5f70d42ad44f1a2202cee8d36d7c7","version_major":2,"version_minor":0},"text/plain":["Downloading:   0%|          | 0.00/2.17G [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"}],"source":["# from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n","# from normalizer import normalize # pip install git+https://github.com/csebuetnlp/normalizer\n","CKPT = 'google/mt5-base'\n","from transformers import AutoTokenizer, T5ForConditionalGeneration\n","tokenizer = AutoTokenizer.from_pretrained(CKPT)\n","model = T5ForConditionalGeneration.from_pretrained(CKPT)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]},{"cell_type":"markdown","metadata":{"id":"OcC0F2ED2CKJ"},"source":["## Dataset preparation"]},{"cell_type":"code","execution_count":15,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:56:00.387045Z","iopub.status.busy":"2022-10-08T16:56:00.385877Z","iopub.status.idle":"2022-10-08T16:56:34.367520Z","shell.execute_reply":"2022-10-08T16:56:34.366595Z","shell.execute_reply.started":"2022-10-08T16:56:00.387002Z"},"id":"DptkSlj1V6S8","outputId":"ddae9dc3-514a-49f8-a11b-a635d51947e6","trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1a026e7c4d4c48738a07256159f27987","version_major":2,"version_minor":0},"text/plain":["Downloading builder script:   0%|          | 0.00/4.20k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Downloading and preparing dataset squad_bn/squad_bn to /root/.cache/huggingface/datasets/csebuetnlp___squad_bn/squad_bn/0.0.1/17a6d6abc976f299afda17ca9b5ce08a022ecafabe24b3362e16a3093c32df4b...\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"504bb5ab078143b082af6973ad1ad923","version_major":2,"version_minor":0},"text/plain":["Downloading data:   0%|          | 0.00/8.43M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Generating train split: 0 examples [00:00, ? examples/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Generating test split: 0 examples [00:00, ? examples/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Generating validation split: 0 examples [00:00, ? examples/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Dataset squad_bn downloaded and prepared to /root/.cache/huggingface/datasets/csebuetnlp___squad_bn/squad_bn/0.0.1/17a6d6abc976f299afda17ca9b5ce08a022ecafabe24b3362e16a3093c32df4b. Subsequent calls will reuse this data.\n"]}],"source":["from datasets import concatenate_datasets, load_dataset\n","\n","train_data = load_dataset(\"csebuetnlp/squad_bn\", split=\"train\")\n","valid_data = load_dataset(\"csebuetnlp/squad_bn\", split=\"validation\")\n","test_data = load_dataset(\"csebuetnlp/squad_bn\", split=\"test\")\n","\n","concat_dataset = concatenate_datasets([train_data, valid_data, test_data])\n","dataset=concat_dataset"]},{"cell_type":"code","execution_count":16,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:56:34.369566Z","iopub.status.busy":"2022-10-08T16:56:34.369210Z","iopub.status.idle":"2022-10-08T16:56:34.377373Z","shell.execute_reply":"2022-10-08T16:56:34.376522Z","shell.execute_reply.started":"2022-10-08T16:56:34.369530Z"},"trusted":true},"outputs":[],"source":["del concat_dataset, train_data, valid_data, test_data"]},{"cell_type":"code","execution_count":17,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:56:41.848681Z","iopub.status.busy":"2022-10-08T16:56:41.848316Z","iopub.status.idle":"2022-10-08T16:56:47.526412Z","shell.execute_reply":"2022-10-08T16:56:47.525287Z","shell.execute_reply.started":"2022-10-08T16:56:41.848649Z"},"id":"EE2ErtwH3-WG","outputId":"530adb94-8317-4b68-fd00-da59a5ae2856","trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"8e2205555cc54dd0a505a299c3141f25","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/124 [00:00<?, ?ba/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"25b6fe1fba9543a8b3faaa5b689f24cd","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/72 [00:00<?, ?ba/s]"]},"metadata":{},"output_type":"display_data"}],"source":["\n","small_dataset= dataset.filter(lambda example: len(example['answers']['text'])>0)\n","small_dataset= small_dataset.filter(lambda example: len(example['context'])>300)\n","# for i in range(0,10):\n","#     print(small_dataset['train'][i])"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-10-06T10:46:16.870726Z","iopub.status.busy":"2022-10-06T10:46:16.870307Z","iopub.status.idle":"2022-10-06T10:46:16.886976Z","shell.execute_reply":"2022-10-06T10:46:16.88581Z","shell.execute_reply.started":"2022-10-06T10:46:16.87069Z"},"id":"UNGw1vdnV6S-","outputId":"55f35696-8038-4014-f0df-ad051bfb22aa","trusted":true},"outputs":[],"source":["# 0.2 * len(dataset)"]},{"cell_type":"code","execution_count":19,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:58:14.722267Z","iopub.status.busy":"2022-10-08T16:58:14.721869Z","iopub.status.idle":"2022-10-08T16:58:14.735897Z","shell.execute_reply":"2022-10-08T16:58:14.734799Z","shell.execute_reply.started":"2022-10-08T16:58:14.722233Z"},"id":"Yup_2HAaV6S-","trusted":true},"outputs":[],"source":["split_data = small_dataset.train_test_split(test_size=0.2, seed= 42, shuffle=False)"]},{"cell_type":"code","execution_count":20,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:58:15.030450Z","iopub.status.busy":"2022-10-08T16:58:15.029526Z","iopub.status.idle":"2022-10-08T16:58:15.036281Z","shell.execute_reply":"2022-10-08T16:58:15.035166Z","shell.execute_reply.started":"2022-10-08T16:58:15.030401Z"},"id":"T-U9AacyV6S-","trusted":true},"outputs":[],"source":["dataset = split_data"]},{"cell_type":"code","execution_count":21,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:58:15.552475Z","iopub.status.busy":"2022-10-08T16:58:15.551755Z","iopub.status.idle":"2022-10-08T16:58:15.557224Z","shell.execute_reply":"2022-10-08T16:58:15.556139Z","shell.execute_reply.started":"2022-10-08T16:58:15.552428Z"},"trusted":true},"outputs":[],"source":["del split_data"]},{"cell_type":"code","execution_count":22,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:58:16.655696Z","iopub.status.busy":"2022-10-08T16:58:16.655326Z","iopub.status.idle":"2022-10-08T16:58:16.664788Z","shell.execute_reply":"2022-10-08T16:58:16.663838Z","shell.execute_reply.started":"2022-10-08T16:58:16.655663Z"},"id":"9jx2Gr4-V6TA","outputId":"6aad3e37-12d3-46e5-de84-8ae0858d4a7b","trusted":true},"outputs":[{"data":{"text/plain":["{'id': 'bengali-6703107172676988032-5-4355',\n"," 'title': 'পাল সাম্রাজ্য',\n"," 'context': 'শশাঙ্কের রাজ্যের পতনের পর বাংলা অঞ্চলে নৈরাজ্য দেখা দেয়। এই সময় এই অঞ্চলে কোনও কেন্দ্রীয় শাসক ছিলেন না। ক্ষুদ্র গোষ্ঠীপতিরা নিরন্তর নিজেদের মধ্যে যুদ্ধে লিপ্ত ছিলেন। সমসাময়িক গ্রন্থে এই অবস্থাটিকে ‘মাৎস্যন্যায়’ (অর্থাৎ বড়ো মাছ যেমন ছোটো মাছকে খেয়ে ফেলে, সেই রকম অবস্থা) বলে বর্ণনা করা হয়েছে। এই সময়েই গোপাল প্রথম পাল রাজা হিসেবে সিংহাসনে আরোহণ করেন। খালিমপুর তাম্রলিপি থেকে অনুমিত হয়, বাংলা অঞ্চলের ‘প্রকৃতি’ (জনসাধারণ) তাঁকে রাজা নির্বাচিত করেছিল।[7] প্রায় ৮০০ বছর পরে তারানাথও লিখেছেন যে, বাংলার জনসাধারণ গণতান্ত্রিক পদ্ধতিতে তাঁকে নির্বাচিত করেছিল। যদিও এই ঘটনাটি কিংবদন্তির আকারে প্রচলিত এবং ঐতিহাসিকভাবে নির্ভরযোগ্য তথ্য নয়। এই কিংবদন্তি অনুসারে, নৈরাজ্যের এক যুগের পর জনসাধারণ পরপর একাধিক রাজাকে নির্বাচিত করেছিলেন। কিন্তু তাঁদের সকলকেই নির্বাচনের পরের রাতেই এক নাগ রানি ভক্ষণ করেন। গোপাল সেই নাগ রানিকে হত্যা করতে সমর্থ হন এবং সিংহাসনে আসীন থাকতে সমর্থ হন।[17] ঐতিহাসিক প্রমাণ নির্দেশ করে যে, গোপাল প্রত্যক্ষভাবে জনসাধারণ কর্তৃক নির্বাচিত হননি। একদল সামন্ত গোষ্ঠীপতি তাঁকে নির্বাচিত করেন। এই ধরনের নির্বাচন বাংলা অঞ্চলের সমসাময়িক সমাজে খুবই সাধারণ ঘটনা ছিল।[7][17]',\n"," 'question': 'বাংলার পালবংশের প্রতিষ্ঠাতা কে ?',\n"," 'answers': {'text': ['গোপাল'], 'answer_start': [310]}}"]},"execution_count":22,"metadata":{},"output_type":"execute_result"}],"source":["dataset['test'][13537]"]},{"cell_type":"code","execution_count":23,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:58:52.854939Z","iopub.status.busy":"2022-10-08T16:58:52.854549Z","iopub.status.idle":"2022-10-08T16:58:52.860012Z","shell.execute_reply":"2022-10-08T16:58:52.859035Z","shell.execute_reply.started":"2022-10-08T16:58:52.854906Z"},"id":"gaTsiEd0D-hU","trusted":true},"outputs":[],"source":["def format_dataset(example):\n","  \n","        return {'input': 'answer: ' + example['answers']['text'][0] + ' context: ' + example['context'], 'target': example['question']}"]},{"cell_type":"code","execution_count":24,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:58:53.409300Z","iopub.status.busy":"2022-10-08T16:58:53.408912Z","iopub.status.idle":"2022-10-08T16:59:04.166162Z","shell.execute_reply":"2022-10-08T16:59:04.165176Z","shell.execute_reply.started":"2022-10-08T16:58:53.409267Z"},"id":"mt-c0E0GEPI7","outputId":"9f152292-77c6-4daa-88ea-ccc3df4a8989","trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ca9596b5816241769ff8daed5c332a19","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/54150 [00:00<?, ?ex/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c73da2ac3f17407eb5e232d4ef3ee637","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/13538 [00:00<?, ?ex/s]"]},"metadata":{},"output_type":"display_data"}],"source":["dataset = dataset.map(format_dataset, remove_columns=dataset['train'].column_names)\n"]},{"cell_type":"code","execution_count":25,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T16:59:27.020654Z","iopub.status.busy":"2022-10-08T16:59:27.020180Z","iopub.status.idle":"2022-10-08T16:59:27.029812Z","shell.execute_reply":"2022-10-08T16:59:27.027860Z","shell.execute_reply.started":"2022-10-08T16:59:27.020611Z"},"id":"ppkB_UwQH7a6","trusted":true},"outputs":[],"source":["# from transformers import set_seed\n","\n","# tokenize the examples\n","def convert_to_features(example_batch):\n","    input_encodings = tokenizer.batch_encode_plus(example_batch['input'], truncation=True, padding='max_length', max_length=564)\n","    target_encodings = tokenizer.batch_encode_plus(example_batch['target'], truncation=True, padding='max_length', max_length=64)\n","\n","    encodings = {\n","        'input_ids': input_encodings['input_ids'], \n","        'attention_mask': input_encodings['attention_mask'],\n","        'labels': target_encodings['input_ids'],\n","        'decoder_attention_mask': target_encodings['attention_mask']\n","    }\n","\n","    return encodings"]},{"cell_type":"code","execution_count":26,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:00:09.583845Z","iopub.status.busy":"2022-10-08T17:00:09.583433Z","iopub.status.idle":"2022-10-08T17:01:29.762033Z","shell.execute_reply":"2022-10-08T17:01:29.761050Z","shell.execute_reply.started":"2022-10-08T17:00:09.583802Z"},"id":"JxPcQH1aIEj_","outputId":"211edddb-eb75-4a00-ea31-ff29d56ea7ee","trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a703874370f34164b29ff0350eaf3cc9","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/55 [00:00<?, ?ba/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"357192a9110a454b8d1a748f9758141b","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/14 [00:00<?, ?ba/s]"]},"metadata":{},"output_type":"display_data"}],"source":["dataset = dataset.map(convert_to_features, batched=True, remove_columns=dataset['train'].column_names)\n","\n","columns = ['input_ids', 'attention_mask', 'labels', 'decoder_attention_mask']\n","\n","dataset.set_format(type='torch', columns=columns)"]},{"cell_type":"code","execution_count":28,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:03:34.046981Z","iopub.status.busy":"2022-10-08T17:03:34.046162Z","iopub.status.idle":"2022-10-08T17:03:34.052065Z","shell.execute_reply":"2022-10-08T17:03:34.051041Z","shell.execute_reply.started":"2022-10-08T17:03:34.046947Z"},"id":"wRNxG52qIYMf","trusted":true},"outputs":[],"source":["from transformers import Seq2SeqTrainer\n","from transformers import Seq2SeqTrainingArguments"]},{"cell_type":"markdown","metadata":{"id":"ZI7fTwDYV6TF"},"source":["# # Metric"]},{"cell_type":"code","execution_count":29,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:03:35.966373Z","iopub.status.busy":"2022-10-08T17:03:35.966013Z","iopub.status.idle":"2022-10-08T17:03:36.042129Z","shell.execute_reply":"2022-10-08T17:03:36.041142Z","shell.execute_reply.started":"2022-10-08T17:03:35.966343Z"},"id":"t1BhI4r9V6TF","trusted":true},"outputs":[],"source":["from rouge_score import rouge_scorer, scoring\n","\n","def add_newline_to_end_of_each_sentence(x):\n","        return \"\\n\".join(nltk.sent_tokenize(x))\n","\n","def extract_rouge_mid_statistics(dct):\n","        new_dict = {}\n","        for k1, v1 in dct.items():\n","            mid = v1.mid\n","            new_dict[k1] = {stat: round(getattr(mid, stat), 4) for stat in [\"precision\", \"recall\", \"fmeasure\"]}\n","        return new_dict\n","\n","\n","def calculate_rouge(\n","        pred_lns,\n","        tgt_lns,\n","        use_stemmer=True,\n","        rouge_keys=[\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"],\n","        return_precision_and_recall=False,\n","        bootstrap_aggregation=True,\n","        newline_sep=True,\n","       rouge_lang='bengali'\n","    ):\n","        \n","        logger.info(\"Rouge lang: \" + str(rouge_lang))\n","        scorer = rouge_scorer.RougeScorer(\n","            rouge_keys, lang=rouge_lang,\n","            use_stemmer=use_stemmer\n","        )\n","        aggregator = scoring.BootstrapAggregator()\n","        for pred, tgt in zip(tgt_lns, pred_lns):\n","            # rougeLsum expects \"\\n\" separated sentences within a summary\n","            if newline_sep:\n","                pred = add_newline_to_end_of_each_sentence(pred)\n","                tgt = add_newline_to_end_of_each_sentence(tgt)\n","            scores = scorer.score(pred, tgt)\n","            aggregator.add_scores(scores)\n","\n","        if bootstrap_aggregation:\n","            result = aggregator.aggregate()\n","            if return_precision_and_recall:\n","                return extract_rouge_mid_statistics(result)  # here we return dict\n","            else:\n","                results_precision = {k: round(v.mid.precision * 100, 4) for k, v in result.items()}\n","                results_recall = {k: round(v.mid.recall * 100, 4) for k, v in result.items()}\n","                results_fmeasure = {k: round(v.mid.fmeasure * 100, 4) for k, v in result.items()}\n","            \n","            return {\n","                    \"rouge1_precision\": results_precision.get('rouge1'),\n","                    \"rouge1_recall\": results_recall.get('rouge1'),\n","                    \"rouge1_fmeasure\": results_fmeasure.get('rouge1'),\n","\n","                    \"rouge2_precision\": results_precision.get('rouge2'),\n","                    \"rouge2_recall\": results_recall.get('rouge2'),\n","                    \"rouge2_fmeasure\": results_fmeasure.get('rouge2'),\n","\n","                    \"rougeL_precision\": results_precision.get('rougeL'),\n","                    \"rougeL_recall\": results_recall.get('rougeL'),\n","                    \"rougeL_fmeasure\": results_fmeasure.get('rougeL'),\n","                \n","                    \"rougeLsum_precision\": results_precision.get('rougeLsum'),\n","                    \"rougeLsum_recall\": results_recall.get('rougeLsum'),\n","                    \"rougeLsum_fmeasure\": results_fmeasure.get('rougeLsum'),\n","                }\n","#                 return {k: round(v.mid.fmeasure * 100, 4) for k, v in result.items()}\n","\n","        else:\n","            return aggregator._scores  # here we return defaultdict(list)\n","\n","        \n","metric_fn = calculate_rouge\n"]},{"cell_type":"code","execution_count":30,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:03:43.877155Z","iopub.status.busy":"2022-10-08T17:03:43.876717Z","iopub.status.idle":"2022-10-08T17:03:46.628884Z","shell.execute_reply":"2022-10-08T17:03:46.627913Z","shell.execute_reply.started":"2022-10-08T17:03:43.877121Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d3366b469c9245b9ae36ecf3f959d636","version_major":2,"version_minor":0},"text/plain":["Downloading builder script:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["[nltk_data] Downloading package wordnet to /usr/share/nltk_data...\n","[nltk_data]   Package wordnet is already up-to-date!\n","[nltk_data] Downloading package punkt to /usr/share/nltk_data...\n","[nltk_data]   Package punkt is already up-to-date!\n","[nltk_data] Downloading package omw-1.4 to /usr/share/nltk_data...\n"]}],"source":["import numpy as np\n","from datasets import load_metric\n","# from sacrebleu.metrics import BLEU\n","# bleu = BLEU(max_ngram_order=4)\n","# bleu1 = BLEU(max_ngram_order=1)\n","# bleu2 = BLEU(max_ngram_order=2)\n","# bleu3 = BLEU(max_ngram_order=3)\n","\n","from torchmetrics import SacreBLEUScore\n","\n","bleu_metric1 = SacreBLEUScore(n_gram=1)\n","bleu_metric2 = SacreBLEUScore(n_gram=2)\n","bleu_metric3 = SacreBLEUScore(n_gram=3)\n","bleu_metric4 = SacreBLEUScore(n_gram=4)\n","\n","# from nlgeval import NLGEval\n","# nlgeval = NLGEval()  # loads the models\n","# metrics_dict = nlgeval.compute_metrics(references, hypothesis)\n","\n","# metric = load_metric(\"sacrebleu\")\n","meteor = load_metric('meteor')\n","\n","def postprocess_text(preds, labels):\n","    preds = [pred.strip() for pred in preds]\n","    labels = [[label.strip()] for label in labels]\n","    return preds, labels\n","\n","def compute_metrics(eval_preds):\n","    preds, labels = eval_preds\n","    if isinstance(preds, tuple):\n","        preds = preds[0]\n","        \n","    # Replace -100 in the labels as we can't decode them.\n","    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n","    \n","    \n","    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n","    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n","    \n","    \n","    decoded_preds_rouge = tokenizer.batch_decode(preds, skip_special_tokens=True)\n","    decoded_labels_rouge = tokenizer.batch_decode(labels, skip_special_tokens=True)\n","   # Some simple post-processing\n","    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n","    \n","    result_rouge = metric_fn(decoded_preds_rouge, decoded_labels_rouge)\n","    \n","#     result_bleu = metric.compute(predictions=decoded_preds, references=decoded_labels)\n","#     result_bleu4 = round(bleu.corpus_score(decoded_preds, decoded_labels).score, 4)\n","#     result_bleu1 = round(bleu1.corpus_score(decoded_preds, decoded_labels).score, 4)\n","#     result_bleu2 = round(bleu2.corpus_score(decoded_preds, decoded_labels).score, 4)\n","#     result_bleu3 = round(bleu3.corpus_score(decoded_preds, decoded_labels).score, 4)\n","    result_bleu1 = bleu_metric1(decoded_preds, decoded_labels)\n","    result_bleu2 = bleu_metric2(decoded_preds, decoded_labels)\n","    result_bleu3 = bleu_metric3(decoded_preds, decoded_labels)\n","    result_bleu4 = bleu_metric4(decoded_preds, decoded_labels)\n","    \n","    meteor_result = meteor.compute(predictions=decoded_preds, references=decoded_labels)\n","    \n","#     prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n","    \n","    result = result_rouge\n","#     result[\"bleu-1\"] = result_bleu1\n","#     result[\"bleu-2\"] = result_bleu2\n","#     result[\"bleu-3\"] = result_bleu3\n","#     result[\"bleu-4\"] = result_bleu4\n","    \n","    result[\"bleu-1\"] = round((torch.FloatTensor(result_bleu1).item() * 100), 4)\n","    result[\"bleu-2\"] = round((torch.FloatTensor(result_bleu2).item() * 100), 4)\n","    result[\"bleu-3\"] = round((torch.FloatTensor(result_bleu3).item() * 100), 4)\n","    result[\"bleu-4\"] = round((torch.FloatTensor(result_bleu4).item() * 100), 4)\n","#     result[\"sacrebleu\"] =  round(result_bleu['score'], 4)\n","\n","    \n","    result[\"meteor\"] = round(meteor_result[\"meteor\"],4)\n","#     metrics_dict = nlgeval.compute_metrics(decoded_preds, decoded_labels)\n","#     result['nlg'] = metrics_dict\n","    \n","#     result[\"gen_len\"] = np.mean(prediction_lens)\n","#     result = {k: round(v,4) for k, v in result.items()}\n","    return result"]},{"cell_type":"markdown","metadata":{"id":"A2cbQnUJSG4i"},"source":["### Set the training arguments"]},{"cell_type":"code","execution_count":46,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:11:46.009363Z","iopub.status.busy":"2022-10-08T17:11:46.008989Z","iopub.status.idle":"2022-10-08T17:11:46.024828Z","shell.execute_reply":"2022-10-08T17:11:46.023828Z","shell.execute_reply.started":"2022-10-08T17:11:46.009330Z"},"id":"ZH-HHDI0IWrX","outputId":"1ced050a-e8c1-4549-d680-35ea19aa58df","trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["PyTorch: setting up devices\n","The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n","Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"]}],"source":["from transformers import set_seed\n","set_seed(42)\n","# set training arguments - Feel free to adapt it\n","training_args = Seq2SeqTrainingArguments(\n","    output_dir=\"squad-bn-mt5-base2\",\n","    per_device_train_batch_size=4,\n","    num_train_epochs=4,\n","    per_device_eval_batch_size=4,\n","    predict_with_generate=True,\n","    evaluation_strategy=\"epoch\",\n","    do_train=True,\n","    do_eval=True,\n","    logging_steps=500,\n","    save_strategy=\"epoch\",\n","#     evaluation_strategy = \"steps\",\n","    #save_steps=1000,\n","    #eval_steps=1000,\n","    overwrite_output_dir=True,\n","    save_total_limit=3,\n","    load_best_model_at_end=True,\n","    push_to_hub=True\n","    #fp16=True, \n",")"]},{"cell_type":"code","execution_count":47,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:11:49.170946Z","iopub.status.busy":"2022-10-08T17:11:49.170574Z","iopub.status.idle":"2022-10-08T17:11:49.176171Z","shell.execute_reply":"2022-10-08T17:11:49.175050Z","shell.execute_reply.started":"2022-10-08T17:11:49.170914Z"},"id":"2tftoCLT3-cE","trusted":true},"outputs":[],"source":["# from transformers import default_data_collator\n","from transformers import AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq,Seq2SeqTrainingArguments, Seq2SeqTrainer\n","# data_collator = default_data_collator\n","data_collator = DataCollatorForSeq2Seq(tokenizer)"]},{"cell_type":"code","execution_count":48,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:11:50.112732Z","iopub.status.busy":"2022-10-08T17:11:50.112356Z","iopub.status.idle":"2022-10-08T17:12:03.034859Z","shell.execute_reply":"2022-10-08T17:12:03.033479Z","shell.execute_reply.started":"2022-10-08T17:11:50.112700Z"},"id":"IAbtaix-KnXC","outputId":"d1586096-cc00-4520-f941-11432319182e","trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["Cloning https://huggingface.co/Tahsin-Mayeesha/squad-bn-mt5-base2 into local empty directory.\n"]}],"source":["# instantiate trainer\n","trainer = Seq2SeqTrainer(\n","    model=model,\n","    args=training_args,\n","    train_dataset=dataset['train'],\n","    eval_dataset=dataset['test'],\n","    # data_collator=T2TDataCollator(),\n","    data_collator=data_collator,\n","    tokenizer=tokenizer,\n","    compute_metrics=compute_metrics\n",")"]},{"cell_type":"code","execution_count":49,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:14:11.179678Z","iopub.status.busy":"2022-10-08T17:14:11.178836Z","iopub.status.idle":"2022-10-08T17:14:11.689652Z","shell.execute_reply":"2022-10-08T17:14:11.688564Z","shell.execute_reply.started":"2022-10-08T17:14:11.179635Z"},"id":"sSC-LgnHV6TH","trusted":true},"outputs":[],"source":["import torch, gc\n","gc.collect()\n","torch.cuda.empty_cache()"]},{"cell_type":"code","execution_count":50,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:14:11.692324Z","iopub.status.busy":"2022-10-08T17:14:11.691722Z","iopub.status.idle":"2022-10-08T17:14:13.788561Z","shell.execute_reply":"2022-10-08T17:14:13.786346Z","shell.execute_reply.started":"2022-10-08T17:14:11.692289Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.7/site-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n","  FutureWarning,\n","***** Running training *****\n","  Num examples = 54150\n","  Num Epochs = 4\n","  Instantaneous batch size per device = 4\n","  Total train batch size (w. parallel, distributed & accumulation) = 4\n","  Gradient Accumulation steps = 1\n","  Total optimization steps = 54152\n"]},{"data":{"text/html":["\n","    <div>\n","      \n","      <progress value='4' max='54152' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [    4/54152 00:01 < 9:17:06, 1.62 it/s, Epoch 0.00/4]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n"," <tr style=\"text-align: left;\">\n","      <th>Epoch</th>\n","      <th>Training Loss</th>\n","      <th>Validation Loss</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table><p>"],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"},{"ename":"RuntimeError","evalue":"CUDA out of memory. Tried to allocate 734.00 MiB (GPU 0; 15.90 GiB total capacity; 13.52 GiB already allocated; 447.75 MiB free; 14.64 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF","output_type":"error","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_17/4032920361.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1411\u001b[0m             \u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1412\u001b[0m             \u001b[0mtrial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1413\u001b[0;31m             \u001b[0mignore_keys_for_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mignore_keys_for_eval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1414\u001b[0m         )\n\u001b[1;32m   1415\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   1649\u001b[0m                         \u001b[0mtr_loss_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1650\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1651\u001b[0;31m                     \u001b[0mtr_loss_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1652\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1653\u001b[0m                 if (\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   2361\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepspeed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2362\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2363\u001b[0;31m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2365\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    361\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 363\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    173\u001b[0m     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[1;32m    174\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 175\u001b[0;31m         allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m def grad(\n","\u001b[0;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 734.00 MiB (GPU 0; 15.90 GiB total capacity; 13.52 GiB already allocated; 447.75 MiB free; 14.64 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"]}],"source":["trainer.train()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"cwwdZwFX_JdJ","outputId":"95490b1b-f6f2-4b89-845a-75adb05e46e5","trusted":true},"outputs":[],"source":["trainer.evaluate(eval_dataset=dataset['test'])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"M4f_eBTCeTHA","outputId":"044297d6-beac-4311-8b5b-419ef56d1523","trusted":true},"outputs":[],"source":["trainer.save_model()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"5d-QcwKnebYY","outputId":"94773f75-f57a-4237-b9e8-869c1716e725","trusted":true},"outputs":[],"source":["# tokenizer.save_pretrained('/content/test2-qgen')\n","tokenizer.save_pretrained('./squad-bn-mt5-base')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"JrBFmJbdW9XV","outputId":"40528aad-7b7a-4eed-b764-4b1b7491c9c3","trusted":true},"outputs":[],"source":["trainer.create_model_card()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"nTn9FjGKo0S7","outputId":"5e4b170e-51af-4436-b7f1-d7c4b763f7a7","trusted":true},"outputs":[],"source":["trainer.push_to_hub()"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-10-06T00:41:33.57423Z","iopub.status.busy":"2022-10-06T00:41:33.573857Z","iopub.status.idle":"2022-10-06T00:41:34.714185Z","shell.execute_reply":"2022-10-06T00:41:34.713216Z","shell.execute_reply.started":"2022-10-06T00:41:33.5742Z"},"trusted":true},"outputs":[],"source":["#import shutil shutil.rmtree(\"./final-squad-bn-qgen-mt5-base-all-metric\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Bqya4Ret3-cM","outputId":"f37a9513-5c31-4c3e-f4e2-0457e42ffbec","trusted":true},"outputs":[],"source":["tokenizer.push_to_hub(\"jannatul17/squad-bn-mt5-base\")"]},{"cell_type":"code","execution_count":51,"metadata":{"execution":{"iopub.execute_input":"2022-10-08T17:15:31.526875Z","iopub.status.busy":"2022-10-08T17:15:31.526455Z","iopub.status.idle":"2022-10-08T17:15:34.401171Z","shell.execute_reply":"2022-10-08T17:15:34.399815Z","shell.execute_reply.started":"2022-10-08T17:15:31.526840Z"},"trusted":true},"outputs":[],"source":[" !pip freeze > '../working/requirement.txt'\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat":4,"nbformat_minor":4}