generate-function-d722a3.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !huggingface-cli login\nfrom huggingface_hub import notebook_login\n\nnotebook_login()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-08T17:30:00.190679Z","iopub.execute_input":"2022-10-08T17:30:00.191224Z","iopub.status.idle":"2022-10-08T17:30:00.419907Z","shell.execute_reply.started":"2022-10-08T17:30:00.191116Z","shell.execute_reply":"2022-10-08T17:30:00.418484Z"},"trusted":true},"execution_count":1,"outputs":[{"output_type":"display_data","data":{"text/plain":"VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"63eba0eb3aa64c6a800aab1adb2e7de7"}},"metadata":{}}]},{"cell_type":"code","source":"!pip install transformers\n!pip install SentencePiece\n!pip install openpyxl\n!pip install xlsxwriter","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:34:27.068040Z","iopub.execute_input":"2022-10-08T17:34:27.069016Z","iopub.status.idle":"2022-10-08T17:35:12.937471Z","shell.execute_reply.started":"2022-10-08T17:34:27.068958Z","shell.execute_reply":"2022-10-08T17:35:12.936065Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"Requirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (4.20.1)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.28.1)\nRequirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.12.1)\nRequirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.8.1)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.64.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.7.1)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (6.0)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from transformers) (4.12.0)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (1.21.6)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (21.3)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2021.11.10)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.3.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=20.0->transformers) (3.0.9)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.8.0)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (3.3)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2022.6.15)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.26.11)\nRequirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.1.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mRequirement already satisfied: SentencePiece in /opt/conda/lib/python3.7/site-packages (0.1.97)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mRequirement already satisfied: openpyxl in /opt/conda/lib/python3.7/site-packages (3.0.10)\nRequirement already satisfied: et-xmlfile in /opt/conda/lib/python3.7/site-packages (from openpyxl) (1.1.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mRequirement already satisfied: xlsxwriter in /opt/conda/lib/python3.7/site-packages (3.0.3)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"! git clone https://github.com/csebuetnlp/xl-sum\n! pip install --upgrade -r ./xl-sum/seq2seq/requirements.txt\n! pip install --upgrade transformers/\n# install rouge module and dependecies\n! pip install -r ./xl-sum/multilingual_rouge_scoring/requirements.txt\n! python -m unidic download # for japanese segmentation\n! pip install --upgrade ./xl-sum/multilingual_rouge_scoring\n! python -m nltk.downloader punkt","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:35:12.940016Z","iopub.execute_input":"2022-10-08T17:35:12.940453Z","iopub.status.idle":"2022-10-08T17:37:02.870564Z","shell.execute_reply.started":"2022-10-08T17:35:12.940404Z","shell.execute_reply":"2022-10-08T17:37:02.868235Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"Cloning into 'xl-sum'...\nremote: Enumerating objects: 1156, done.\u001b[K\nremote: Counting objects: 100% (47/47), done.\u001b[K\nremote: Compressing objects: 100% (33/33), done.\u001b[K\nremote: Total 1156 (delta 19), reused 30 (delta 12), pack-reused 1109\u001b[K\nReceiving objects: 100% (1156/1156), 5.53 MiB | 12.05 MiB/s, done.\nResolving deltas: 100% (309/309), done.\nRequirement already satisfied: tensorboard in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.10.0)\nCollecting tensorboard\n  Downloading tensorboard-2.10.1-py3-none-any.whl (5.9 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 2)) (1.0.2)\nCollecting seqeval\n  Downloading seqeval-1.2.2.tar.gz (43 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: psutil in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 4)) (5.9.1)\nCollecting psutil\n  Downloading psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (281 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.3/281.3 kB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting sacrebleu\n  Downloading sacrebleu-2.2.1-py3-none-any.whl (116 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.9/116.9 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: sentencepiece!=0.1.92 in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 6)) (0.1.97)\nRequirement already satisfied: protobuf in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/seq2seq/requirements.txt (line 7)) (3.19.4)\nCollecting protobuf\n  Downloading protobuf-4.21.7-cp37-abi3-manylinux2014_x86_64.whl (408 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m408.4/408.4 kB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.4.6)\nRequirement already satisfied: setuptools>=41.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (59.8.0)\nRequirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.6.1)\nRequirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.8.1)\nRequirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.3.7)\nRequirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.35.0)\nRequirement already satisfied: grpcio>=1.24.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.43.0)\nRequirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.28.1)\nRequirement already satisfied: numpy>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.21.6)\nRequirement already satisfied: wheel>=0.26 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.37.1)\nRequirement already satisfied: absl-py>=0.4 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.15.0)\nRequirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.2.2)\n  Downloading protobuf-3.19.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m40.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->-r ./xl-sum/seq2seq/requirements.txt (line 2)) (1.0.1)\nRequirement already satisfied: scipy>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->-r ./xl-sum/seq2seq/requirements.txt (line 2)) (1.7.3)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->-r ./xl-sum/seq2seq/requirements.txt (line 2)) (3.1.0)\nRequirement already satisfied: lxml in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (4.9.1)\nRequirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (0.8.10)\nRequirement already satisfied: regex in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (2021.11.10)\nRequirement already satisfied: portalocker in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (2.5.1)\nRequirement already satisfied: colorama in /opt/conda/lib/python3.7/site-packages (from sacrebleu->-r ./xl-sum/seq2seq/requirements.txt (line 5)) (0.4.5)\nRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from absl-py>=0.4->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.15.0)\nRequirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.2.7)\nRequirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.8)\nRequirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.2.4)\nRequirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.3.1)\nRequirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.12.0)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.3)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2022.6.15)\nRequirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.1.0)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (1.26.11)\nRequirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.7/site-packages (from werkzeug>=1.0.1->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (2.1.1)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.8.0)\nRequirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (4.3.0)\nRequirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (0.4.8)\nRequirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r ./xl-sum/seq2seq/requirements.txt (line 1)) (3.2.0)\nBuilding wheels for collected packages: seqeval\n  Building wheel for seqeval (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16180 sha256=a0a3e9237a8a3a9173bcdf2b84135ac81f94383b03e9ac32ab9d4a6e00ca282a\n  Stored in directory: /root/.cache/pip/wheels/05/96/ee/7cac4e74f3b19e3158dce26a20a1c86b3533c43ec72a549fd7\nSuccessfully built seqeval\nInstalling collected packages: sacrebleu, psutil, protobuf, seqeval, tensorboard\n  Attempting uninstall: psutil\n    Found existing installation: psutil 5.9.1\n    Uninstalling psutil-5.9.1:\n      Successfully uninstalled psutil-5.9.1\n  Attempting uninstall: protobuf\n    Found existing installation: protobuf 3.19.4\n    Uninstalling protobuf-3.19.4:\n      Successfully uninstalled protobuf-3.19.4\n  Attempting uninstall: tensorboard\n    Found existing installation: tensorboard 2.10.0\n    Uninstalling tensorboard-2.10.0:\n      Successfully uninstalled tensorboard-2.10.0\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ntensorflow-io 0.21.0 requires tensorflow-io-gcs-filesystem==0.21.0, which is not installed.\nbeatrix-jupyterlab 3.1.7 requires google-cloud-bigquery-storage, which is not installed.\ntfx-bsl 1.9.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\ntfx-bsl 1.9.0 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<3,>=1.15.5, but you have tensorflow 2.6.4 which is incompatible.\ntensorflow 2.6.4 requires h5py~=3.1.0, but you have h5py 3.7.0 which is incompatible.\ntensorflow 2.6.4 requires numpy~=1.19.2, but you have numpy 1.21.6 which is incompatible.\ntensorflow 2.6.4 requires tensorboard<2.7,>=2.6.0, but you have tensorboard 2.10.1 which is incompatible.\ntensorflow 2.6.4 requires typing-extensions<3.11,>=3.7, but you have typing-extensions 4.3.0 which is incompatible.\ntensorflow-transform 1.9.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\ntensorflow-transform 1.9.0 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,<2.10,>=1.15.5, but you have tensorflow 2.6.4 which is incompatible.\ntensorflow-serving-api 2.9.0 requires tensorflow<3,>=2.9.0, but you have tensorflow 2.6.4 which is incompatible.\npdpbox 0.2.1 requires matplotlib==3.1.1, but you have matplotlib 3.5.3 which is incompatible.\nnnabla 1.29.0 requires protobuf<=3.19.4; platform_system != \"Windows\", but you have protobuf 3.19.6 which is incompatible.\ngrpcio-status 1.47.0 requires grpcio>=1.47.0, but you have grpcio 1.43.0 which is incompatible.\ngcsfs 2022.5.0 requires fsspec==2022.5.0, but you have fsspec 2022.7.1 which is incompatible.\napache-beam 2.40.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.5.1 which is incompatible.\napache-beam 2.40.0 requires pyarrow<8.0.0,>=0.15.1, but you have pyarrow 8.0.0 which is incompatible.\nallennlp 2.10.0 requires protobuf==3.20.0, but you have protobuf 3.19.6 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed protobuf-3.19.6 psutil-5.9.2 sacrebleu-2.2.1 seqeval-1.2.2 tensorboard-2.10.1\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\u001b[31mERROR: Invalid requirement: 'transformers/'\nHint: It looks like a path. File 'transformers/' does not exist.\u001b[0m\u001b[31m\n\u001b[0mCollecting git+https://github.com/otuncelli/turkish-stemmer-python (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 1))\n  Cloning https://github.com/otuncelli/turkish-stemmer-python to /tmp/pip-req-build-ycw7lbiy\n  Running command git clone --filter=blob:none --quiet https://github.com/otuncelli/turkish-stemmer-python /tmp/pip-req-build-ycw7lbiy\n  Resolved https://github.com/otuncelli/turkish-stemmer-python to commit 0c22380bf84a5ab1f219f4a905274c78afa04ed1\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hCollecting git+https://github.com/abhik1505040/bengali-stemmer (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 2))\n  Cloning https://github.com/abhik1505040/bengali-stemmer to /tmp/pip-req-build-6r7z4kbr\n  Running command git clone --filter=blob:none --quiet https://github.com/abhik1505040/bengali-stemmer /tmp/pip-req-build-6r7z4kbr\n  Resolved https://github.com/abhik1505040/bengali-stemmer to commit 375186caee8e50e3260dd6bc02d20d50277f3e39\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: absl-py in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 3)) (0.15.0)\nRequirement already satisfied: nltk in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (3.7)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 5)) (1.21.6)\nRequirement already satisfied: six>=1.14 in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 6)) (1.15.0)\nCollecting pythainlp\n  Downloading pythainlp-3.1.0-py3-none-any.whl (9.6 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting pyonmttok\n  Downloading pyonmttok-1.34.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.9 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.9/16.9 MB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: jieba in /opt/conda/lib/python3.7/site-packages (from -r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 9)) (0.42.1)\nCollecting fugashi[unidic]\n  Downloading fugashi-1.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (583 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m583.5/583.5 kB\u001b[0m \u001b[31m29.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (2021.11.10)\nRequirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (8.0.4)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (1.0.1)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (4.64.0)\nRequirement already satisfied: requests>=2.22.0 in /opt/conda/lib/python3.7/site-packages (from pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (2.28.1)\nCollecting unidic\n  Downloading unidic-1.1.0.tar.gz (7.7 kB)\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (2.1.0)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (2022.6.15)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (3.3)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests>=2.22.0->pythainlp->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 7)) (1.26.11)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from click->nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (4.12.0)\nRequirement already satisfied: wasabi<1.0.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from unidic->fugashi[unidic]->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 10)) (0.10.1)\nCollecting plac<2.0.0,>=1.1.3\n  Downloading plac-1.3.5-py2.py3-none-any.whl (22 kB)\nRequirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (4.3.0)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->-r ./xl-sum/multilingual_rouge_scoring/requirements.txt (line 4)) (3.8.0)\nBuilding wheels for collected packages: TurkishStemmer, bengali-stemmer, unidic\n  Building wheel for TurkishStemmer (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for TurkishStemmer: filename=TurkishStemmer-1.3-py3-none-any.whl size=19871 sha256=6db51b83cddd2cfb0cd8cb3864ccb34a31cc1dcb3f40857e427de7bb88909525\n  Stored in directory: /tmp/pip-ephem-wheel-cache-3n9vembu/wheels/ee/65/db/c127b9a272f949d5a421a9b7b43128aa8b1d143bafa52f10d1\n  Building wheel for bengali-stemmer (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for bengali-stemmer: filename=bengali_stemmer-0.0.1-py2.py3-none-any.whl size=6408 sha256=a9a5a96cfb3a0da2eaedebfd682e155100d9bba7223ff0594f622e8a454da719\n  Stored in directory: /tmp/pip-ephem-wheel-cache-3n9vembu/wheels/6a/f4/ee/9298bdab6928b70071ca2876ff3950a7d4adb36ed489be77d2\n  Building wheel for unidic (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for unidic: filename=unidic-1.1.0-py3-none-any.whl size=7426 sha256=8484f3989237472d7ea315c03b8738be8eb53cf9854dc878ae446e5b6e379a60\n  Stored in directory: /root/.cache/pip/wheels/ce/4d/f1/170bb74b559ca338113c0315c9805e16dfd0a12411ec6b1122\nSuccessfully built TurkishStemmer bengali-stemmer unidic\nInstalling collected packages: TurkishStemmer, plac, bengali-stemmer, pyonmttok, fugashi, unidic, pythainlp\nSuccessfully installed TurkishStemmer-1.3 bengali-stemmer-0.0.1 fugashi-1.2.0 plac-1.3.5 pyonmttok-1.34.0 pythainlp-3.1.0 unidic-1.1.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mdownload url: https://cotonoha-dic.s3-ap-northeast-1.amazonaws.com/unidic-3.1.0.zip\nDictionary version: 3.1.0+2021-08-31\nDownloading UniDic v3.1.0+2021-08-31...\nunidic-3.1.0.zip: 100%|██████████████████████| 526M/526M [00:19<00:00, 26.9MB/s]\nFinished download.\nDownloaded UniDic v3.1.0+2021-08-31 to /opt/conda/lib/python3.7/site-packages/unidic/dicdir\nProcessing ./xl-sum/multilingual_rouge_scoring\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: absl-py in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (0.15.0)\nRequirement already satisfied: nltk in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (3.7)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (1.21.6)\nRequirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.7/site-packages (from rouge-score==0.0.0) (1.15.0)\nRequirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (2021.11.10)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (1.0.1)\nRequirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (8.0.4)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from nltk->rouge-score==0.0.0) (4.64.0)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from click->nltk->rouge-score==0.0.0) (4.12.0)\nRequirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->rouge-score==0.0.0) (4.3.0)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click->nltk->rouge-score==0.0.0) (3.8.0)\nBuilding wheels for collected packages: rouge-score\n  Building wheel for rouge-score (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for rouge-score: filename=rouge_score-0.0.0-py3-none-any.whl size=18182 sha256=df173caec1acc35c1acf28862dc23331c8ad6f7de363cf9efe5745b777116400\n  Stored in directory: /tmp/pip-ephem-wheel-cache-jcd7rjvl/wheels/4f/07/35/339fd97dbcdebcdd6421137bf0930ce61a48e307e357213aa0\nSuccessfully built rouge-score\nInstalling collected packages: rouge-score\nSuccessfully installed rouge-score-0.0.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m/opt/conda/lib/python3.7/runpy.py:125: RuntimeWarning: 'nltk.downloader' found in sys.modules after import of package 'nltk', but prior to execution of 'nltk.downloader'; this may result in unpredictable behaviour\n  warn(RuntimeWarning(msg))\n[nltk_data] Downloading package punkt to /usr/share/nltk_data...\n[nltk_data]   Package punkt is already up-to-date!\n","output_type":"stream"}]},{"cell_type":"code","source":"import datasets\nimport nltk\nimport numpy as np\nimport logging\n\nlogger = logging.getLogger(__name__)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:37:14.843893Z","iopub.execute_input":"2022-10-08T17:37:14.846390Z","iopub.status.idle":"2022-10-08T17:37:14.853132Z","shell.execute_reply.started":"2022-10-08T17:37:14.846312Z","shell.execute_reply":"2022-10-08T17:37:14.851645Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"CKPT = 'jannatul17/squad-bn-qgen-banglat5'\nfrom transformers import AutoTokenizer, T5ForConditionalGeneration\nfrom datasets import load_dataset\ntokenizer_test = AutoTokenizer.from_pretrained(CKPT, use_auth_token=False)\nmodel_test = T5ForConditionalGeneration.from_pretrained(CKPT, use_auth_token=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:37:42.729734Z","iopub.execute_input":"2022-10-08T17:37:42.730301Z","iopub.status.idle":"2022-10-08T17:38:21.829937Z","shell.execute_reply.started":"2022-10-08T17:37:42.730265Z","shell.execute_reply":"2022-10-08T17:38:21.828317Z"},"trusted":true},"execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/2.49k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"22b45e55d94349c3837777c9c2375285"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/1.06M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2300bc350c3f45978bce1e468c8286a3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/2.62M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6624bace59f74062a633ba9e1c904f29"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/2.15k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f710c20d418847709ebe2e33c00cc22c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/798 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"61d3f96cddf74991ad2e43a8941b4297"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/945M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"be515633b0804a09b72d02625e2e4a2b"}},"metadata":{}}]},{"cell_type":"markdown","source":"# Getting Dataset","metadata":{}},{"cell_type":"code","source":"from datasets import concatenate_datasets, load_dataset\n\ntrain_data = load_dataset(\"csebuetnlp/squad_bn\", split=\"train\")\nvalid_data = load_dataset(\"csebuetnlp/squad_bn\", split=\"validation\")\ntest_data = load_dataset(\"csebuetnlp/squad_bn\", split=\"test\")\n\nconcat_dataset = concatenate_datasets([train_data, valid_data, test_data])\ndataset=concat_dataset","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:41:10.628726Z","iopub.execute_input":"2022-10-08T17:41:10.630231Z","iopub.status.idle":"2022-10-08T17:41:39.100234Z","shell.execute_reply.started":"2022-10-08T17:41:10.630163Z","shell.execute_reply":"2022-10-08T17:41:39.098975Z"},"trusted":true},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/4.20k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b8d0489d194f45688ec31a3b5147f5f1"}},"metadata":{}},{"name":"stdout","text":"Downloading and preparing dataset squad_bn/squad_bn to /root/.cache/huggingface/datasets/csebuetnlp___squad_bn/squad_bn/0.0.1/17a6d6abc976f299afda17ca9b5ce08a022ecafabe24b3362e16a3093c32df4b...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading data:   0%|          | 0.00/8.43M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"320f4b6e21b6455286d1950d48879107"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"name":"stdout","text":"Dataset squad_bn downloaded and prepared to /root/.cache/huggingface/datasets/csebuetnlp___squad_bn/squad_bn/0.0.1/17a6d6abc976f299afda17ca9b5ce08a022ecafabe24b3362e16a3093c32df4b. Subsequent calls will reuse this data.\n","output_type":"stream"}]},{"cell_type":"code","source":"del concat_dataset, train_data, valid_data, test_data","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:41:40.368945Z","iopub.execute_input":"2022-10-08T17:41:40.369442Z","iopub.status.idle":"2022-10-08T17:41:40.376795Z","shell.execute_reply.started":"2022-10-08T17:41:40.369405Z","shell.execute_reply":"2022-10-08T17:41:40.375103Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"\nsmall_dataset= dataset.filter(lambda example: len(example['answers']['text'])>0)\nsmall_dataset= small_dataset.filter(lambda example: len(example['context'])>300)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:41:41.665329Z","iopub.execute_input":"2022-10-08T17:41:41.666060Z","iopub.status.idle":"2022-10-08T17:41:49.840790Z","shell.execute_reply.started":"2022-10-08T17:41:41.666021Z","shell.execute_reply":"2022-10-08T17:41:49.839849Z"},"trusted":true},"execution_count":10,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/124 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2ba461f443db40188d33317ab6895c40"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/72 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"62eea335da51499a857110d0abe4a898"}},"metadata":{}}]},{"cell_type":"code","source":"split_data = small_dataset.train_test_split(test_size=0.2, seed= None, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:42:09.565389Z","iopub.execute_input":"2022-10-08T17:42:09.565889Z","iopub.status.idle":"2022-10-08T17:42:09.604974Z","shell.execute_reply.started":"2022-10-08T17:42:09.565851Z","shell.execute_reply":"2022-10-08T17:42:09.603494Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"dataset = split_data","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:42:19.414168Z","iopub.execute_input":"2022-10-08T17:42:19.414602Z","iopub.status.idle":"2022-10-08T17:42:19.420631Z","shell.execute_reply.started":"2022-10-08T17:42:19.414570Z","shell.execute_reply":"2022-10-08T17:42:19.419289Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"del split_data","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:42:27.145797Z","iopub.execute_input":"2022-10-08T17:42:27.146267Z","iopub.status.idle":"2022-10-08T17:42:27.152724Z","shell.execute_reply.started":"2022-10-08T17:42:27.146221Z","shell.execute_reply":"2022-10-08T17:42:27.151559Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"def format_dataset(example):\n  \n        return {'input': 'answer: ' + example['answers']['text'][0] + ' context: ' + example['context'], 'target': example['question']}","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:42:40.311033Z","iopub.execute_input":"2022-10-08T17:42:40.311457Z","iopub.status.idle":"2022-10-08T17:42:40.318972Z","shell.execute_reply.started":"2022-10-08T17:42:40.311421Z","shell.execute_reply":"2022-10-08T17:42:40.317216Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"#test_data = load_dataset('csebuetnlp/squad_bn', split='test')\n\n# small_dataset= test_data.filter(lambda example: len(example['answers']['text'])>0)\n# small_dataset= small_dataset.filter(lambda example: len(example['context'])>300)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T14:05:53.963912Z","iopub.execute_input":"2022-09-27T14:05:53.965262Z","iopub.status.idle":"2022-09-27T14:06:17.519012Z","shell.execute_reply.started":"2022-09-27T14:05:53.965215Z","shell.execute_reply":"2022-09-27T14:06:17.517498Z"},"trusted":true},"execution_count":6,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/4.20k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6c2999b5c0e24bf4aeeb821d4dfb1d2b"}},"metadata":{}},{"name":"stdout","text":"Downloading and preparing dataset squad_bn/squad_bn to /root/.cache/huggingface/datasets/csebuetnlp___squad_bn/squad_bn/0.0.1/17a6d6abc976f299afda17ca9b5ce08a022ecafabe24b3362e16a3093c32df4b...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading data:   0%|          | 0.00/8.43M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6311e470a3ff4036b42078b8bcf10dd4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"name":"stdout","text":"Dataset squad_bn downloaded and prepared to /root/.cache/huggingface/datasets/csebuetnlp___squad_bn/squad_bn/0.0.1/17a6d6abc976f299afda17ca9b5ce08a022ecafabe24b3362e16a3093c32df4b. Subsequent calls will reuse this data.\n","output_type":"stream"}]},{"cell_type":"code","source":"def format_dataset(example):\n    if (len(example['answers']['text'])>0):\n        return {'input': 'answer: ' + example['answers']['text'][0] + ' context: ' + example['context'], 'target': example['question']}","metadata":{"execution":{"iopub.status.busy":"2022-09-27T14:06:17.520744Z","iopub.execute_input":"2022-09-27T14:06:17.521107Z","iopub.status.idle":"2022-09-27T14:06:17.528218Z","shell.execute_reply.started":"2022-09-27T14:06:17.521076Z","shell.execute_reply":"2022-09-27T14:06:17.526815Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"dataset = dataset.map(format_dataset, remove_columns=dataset['train'].column_names)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:42:59.401147Z","iopub.execute_input":"2022-10-08T17:42:59.401645Z","iopub.status.idle":"2022-10-08T17:43:13.130821Z","shell.execute_reply.started":"2022-10-08T17:42:59.401608Z","shell.execute_reply":"2022-10-08T17:43:13.129943Z"},"trusted":true},"execution_count":15,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/54150 [00:00<?, ?ex/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"93ba631f459e48ac98477cf98648162b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/13538 [00:00<?, ?ex/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"23a3fe877ff6400581d6697da4ac0b6e"}},"metadata":{}}]},{"cell_type":"code","source":"test_data = dataset['test']","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:43:56.795249Z","iopub.execute_input":"2022-10-08T17:43:56.795712Z","iopub.status.idle":"2022-10-08T17:43:56.802701Z","shell.execute_reply.started":"2022-10-08T17:43:56.795676Z","shell.execute_reply":"2022-10-08T17:43:56.800950Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"#dataset = test_data.map(format_dataset, remove_columns=test_data.column_names)\n\n# # dataset = test_data.map(format_dataset)\n#test_data['answers'][6]['text']\n#dataset[0]\n#small_dataset= dataset.filter(lambda example: len(example['answers']['text'])>0)\n# small_dataset= small_dataset.filter(lambda example: len(example['context'])>300)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:43:35.105579Z","iopub.execute_input":"2022-10-08T17:43:35.106092Z","iopub.status.idle":"2022-10-08T17:43:35.113016Z","shell.execute_reply.started":"2022-10-08T17:43:35.106056Z","shell.execute_reply":"2022-10-08T17:43:35.111553Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"len(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:44:09.965156Z","iopub.execute_input":"2022-10-08T17:44:09.965615Z","iopub.status.idle":"2022-10-08T17:44:09.973961Z","shell.execute_reply.started":"2022-10-08T17:44:09.965563Z","shell.execute_reply":"2022-10-08T17:44:09.972071Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"13538"},"metadata":{}}]},{"cell_type":"code","source":"test_data[0]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:44:44.574051Z","iopub.execute_input":"2022-10-08T17:44:44.574501Z","iopub.status.idle":"2022-10-08T17:44:44.584609Z","shell.execute_reply.started":"2022-10-08T17:44:44.574466Z","shell.execute_reply":"2022-10-08T17:44:44.583106Z"},"trusted":true},"execution_count":22,"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"{'input': 'answer: ১৯৮৭ context: ফলস্বরূপ, ১৯৭৯ সালে, সনি এবং ফিলিপস একটি নতুন ডিজিটাল অডিও ডিস্ক ডিজাইন করার জন্য প্রকৌশলীদের একটি যৌথ টাস্ক ফোর্স গঠন করে। ইঞ্জিনিয়ার কিস শুহামার ইমমিনক এবং তোশিতাদা দোই এর নেতৃত্বে, গবেষণাটি লেজার এবং অপটিক্যাল ডিস্ক প্রযুক্তিকে এগিয়ে নিয়ে যায়। এক বছর পরীক্ষা-নিরীক্ষা ও আলোচনার পর টাস্ক ফোর্স রেড বুক সিডি-ডিএ স্ট্যান্ডার্ড তৈরি করে। প্রথম প্রকাশিত হয় ১৯৮০ সালে। আইইসি কর্তৃক ১৯৮৭ সালে আন্তর্জাতিক মান হিসেবে আনুষ্ঠানিকভাবে এই মান গৃহীত হয় এবং ১৯৯৬ সালে বিভিন্ন সংশোধনী মানের অংশ হয়ে ওঠে।',\n 'target': 'কখন আইইসি রেড বুক সিডি-ডিএকে একটি আন্তর্জাতিক মান হিসাবে চালু করেছিল?'}"},"metadata":{}}]},{"cell_type":"code","source":"# import torch\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model_test.to(device)\n# # To manage one sentence\n# prompt = \"answer: Pseudozizeeria maha context: ধুপি (বৈজ্ঞানিক নাম: Pseudozizeeria maha(Kollar))[1] এক প্রজাতির ছোট আকারের প্রজাপতি যার শরীর ও ডানা হালকা ধূসর-খয়েরি বর্ণের এবং বিন্দু দেখা যায়। এরা 'লাইসিনিডি' গোত্রের এবং 'পলিয়োম্যাটিনি' উপগোত্রের সদস্য।\"\n# input_ids = tokenizer_test(prompt, return_tensors=\"pt\").input_ids.to(device)\n# generated_ids = model_test.generate(input_ids,\n#                                     num_beams=7,\n#                                     num_return_sequences=7,\n# #                                     top_k=10,\n# #                                     top_p=.2,\n# #                                     temperature=0.95,\n                                    \n#                                     max_length=200)\n# for generated in generated_ids:\n#     generated_text = tokenizer_test.decode(generated,skip_special_tokens=True)\n# print(generated_text)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T19:30:01.218184Z","iopub.execute_input":"2022-09-15T19:30:01.218655Z","iopub.status.idle":"2022-09-15T19:30:02.226942Z","shell.execute_reply.started":"2022-09-15T19:30:01.218618Z","shell.execute_reply":"2022-09-15T19:30:02.225811Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"থুপির বৈজ্ঞানিক নাম কি?\n","output_type":"stream"}]},{"cell_type":"code","source":"# Function for handling multiple sentences\ndef custom_gen(model, tokenizer, input_text, max_length=50, do_sample=False, temperature=None, num_beams=None, top_k=None, top_p=None, early_stopping=True, num_return_sequences=1):\n               input_ids = tokenizer(input_text, return_tensors=\"pt\").input_ids.to(device)\n               generated_ids = model.generate(input_ids, max_length=max_length, do_sample=do_sample, top_k=top_k, temperature=temperature, num_beams=num_beams)\n#generated_text = tokenizer.decode(generated_ids[0], skip_special_tokens=True)\n               return generated_ids","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:45:11.485544Z","iopub.execute_input":"2022-10-08T17:45:11.486081Z","iopub.status.idle":"2022-10-08T17:45:11.495217Z","shell.execute_reply.started":"2022-10-08T17:45:11.486041Z","shell.execute_reply":"2022-10-08T17:45:11.493823Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"input_data = []\nreference_question = []\nanswer = []\ncontext = []\n#for i in range(0, len(small_dataset)):\nfor i in range(0,5):\n    input_data.append('answer: ' + small_dataset[i]['answers']['text'][0]  + ' context: ' + small_dataset[i]['context'])\n    reference_question.append(small_dataset[i]['question'])\n    answer.append(small_dataset[i]['answers']['text'][0])\n    context.append(small_dataset[i]['context'])","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:46:01.137512Z","iopub.execute_input":"2022-10-08T17:46:01.138689Z","iopub.status.idle":"2022-10-08T17:46:01.154137Z","shell.execute_reply.started":"2022-10-08T17:46:01.138621Z","shell.execute_reply":"2022-10-08T17:46:01.152771Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"reference_question","metadata":{"execution":{"iopub.status.busy":"2022-10-08T17:46:23.435124Z","iopub.execute_input":"2022-10-08T17:46:23.436659Z","iopub.status.idle":"2022-10-08T17:46:23.443649Z","shell.execute_reply.started":"2022-10-08T17:46:23.436605Z","shell.execute_reply":"2022-10-08T17:46:23.442772Z"},"trusted":true},"execution_count":27,"outputs":[{"execution_count":27,"output_type":"execute_result","data":{"text/plain":"['কোন দেশে নরম্যান্ডি অবস্থিত?',\n 'নরমান্ডিতে নরম্যানরা কখন ছিল?',\n 'কোন দেশ থেকে নর্সের উৎপত্তি হয়েছিল?',\n 'নর্স নেতা কে ছিলেন?',\n 'নরম্যানরা প্রথম কোন শতাব্দীতে তাদের আলাদা পরিচয় লাভ করেছিল?']"},"metadata":{}}]},{"cell_type":"code","source":"# ('answer: ' + small_dataset[0]['answers']['text'][0]  + ' context: ' + small_dataset[0]['context'])\n# input_data","metadata":{"execution":{"iopub.status.busy":"2022-09-16T08:57:49.977646Z","iopub.execute_input":"2022-09-16T08:57:49.978126Z","iopub.status.idle":"2022-09-16T08:57:49.983328Z","shell.execute_reply.started":"2022-09-16T08:57:49.97808Z","shell.execute_reply":"2022-09-16T08:57:49.981922Z"},"trusted":true},"execution_count":43,"outputs":[]},{"cell_type":"code","source":"import torch\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel_test.to(device)\n\n# Using the custom_gen. Will have to change for our code.\nbn_pred_ques = {}\nall_generated_ids = []\nfor prompt in input_data:\n#     print(prompt)\n    generated_ids = custom_gen(model_test,tokenizer_test,prompt,max_length=50,do_sample=True,top_k=40, top_p=0.95,num_return_sequences=20)\n    all_generated_ids.append(generated_ids)\npred = []\npred_ques =[]\nfor text in all_generated_ids:\n    for generated in text:\n        generated_text = tokenizer_test.decode(generated,skip_special_tokens=True)\n#         print(generated_text)\n        pred.append(generated_text)\n        bn_pred_ques[prompt] = pred\n#     pred_ques.append(pred)\n# print(pred)\n# for generated in generated_ids:\n#     generated_text = tokenizer_test.decode(generated,skip_special_tokens=True)\n#     print(generated_text)\n#     pred.append(generated_text)\n#     bn_pred_ques[prompt] = pred\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-27T14:06:19.06705Z","iopub.execute_input":"2022-09-27T14:06:19.067445Z","iopub.status.idle":"2022-09-27T14:23:09.124543Z","shell.execute_reply.started":"2022-09-27T14:06:19.067411Z","shell.execute_reply":"2022-09-27T14:23:09.123015Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stderr","text":"Token indices sequence length is longer than the specified maximum sequence length for this model (537 > 512). Running this sequence through the model will result in indexing errors\n","output_type":"stream"}]},{"cell_type":"code","source":"print(pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(reference_question)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from rouge_score import rouge_scorer, scoring\n\ndef add_newline_to_end_of_each_sentence(x):\n        return \"\\n\".join(nltk.sent_tokenize(x))\n\n\n\ndef extract_rouge_mid_statistics(dct):\n        new_dict = {}\n        for k1, v1 in dct.items():\n            mid = v1.mid\n            new_dict[k1] = {stat: round(getattr(mid, stat), 4) for stat in [\"precision\", \"recall\", \"fmeasure\"]}\n        return new_dict\n\n\ndef calculate_rouge(\n        pred_lns,\n        tgt_lns,\n        use_stemmer=True,\n        rouge_keys=[\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"],\n        return_precision_and_recall=False,\n        bootstrap_aggregation=True,\n        newline_sep=True,\n       rouge_lang='bengali'\n    ):\n        \n        logger.info(\"Rouge lang: \" + str(rouge_lang))\n        scorer = rouge_scorer.RougeScorer(\n            rouge_keys, lang=rouge_lang,\n            use_stemmer=use_stemmer\n        )\n        aggregator = scoring.BootstrapAggregator()\n        for pred, tgt in zip(tgt_lns, pred_lns):\n            # rougeLsum expects \"\\n\" separated sentences within a summary\n            if newline_sep:\n                pred = add_newline_to_end_of_each_sentence(pred)\n                tgt = add_newline_to_end_of_each_sentence(tgt)\n            scores = scorer.score(pred, tgt)\n            aggregator.add_scores(scores)\n\n        if bootstrap_aggregation:\n            result = aggregator.aggregate()\n            if return_precision_and_recall:\n                return extract_rouge_mid_statistics(result)  # here we return dict\n            else:\n                results_precision = {k: round(v.mid.precision * 100, 4) for k, v in result.items()}\n                results_recall = {k: round(v.mid.recall * 100, 4) for k, v in result.items()}\n                results_fmeasure = {k: round(v.mid.fmeasure * 100, 4) for k, v in result.items()}\n            \n            return {\n                    \"rouge1_precision\": results_precision.get('rouge1'),\n                    \"rouge1_recall\": results_recall.get('rouge1'),\n                    \"rouge1_fmeasure\": results_fmeasure.get('rouge1'),\n\n                    \"rouge2_precision\": results_precision.get('rouge2'),\n                    \"rouge2_recall\": results_recall.get('rouge2'),\n                    \"rouge2_fmeasure\": results_fmeasure.get('rouge2'),\n\n                    \"rougeL_precision\": results_precision.get('rougeL'),\n                    \"rougeL_recall\": results_recall.get('rougeL'),\n                    \"rougeL_fmeasure\": results_fmeasure.get('rougeL'),\n                \n                    \"rougeLsum_precision\": results_precision.get('rougeLsum'),\n                    \"rougeLsum_recall\": results_recall.get('rougeLsum'),\n                    \"rougeLsum_fmeasure\": results_fmeasure.get('rougeLsum'),\n                }\n#                 return {k: round(v.mid.fmeasure * 100, 4) for k, v in result.items()}\n\n        else:\n            return aggregator._scores  # here we return defaultdict(list)\n\n        \nmetric_fn = calculate_rouge\nimport numpy as np\nfrom datasets import load_metric\nmetric = load_metric(\"sacrebleu\")\nmeteor = load_metric('meteor')\n\ndef postprocess_text(preds, labels):\n    preds = [pred.strip() for pred in preds]\n    labels = [[label.strip()] for label in labels]\n    return preds, labels","metadata":{"execution":{"iopub.status.busy":"2022-09-27T14:34:57.730284Z","iopub.execute_input":"2022-09-27T14:34:57.731721Z","iopub.status.idle":"2022-09-27T14:34:59.990809Z","shell.execute_reply.started":"2022-09-27T14:34:57.731669Z","shell.execute_reply":"2022-09-27T14:34:59.989493Z"},"trusted":true},"execution_count":13,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/2.85k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"88456e04d30241d9b5dc4428a9988443"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/2.20k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"93bdd340e74040f2a31372b386dc9720"}},"metadata":{}},{"name":"stderr","text":"[nltk_data] Downloading package wordnet to /usr/share/nltk_data...\n[nltk_data]   Package wordnet is already up-to-date!\n[nltk_data] Downloading package punkt to /usr/share/nltk_data...\n[nltk_data]   Package punkt is already up-to-date!\n[nltk_data] Downloading package omw-1.4 to /usr/share/nltk_data...\n","output_type":"stream"}]},{"cell_type":"code","source":"def compute_metrics(ref, pred):\n#     preds, labels = eval_preds\n#     if isinstance(preds, tuple):\n#         preds = preds[0]\n        \n#     # Replace -100 in the labels as we can't decode them.\n#     labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    \n    \n#     decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n#     decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n    \n    \n#     decoded_preds_rouge = tokenizer.batch_decode(preds, skip_special_tokens=True)\n#     decoded_labels_rouge = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n#    # Some simple post-processing\n    decoded_preds, decoded_labels = postprocess_text(pred, ref)\n    \n    result_rouge = metric_fn(ref, pred)\n    \n    result_bleu = metric.compute(predictions=decoded_preds, references=decoded_labels)\n    meteor_result = meteor.compute(predictions=decoded_preds, references=decoded_labels)\n#     prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n    \n    result = result_rouge\n    result[\"sacrebleu\"] = round(result_bleu['score'], 4)\n    \n    result[\"meteor\"] = round(meteor_result[\"meteor\"],4)\n    \n#     result[\"gen_len\"] = np.mean(prediction_lens)\n#     result = {k: round(v,4) for k, v in result.items()}\n    return result","metadata":{"execution":{"iopub.status.busy":"2022-09-27T14:35:40.511085Z","iopub.execute_input":"2022-09-27T14:35:40.51159Z","iopub.status.idle":"2022-09-27T14:35:40.521047Z","shell.execute_reply.started":"2022-09-27T14:35:40.51155Z","shell.execute_reply":"2022-09-27T14:35:40.519584Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T14:35:45.908078Z","iopub.execute_input":"2022-09-27T14:35:45.908586Z","iopub.status.idle":"2022-09-27T14:35:51.498115Z","shell.execute_reply.started":"2022-09-27T14:35:45.908543Z","shell.execute_reply":"2022-09-27T14:35:51.496711Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n## convert your array into a dataframe\n# df = pd.DataFrame (pred)\ndf = pd.DataFrame({'Reference_question':reference_question, 'Predicted_question':pred, 'Answer': answer, 'Context': context})\ndf_result = pd.DataFrame({'Rouge1_Precision': [result['rouge1_precision']], 'Rouge1_Recall': [result['rouge1_recall']],\n                         'Rouge1_fmeasure': [result['rouge1_fmeasure']],\n                         'Rouge2_Precision': [result['rouge2_precision']], 'Rouge2_Recall': [result['rouge2_recall']],\n                         'Rouge2_fmeasure': [result['rouge2_fmeasure']],\n                         'RougeL_Precision': [result['rougeL_precision']], 'RougeL_Recall': [result['rougeL_recall']],\n                         'RougeL_fmeasure': [result['rougeL_fmeasure']],\n                         'RougeLsum_Precision': [result['rougeLsum_precision']], 'RougeLsum_Recall': [result['rougeLsum_recall']],\n                         'RougeLsum_fmeasure': [result['rougeLsum_fmeasure']],\n                         'SacreBleu': [result['sacrebleu']], 'Meteor': [result['meteor']]})\n## save to xlsx file\n\nfilepath = 'top_k401_top_p095.xlsx'\nwriter = pd.ExcelWriter(filepath, engine='xlsxwriter')\ndf.to_excel(writer, index=False, sheet_name='Sheet1')\ndf_result.to_excel(writer, index=False, sheet_name='Result')\nwriter.save()","metadata":{"execution":{"iopub.status.busy":"2022-09-16T21:44:53.509502Z","iopub.execute_input":"2022-09-16T21:44:53.509883Z","iopub.status.idle":"2022-09-16T21:44:53.975802Z","shell.execute_reply.started":"2022-09-16T21:44:53.50985Z","shell.execute_reply":"2022-09-16T21:44:53.974469Z"},"trusted":true},"execution_count":100,"outputs":[]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)\nprint(\"top_k = 40 top_p=0.95\")\nresult","metadata":{"execution":{"iopub.status.busy":"2022-09-27T14:35:56.939298Z","iopub.execute_input":"2022-09-27T14:35:56.939722Z","iopub.status.idle":"2022-09-27T14:36:00.038765Z","shell.execute_reply.started":"2022-09-27T14:35:56.939687Z","shell.execute_reply":"2022-09-27T14:36:00.037482Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"top_k = 40 top_p=0.95\n","output_type":"stream"},{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"{'rouge1_precision': 25.8191,\n 'rouge1_recall': 26.9641,\n 'rouge1_fmeasure': 25.4519,\n 'rouge2_precision': 8.7354,\n 'rouge2_recall': 9.2826,\n 'rouge2_fmeasure': 8.6548,\n 'rougeL_precision': 24.4047,\n 'rougeL_recall': 25.5485,\n 'rougeL_fmeasure': 24.0773,\n 'rougeLsum_precision': 24.4055,\n 'rougeLsum_recall': 25.579,\n 'rougeLsum_fmeasure': 24.1118,\n 'sacrebleu': 4.8759,\n 'meteor': 0.1388}"},"metadata":{}}]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)\nprint(\"top_p = 1 temperature=0.95\")\nresult","metadata":{"execution":{"iopub.status.busy":"2022-09-16T21:22:31.42693Z","iopub.execute_input":"2022-09-16T21:22:31.427608Z","iopub.status.idle":"2022-09-16T21:22:34.620823Z","shell.execute_reply.started":"2022-09-16T21:22:31.427553Z","shell.execute_reply":"2022-09-16T21:22:34.619521Z"},"trusted":true},"execution_count":95,"outputs":[{"name":"stdout","text":"top_p = 1 temperature=0.95\n","output_type":"stream"},{"execution_count":95,"output_type":"execute_result","data":{"text/plain":"{'rouge1_precision': 25.3467,\n 'rouge1_recall': 26.6861,\n 'rouge1_fmeasure': 25.1342,\n 'rouge2_precision': 8.1951,\n 'rouge2_recall': 8.8998,\n 'rouge2_fmeasure': 8.2359,\n 'rougeL_precision': 24.0994,\n 'rougeL_recall': 25.494,\n 'rougeL_fmeasure': 23.9426,\n 'rougeLsum_precision': 24.1085,\n 'rougeLsum_recall': 25.5208,\n 'rougeLsum_fmeasure': 23.9557,\n 'sacrebleu': 4.4088,\n 'meteor': 0.1363}"},"metadata":{}}]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)\nprint(\"top_k=100 top_p=0.95\")\nresult","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:50:39.008414Z","iopub.execute_input":"2022-09-16T20:50:39.008862Z","iopub.status.idle":"2022-09-16T20:50:42.282725Z","shell.execute_reply.started":"2022-09-16T20:50:39.008817Z","shell.execute_reply":"2022-09-16T20:50:42.281666Z"},"trusted":true},"execution_count":88,"outputs":[{"name":"stdout","text":"top_k = 100 top_p = 0.95\n","output_type":"stream"},{"execution_count":88,"output_type":"execute_result","data":{"text/plain":"{'rouge1_precision': 24.3675,\n 'rouge1_recall': 25.1895,\n 'rouge1_fmeasure': 23.8669,\n 'rouge2_precision': 7.7654,\n 'rouge2_recall': 8.3323,\n 'rouge2_fmeasure': 7.7053,\n 'rougeL_precision': 23.1111,\n 'rougeL_recall': 23.9016,\n 'rougeL_fmeasure': 22.633,\n 'rougeLsum_precision': 23.0785,\n 'rougeLsum_recall': 23.8937,\n 'rougeLsum_fmeasure': 22.6392,\n 'sacrebleu': 4.09,\n 'meteor': 0.1338}"},"metadata":{}}]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)\nprint(\"NUM_BEAMS=10\")\nresult","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:30:23.493186Z","iopub.execute_input":"2022-09-16T20:30:23.493673Z","iopub.status.idle":"2022-09-16T20:30:26.627072Z","shell.execute_reply.started":"2022-09-16T20:30:23.493632Z","shell.execute_reply":"2022-09-16T20:30:26.625668Z"},"trusted":true},"execution_count":82,"outputs":[{"name":"stdout","text":"NUM_BEAMS=10\n","output_type":"stream"},{"execution_count":82,"output_type":"execute_result","data":{"text/plain":"{'rouge1_precision': 33.2037,\n 'rouge1_recall': 35.3239,\n 'rouge1_fmeasure': 33.1948,\n 'rouge2_precision': 14.1174,\n 'rouge2_recall': 15.4019,\n 'rouge2_fmeasure': 14.2558,\n 'rougeL_precision': 31.4554,\n 'rougeL_recall': 33.5326,\n 'rougeL_fmeasure': 31.4925,\n 'rougeLsum_precision': 31.3992,\n 'rougeLsum_recall': 33.4731,\n 'rougeLsum_fmeasure': 31.4722,\n 'sacrebleu': 8.1095,\n 'meteor': 0.1794}"},"metadata":{}}]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)\nprint(\"NUM_BEAMS=7\")\nresult","metadata":{"execution":{"iopub.status.busy":"2022-09-16T19:35:46.015499Z","iopub.execute_input":"2022-09-16T19:35:46.015879Z","iopub.status.idle":"2022-09-16T19:35:49.142762Z","shell.execute_reply.started":"2022-09-16T19:35:46.015845Z","shell.execute_reply":"2022-09-16T19:35:49.141382Z"},"trusted":true},"execution_count":76,"outputs":[{"name":"stdout","text":"NUM_BEAMS=7\n","output_type":"stream"},{"execution_count":76,"output_type":"execute_result","data":{"text/plain":"{'rouge1_precision': 32.9489,\n 'rouge1_recall': 35.2039,\n 'rouge1_fmeasure': 33.0053,\n 'rouge2_precision': 13.9216,\n 'rouge2_recall': 15.2639,\n 'rouge2_fmeasure': 14.0935,\n 'rougeL_precision': 31.2152,\n 'rougeL_recall': 33.4568,\n 'rougeL_fmeasure': 31.3553,\n 'rougeLsum_precision': 31.1402,\n 'rougeLsum_recall': 33.4541,\n 'rougeLsum_fmeasure': 31.3185,\n 'sacrebleu': 8.1074,\n 'meteor': 0.1782}"},"metadata":{}}]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)\nprint(\"NUM_BEAMS=5\")\nresult","metadata":{"execution":{"iopub.status.busy":"2022-09-16T18:27:20.206718Z","iopub.execute_input":"2022-09-16T18:27:20.207232Z","iopub.status.idle":"2022-09-16T18:27:23.366562Z","shell.execute_reply.started":"2022-09-16T18:27:20.207194Z","shell.execute_reply":"2022-09-16T18:27:23.365607Z"},"trusted":true},"execution_count":69,"outputs":[{"name":"stdout","text":"NUM_BEAMS=5\n","output_type":"stream"},{"execution_count":69,"output_type":"execute_result","data":{"text/plain":"{'rouge1_precision': 32.762,\n 'rouge1_recall': 35.2326,\n 'rouge1_fmeasure': 32.9817,\n 'rouge2_precision': 13.9094,\n 'rouge2_recall': 15.3718,\n 'rouge2_fmeasure': 14.1434,\n 'rougeL_precision': 31.159,\n 'rougeL_recall': 33.5899,\n 'rougeL_fmeasure': 31.4034,\n 'rougeLsum_precision': 31.1237,\n 'rougeLsum_recall': 33.547,\n 'rougeLsum_fmeasure': 31.3612,\n 'sacrebleu': 8.0115,\n 'meteor': 0.1767}"},"metadata":{}}]},{"cell_type":"code","source":"result = compute_metrics(reference_question, pred)\nprint(\"NUM_BEAMS=3\")\nresult","metadata":{"execution":{"iopub.status.busy":"2022-09-16T17:51:35.185179Z","iopub.execute_input":"2022-09-16T17:51:35.185771Z","iopub.status.idle":"2022-09-16T17:51:38.322978Z","shell.execute_reply.started":"2022-09-16T17:51:35.18573Z","shell.execute_reply":"2022-09-16T17:51:38.321363Z"},"trusted":true},"execution_count":63,"outputs":[{"name":"stdout","text":"NUM_BEAMS=3\n","output_type":"stream"},{"execution_count":63,"output_type":"execute_result","data":{"text/plain":"{'rouge1_precision': 32.3829,\n 'rouge1_recall': 34.7771,\n 'rouge1_fmeasure': 32.5061,\n 'rouge2_precision': 13.6391,\n 'rouge2_recall': 14.8435,\n 'rouge2_fmeasure': 13.7345,\n 'rougeL_precision': 30.8244,\n 'rougeL_recall': 33.1571,\n 'rougeL_fmeasure': 30.9929,\n 'rougeLsum_precision': 30.8117,\n 'rougeLsum_recall': 33.1441,\n 'rougeLsum_fmeasure': 30.9659,\n 'sacrebleu': 7.7325,\n 'meteor': 0.1743}"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}