main.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "LoadModel.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "veVbRa4f1mkf"
      },
      "source": [
        "# Reading Checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5KSWQwDxQMFy",
        "outputId": "987cd242-76e1-4bca-c6f1-5b6eb76e7666"
      },
      "source": [
        "!pip install tensorflow_hub\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow_hub in /usr/local/lib/python3.7/dist-packages (0.11.0)\n",
            "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_hub) (1.19.5)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_hub) (3.12.4)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.8.0->tensorflow_hub) (1.15.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.8.0->tensorflow_hub) (54.2.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kg1DoPXGQSdn",
        "outputId": "befc54af-0c94-4ddd-e105-77a51ea51259"
      },
      "source": [
        "!pip install keras tf-models-official pydot graphviz"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: keras in /usr/local/lib/python3.7/dist-packages (2.4.3)\n",
            "Requirement already satisfied: tf-models-official in /usr/local/lib/python3.7/dist-packages (2.4.0)\n",
            "Requirement already satisfied: pydot in /usr/local/lib/python3.7/dist-packages (1.3.0)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.7/dist-packages (0.10.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras) (3.13)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from keras) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras) (1.19.5)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras) (2.10.0)\n",
            "Requirement already satisfied: google-cloud-bigquery>=0.31.0 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (1.21.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (3.2.2)\n",
            "Requirement already satisfied: google-api-python-client>=1.6.7 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (1.12.8)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (7.1.2)\n",
            "Requirement already satisfied: tensorflow-datasets in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (4.0.1)\n",
            "Requirement already satisfied: tf-slim>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (1.1.0)\n",
            "Requirement already satisfied: tensorflow>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (2.4.1)\n",
            "Requirement already satisfied: pycocotools in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (2.0.2)\n",
            "Requirement already satisfied: oauth2client in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (4.1.3)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (0.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (1.15.0)\n",
            "Requirement already satisfied: gin-config in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (0.4.0)\n",
            "Requirement already satisfied: kaggle>=1.3.9 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (1.5.12)\n",
            "Requirement already satisfied: seqeval in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (1.2.2)\n",
            "Requirement already satisfied: tensorflow-hub>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (0.11.0)\n",
            "Requirement already satisfied: opencv-python-headless in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (4.5.1.48)\n",
            "Requirement already satisfied: tensorflow-model-optimization>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (0.5.0)\n",
            "Requirement already satisfied: Cython in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (0.29.22)\n",
            "Requirement already satisfied: py-cpuinfo>=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (7.0.0)\n",
            "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (0.12.1)\n",
            "Requirement already satisfied: psutil>=5.4.3 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (5.4.8)\n",
            "Requirement already satisfied: pandas>=0.22.0 in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (1.1.5)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (from tf-models-official) (0.1.95)\n",
            "Requirement already satisfied: pyparsing>=2.1.4 in /usr/local/lib/python3.7/dist-packages (from pydot) (2.4.7)\n",
            "Requirement already satisfied: google-cloud-core<2.0dev,>=1.0.3 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigquery>=0.31.0->tf-models-official) (1.0.3)\n",
            "Requirement already satisfied: google-resumable-media!=0.4.0,<0.5.0dev,>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigquery>=0.31.0->tf-models-official) (0.4.1)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-bigquery>=0.31.0->tf-models-official) (3.12.4)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->tf-models-official) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->tf-models-official) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->tf-models-official) (0.10.0)\n",
            "Requirement already satisfied: google-api-core<2dev,>=1.21.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client>=1.6.7->tf-models-official) (1.26.2)\n",
            "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client>=1.6.7->tf-models-official) (0.0.4)\n",
            "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client>=1.6.7->tf-models-official) (3.0.1)\n",
            "Requirement already satisfied: httplib2<1dev,>=0.15.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client>=1.6.7->tf-models-official) (0.17.4)\n",
            "Requirement already satisfied: google-auth>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from google-api-python-client>=1.6.7->tf-models-official) (1.28.0)\n",
            "Requirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (20.3.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (4.41.1)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (0.12.0)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (1.1.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (0.3.3)\n",
            "Requirement already satisfied: promise in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (2.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (2.23.0)\n",
            "Requirement already satisfied: dm-tree in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (0.1.5)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (0.16.0)\n",
            "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (0.29.0)\n",
            "Requirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from tensorflow-datasets->tf-models-official) (5.1.2)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (1.12.1)\n",
            "Requirement already satisfied: grpcio~=1.32.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (1.32.0)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (1.12)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (3.7.4.3)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (0.2.0)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (0.36.2)\n",
            "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (2.4.1)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (0.3.3)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (1.6.3)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (3.3.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (1.1.2)\n",
            "Requirement already satisfied: tensorflow-estimator<2.5.0,>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.4.0->tf-models-official) (2.4.0)\n",
            "Requirement already satisfied: setuptools>=18.0 in /usr/local/lib/python3.7/dist-packages (from pycocotools->tf-models-official) (54.2.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.7/dist-packages (from oauth2client->tf-models-official) (0.2.8)\n",
            "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.7/dist-packages (from oauth2client->tf-models-official) (0.4.8)\n",
            "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from oauth2client->tf-models-official) (4.7.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kaggle>=1.3.9->tf-models-official) (2020.12.5)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from kaggle>=1.3.9->tf-models-official) (1.24.3)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.7/dist-packages (from kaggle>=1.3.9->tf-models-official) (4.0.1)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from seqeval->tf-models-official) (0.22.2.post1)\n",
            "Requirement already satisfied: typeguard>=2.7 in /usr/local/lib/python3.7/dist-packages (from tensorflow-addons->tf-models-official) (2.7.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.22.0->tf-models-official) (2018.9)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.21.0->google-api-python-client>=1.6.7->tf-models-official) (1.53.0)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2dev,>=1.21.0->google-api-python-client>=1.6.7->tf-models-official) (20.9)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth>=1.16.0->google-api-python-client>=1.6.7->tf-models-official) (4.2.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->tensorflow-datasets->tf-models-official) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->tensorflow-datasets->tf-models-official) (2.10)\n",
            "Requirement already satisfied: zipp>=0.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-resources; python_version < \"3.9\"->tensorflow-datasets->tf-models-official) (3.4.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow>=2.4.0->tf-models-official) (1.0.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow>=2.4.0->tf-models-official) (1.8.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow>=2.4.0->tf-models-official) (3.3.4)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow>=2.4.0->tf-models-official) (0.4.3)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.7/dist-packages (from python-slugify->kaggle>=1.3.9->tf-models-official) (1.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval->tf-models-official) (1.0.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow>=2.4.0->tf-models-official) (3.8.1)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow>=2.4.0->tf-models-official) (1.3.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow>=2.4.0->tf-models-official) (3.1.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ayVgV9NqQVBd"
      },
      "source": [
        "import os\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "from keras.utils import np_utils\n",
        "\n",
        "import official.nlp.bert.bert_models\n",
        "import official.nlp.bert.configs\n",
        "import official.nlp.bert.run_classifier\n",
        "import official.nlp.bert.tokenization as tokenization\n",
        "\n",
        "from official.modeling import tf_utils\n",
        "from official import nlp\n",
        "from official.nlp import bert\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "import matplotlib.pyplot as plt\n"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qdPLZ9HXQjRo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ddde9f6a-d64a-47f6-d202-55851f2384c3"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive')\n"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8IXj-mUIJPJ0"
      },
      "source": [
        "from IPython.display import display, Javascript\n",
        "from google.colab.output import eval_js\n",
        "from base64 import b64decode\n",
        "\n",
        "def take_photo(filename='photo.jpg', quality=0.8):\n",
        "  js = Javascript('''\n",
        "    async function takePhoto(quality) {`s`\n",
        "      const div = document.createElement('div');\n",
        "      const capture = document.createElement('button');\n",
        "      capture.textContent = 'Capture';\n",
        "      div.appendChild(capture);\n",
        "\n",
        "      const video = document.createElement('video');\n",
        "      video.style.display = 'block';\n",
        "      const stream = await navigator.mediaDevices.getUserMedia({video: true});\n",
        "\n",
        "      document.body.appendChild(div);\n",
        "      div.appendChild(video);\n",
        "      video.srcObject = stream;\n",
        "      await video.play();\n",
        "\n",
        "      // Resize the output to fit the video element.\n",
        "      google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);\n",
        "\n",
        "      // Wait for Capture to be clicked.\n",
        "      await new Promise((resolve) => capture.onclick = resolve);\n",
        "\n",
        "      const canvas = document.createElement('canvas');\n",
        "      canvas.width = video.videoWidth;\n",
        "      canvas.height = video.videoHeight;\n",
        "      canvas.getContext('2d').drawImage(video, 0, 0);\n",
        "      stream.getVideoTracks()[0].stop();\n",
        "      div.remove();\n",
        "      return canvas.toDataURL('image/jpeg', quality);\n",
        "    }\n",
        "    ''')\n",
        "  display(js)\n",
        "  data = eval_js('takePhoto({})'.format(quality))\n",
        "  binary = b64decode(data.split(',')[1])\n",
        "  with open(filename, 'wb') as f:\n",
        "    f.write(binary)\n",
        "  return filename"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FbfKEty1nTDa"
      },
      "source": [
        "from tensorflow.keras.models import load_model\n",
        "from tensorflow import keras"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zasWB7oVQZRt"
      },
      "source": [
        "path = \"/content/gdrive/My Drive/checkpoint\" \n",
        "optimizer = nlp.optimization.create_optimizer(\n",
        "    2e-5, num_train_steps=417\n",
        ", num_warmup_steps=41)\n",
        "model = keras.models.load_model(path, custom_objects={'AdamWeightDecay': optimizer})"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K2m5z7liTSnW"
      },
      "source": [
        "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",\n",
        "                            trainable=True)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5RGsQLeTSe2"
      },
      "source": [
        "\n",
        "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
        "tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qZe7ccl8Vbn3"
      },
      "source": [
        "def encode_names(n, tokenizer):\n",
        "   tokens = list(tokenizer.tokenize(n))\n",
        "   tokens.append('[SEP]')\n",
        "   return tokenizer.convert_tokens_to_ids(tokens)\n",
        "\n",
        "def bert_encode(string_list, tokenizer, max_seq_length):\n",
        "  num_examples = len(string_list)\n",
        "  \n",
        "  string_tokens = tf.ragged.constant([\n",
        "      encode_names(n, tokenizer) for n in np.array(string_list)])\n",
        "\n",
        "  cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*string_tokens.shape[0]\n",
        "  input_word_ids = tf.concat([cls, string_tokens], axis=-1)\n",
        "\n",
        "  input_mask = tf.ones_like(input_word_ids).to_tensor(shape=(None, max_seq_length))\n",
        "\n",
        "  type_cls = tf.zeros_like(cls)\n",
        "  type_tokens = tf.ones_like(string_tokens)\n",
        "  input_type_ids = tf.concat(\n",
        "      [type_cls, type_tokens], axis=-1).to_tensor(shape=(None, max_seq_length))\n",
        "\n",
        "  inputs = {\n",
        "      'input_word_ids': input_word_ids.to_tensor(shape=(None, max_seq_length)),\n",
        "      'input_mask': input_mask,\n",
        "      'input_type_ids': input_type_ids}\n",
        "\n",
        "  return inputs"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uPyM__IOU8sf",
        "outputId": "f9bcc493-0a82-49ec-f488-44cccb734e30"
      },
      "source": [
        "tokenizer"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<official.nlp.bert.tokenization.FullTokenizer at 0x7fe774108b90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CGrY18xwT9k6",
        "outputId": "7d2274d7-9b3b-44b1-862d-34ca9d8808c0"
      },
      "source": [
        "pip install transformers==3\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers==3 in /usr/local/lib/python3.7/dist-packages (3.0.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers==3) (0.0.43)\n",
            "Requirement already satisfied: tokenizers==0.8.0-rc4 in /usr/local/lib/python3.7/dist-packages (from transformers==3) (0.8.0rc4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==3) (20.9)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==3) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==3) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==3) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==3) (2019.12.20)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (from transformers==3) (0.1.95)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==3) (1.19.5)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3) (1.0.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==3) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3) (1.24.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fi2YSQZ9BFfD"
      },
      "source": [
        "#Face Emotion detection\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "otPSCa9VEJqo"
      },
      "source": [
        "from keras.models import load_model\n",
        "from IPython.display import display, Javascript\n",
        "from google.colab.output import eval_js\n",
        "from time import sleep\n",
        "from keras.preprocessing.image import img_to_array\n",
        "from keras.preprocessing import image\n",
        "import cv2\n",
        "import numpy as np\n"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B1KomyRlEKQw"
      },
      "source": [
        "face_classifier = cv2.CascadeClassifier('/content/gdrive/My Drive/haarcascade_frontalface_default.xml')\n",
        "classifier = load_model('/content/gdrive/My Drive/Emotion_Detection.h5')\n",
        "\n",
        "class_labels = ['Angry','Happy','Neutral','Sad','Surprise']"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YPSd9MXwegM_"
      },
      "source": [
        "def video_stream():\n",
        "  js = Javascript('''\n",
        "    var video;\n",
        "    var div = null;\n",
        "    var stream;\n",
        "    var captureCanvas;\n",
        "    var imgElement;\n",
        "    var labelElement;\n",
        "    \n",
        "    var pendingResolve = null;\n",
        "    var shutdown = false;\n",
        "    \n",
        "    function removeDom() {\n",
        "       stream.getVideoTracks()[0].stop();\n",
        "       video.remove();\n",
        "       div.remove();\n",
        "       video = null;\n",
        "       div = null;\n",
        "       stream = null;\n",
        "       imgElement = null;\n",
        "       captureCanvas = null;\n",
        "       labelElement = null;\n",
        "    }\n",
        "    \n",
        "    function onAnimationFrame() {\n",
        "      if (!shutdown) {\n",
        "        window.requestAnimationFrame(onAnimationFrame);\n",
        "      }\n",
        "      if (pendingResolve) {\n",
        "        var result = \"\";\n",
        "        if (!shutdown) {\n",
        "          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n",
        "          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n",
        "        }\n",
        "        var lp = pendingResolve;\n",
        "        pendingResolve = null;\n",
        "        lp(result);\n",
        "      }\n",
        "    }\n",
        "    \n",
        "    async function createDom() {\n",
        "      if (div !== null) {\n",
        "        return stream;\n",
        "      }\n",
        "\n",
        "      div = document.createElement('div');\n",
        "      div.style.border = '2px solid black';\n",
        "      div.style.padding = '3px';\n",
        "      div.style.width = '100%';\n",
        "      div.style.maxWidth = '600px';\n",
        "      document.body.appendChild(div);\n",
        "      \n",
        "      const modelOut = document.createElement('div');\n",
        "      modelOut.innerHTML = \"<span>Status:</span>\";\n",
        "      labelElement = document.createElement('span');\n",
        "      labelElement.innerText = 'No data';\n",
        "      labelElement.style.fontWeight = 'bold';\n",
        "      modelOut.appendChild(labelElement);\n",
        "      div.appendChild(modelOut);\n",
        "           \n",
        "      video = document.createElement('video');\n",
        "      video.style.display = 'block';\n",
        "      video.width = div.clientWidth - 6;\n",
        "      video.setAttribute('playsinline', '');\n",
        "      video.onclick = () => { shutdown = true; };\n",
        "      stream = await navigator.mediaDevices.getUserMedia(\n",
        "          {video: { facingMode: \"environment\"}});\n",
        "      div.appendChild(video);\n",
        "\n",
        "      imgElement = document.createElement('img');\n",
        "      imgElement.style.position = 'absolute';\n",
        "      imgElement.style.zIndex = 1;\n",
        "      imgElement.onclick = () => { shutdown = true; };\n",
        "      div.appendChild(imgElement);\n",
        "      \n",
        "      const instruction = document.createElement('div');\n",
        "      instruction.innerHTML = \n",
        "          '<span style=\"color: red; font-weight: bold;\">' +\n",
        "          'When finished, click here or on the video to stop this demo</span>';\n",
        "      div.appendChild(instruction);\n",
        "      instruction.onclick = () => { shutdown = true; };\n",
        "      \n",
        "      video.srcObject = stream;\n",
        "      await video.play();\n",
        "\n",
        "      captureCanvas = document.createElement('canvas');\n",
        "      captureCanvas.width = 640; //video.videoWidth;\n",
        "      captureCanvas.height = 480; //video.videoHeight;\n",
        "      window.requestAnimationFrame(onAnimationFrame);\n",
        "      \n",
        "      return stream;\n",
        "    }\n",
        "    async function stream_frame(label, imgData) {\n",
        "      if (shutdown) {\n",
        "        removeDom();\n",
        "        shutdown = false;\n",
        "        return '';\n",
        "      }\n",
        "\n",
        "      var preCreate = Date.now();\n",
        "      stream = await createDom();\n",
        "      \n",
        "      var preShow = Date.now();\n",
        "      if (label != \"\") {\n",
        "        labelElement.innerHTML = label;\n",
        "      }\n",
        "            \n",
        "      if (imgData != \"\") {\n",
        "        var videoRect = video.getClientRects()[0];\n",
        "        imgElement.style.top = videoRect.top + \"px\";\n",
        "        imgElement.style.left = videoRect.left + \"px\";\n",
        "        imgElement.style.width = videoRect.width + \"px\";\n",
        "        imgElement.style.height = videoRect.height + \"px\";\n",
        "        imgElement.src = imgData;\n",
        "      }\n",
        "      \n",
        "      var preCapture = Date.now();\n",
        "      var result = await new Promise(function(resolve, reject) {\n",
        "        pendingResolve = resolve;\n",
        "      });\n",
        "      shutdown = false;\n",
        "      \n",
        "      return {'create': preShow - preCreate, \n",
        "              'show': preCapture - preShow, \n",
        "              'capture': Date.now() - preCapture,\n",
        "              'img': result};\n",
        "    }\n",
        "    ''')\n",
        "\n",
        "  display(js)\n",
        "  \n",
        "def video_frame(label, bbox):\n",
        "  data = eval_js('stream_frame(\"{}\", \"{}\")'.format(label, bbox))\n",
        "  return data"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s6YR3odC-RpF"
      },
      "source": [
        "face_cascade = cv2.CascadeClassifier(cv2.samples.findFile(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml'))"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EX7SCkSnuAcP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e9a98015-9052-4170-b903-fa85da48eef6"
      },
      "source": [
        "emotions=[]\n",
        "emotions"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m_6xOPUAsojD",
        "outputId": "7d93ccb3-5255-4744-be67-c855f41d6417"
      },
      "source": [
        "emotions"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0nEu0YH5vwXT"
      },
      "source": [
        "def most_frequent(List):\n",
        "    occurence_count = Counter(List)\n",
        "    return occurence_count.most_common(1)[0][0]"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vL09Ss5NN7tQ"
      },
      "source": [
        "##################################### mohema tala3ha foo2\n",
        "def js_to_image(js_reply):\n",
        "  \"\"\"\n",
        "  Params:\n",
        "          js_reply: JavaScript object containing image from webcam\n",
        "  Returns:\n",
        "          img: OpenCV BGR image\n",
        "  \"\"\"\n",
        "  # decode base64 image\n",
        "  image_bytes = b64decode(js_reply.split(',')[1])\n",
        "  # convert bytes to numpy array\n",
        "  jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)\n",
        "  # decode numpy array into OpenCV BGR image\n",
        "  img = cv2.imdecode(jpg_as_np, flags=1)\n",
        "\n",
        "  return img"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7NJXGX7qrZGl"
      },
      "source": [
        "##################################################################################      Working         ##############################################################################\n",
        "from collections import Counter\n",
        "label_html = 'Capturing...'\n",
        "\n",
        "# initialze bounding box to empty\n",
        "bbox = ''\n",
        "count = 0 \n",
        "def read_face_emotion():\n",
        "  video_stream()\n",
        "  emotions=[]\n",
        "  print('awel read')\n",
        "  while True:\n",
        "\n",
        "      js_reply = video_frame(label_html, bbox)\n",
        "      labels=[]\n",
        "      if not js_reply:\n",
        "          break\n",
        "\n",
        "      # convert JS response to OpenCV Image\n",
        "      img = js_to_image(js_reply[\"img\"])\n",
        "\n",
        "      # create transparent overlay for bounding box\n",
        "      bbox_array = np.zeros([480,640,4], dtype=np.uint8)\n",
        "\n",
        "      # grayscale image for face detection\n",
        "      gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n",
        "\n",
        "      # get face region coordinates\n",
        "      faces = face_cascade.detectMultiScale(gray)\n",
        "      # get face bounding box for overlay\n",
        "      for (x,y,w,h) in faces:\n",
        "        bbox_array = cv2.rectangle(bbox_array,(x,y),(x+w,y+h),(255,0,0),2)\n",
        "        roi_gray = gray[y:y+h,x:x+w]\n",
        "        roi_gray = cv2.resize(roi_gray,(48,48),interpolation=cv2.INTER_AREA)\n",
        "        \n",
        "\n",
        "        if np.sum([roi_gray])!=0:\n",
        "            roi = roi_gray.astype('float')/255.0\n",
        "            roi = img_to_array(roi)\n",
        "            roi = np.expand_dims(roi,axis=0)\n",
        "            preds = classifier.predict(roi)[0]\n",
        "           # print(\"\\nprediction = \",preds)\n",
        "            label=class_labels[preds.argmax()]\n",
        "          #  print(\"\\nprediction max = \",preds.argmax())\n",
        "          #  print(\"\\nlabel = \",label)\n",
        "\n",
        "            emotions.append(label)\n",
        "            try:\n",
        "              if(len(emotions)%5==0):\n",
        "                print('ana gowa')\n",
        "                return most_frequent(emotions)\n",
        "\n",
        "              \n",
        "            except Exception as e:\n",
        "              continue\n",
        "               \n",
        "\n",
        "\n",
        "            \n",
        "\n",
        "            label_position = (x,y)\n",
        "            cv2.putText(img,label,label_position,cv2.FONT_HERSHEY_SIMPLEX,2,(0,255,0),3)\n",
        "\n",
        "  # return(mode(emotions))\n",
        "\n",
        "    \n",
        "      # convert overlay of bbox into bytes\n",
        "      \n",
        "      # update bbox so next frame gets new overlay\n",
        "    "
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a7B3vamnS6Cp"
      },
      "source": [
        "#print('most common '+read_face_emotion())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LEF0NRwEtz9c"
      },
      "source": [
        "# def most_common(List):\n",
        "#     return(mode(List))\n",
        "#read_face_emotion()\n",
        "def threaded_face_emotion():\n",
        "  read_face_emotion()\n",
        "  most_common(emotions)\n",
        "  ################################### faksana"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-YQYbbphvVIS"
      },
      "source": [
        "# import threading\n",
        "\n",
        "# t1=threading.Thread(target=read_face_emotion)\n",
        "# t1.start()\n",
        "# print(threading.active_count() )\n",
        "# print('fierst')\n",
        "\n",
        "# print('hello')"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aDRtarPIf3iz"
      },
      "source": [
        "# video_stream()\n",
        "  ################################### faksana  ################################### faksana  ################################### faksana  ################################### faksana\n",
        "# # label for video\n",
        "# label_html = 'Front Camera'\n",
        "# # initialze bounding box to empty\n",
        "# bbox = ''\n",
        "# count = 0 \n",
        "\n",
        "# emotions=[]\n",
        "# ###################################\n",
        "# def face_emotion():\n",
        "#   video_stream()\n",
        "#   print('out')\n",
        "#   while True:\n",
        "#     print('in')\n",
        "#     js_reply = video_frame(label_html, bbox)\n",
        "#     print('2222222222')\n",
        "#     if not js_reply:\n",
        "#         print('not')\n",
        "#         return\n",
        "#     labels = []\n",
        "#     print('1')\n",
        "#     # convert JS response to OpenCV Image\n",
        "#     img = js_to_image(js_reply[\"img\"])\n",
        "#     print('2')\n",
        "#     # create transparent overlay for bounding box\n",
        "#     bbox_array = np.zeros([480,640,4], dtype=np.uint8)\n",
        "#     print('3')\n",
        "#     # grayscale image for face detection\n",
        "#     gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n",
        "#     print('4')\n",
        "#     # get face region coordinates\n",
        "#     faces = face_cascade.detectMultiScale(gray)\n",
        "#     #faces = face_classifier.detectMultiScale(gray,1.3,5)\n",
        "\n",
        "\n",
        "#     print('before loop')\n",
        "\n",
        "#     # get face bounding box for overlay\n",
        "#     for (x,y,w,h) in faces:\n",
        "#         print('in for loop')\n",
        "#        # cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)\n",
        "#         roi_gray = gray[y:y+h,x:x+w]\n",
        "#         roi_gray = cv2.resize(roi_gray,(48,48),interpolation=cv2.INTER_AREA)\n",
        "\n",
        "        \n",
        "#         if np.sum([roi_gray])!=0:\n",
        "#             roi = roi_gray.astype('float')/255.0\n",
        "#             roi = img_to_array(roi)\n",
        "#             roi = np.expand_dims(roi,axis=0)\n",
        "\n",
        "#         # make a prediction on the ROI, then lookup the class\n",
        "\n",
        "#             preds = classifier.predict(roi)[0]\n",
        "#             print(\"\\nprediction = \",preds)\n",
        "#             label=class_labels[preds.argmax()]\n",
        "#             emotions.append(label)\n",
        "#             fs=True\n",
        "#             print(fs)\n",
        "#             print(len(emotions))\n",
        "#             try:\n",
        "#               print('in try')\n",
        "#               print(len(emotions))\n",
        "#               if len(emotions)%5==0:\n",
        "#                 print('most common emotion: '+most_common(emotions))\n",
        "#                 fs=False\n",
        "#                 emotions.clear()\n",
        "#             except Exception as e:\n",
        "#               print('unexpected error in face detection')\n",
        "#               print(\"\\nprediction max = \",preds.argmax())\n",
        "#             #print(\"\\nlabel = \",label)\n",
        "#             label_position = (x,y)\n",
        "#             cv2.putText(img,label,label_position,cv2.FONT_HERSHEY_SIMPLEX,2,(0,255,0),3)\n",
        "#         else:\n",
        "#             cv2.putText(img,'No Face Found',(20,60),cv2.FONT_HERSHEY_SIMPLEX,2,(0,255,0),3)\n",
        "#             print(\"else\")\n",
        "#     # update bbox so next frame gets new overlay\n",
        "# face_emotion()\n",
        "  ################################### faksana  ################################### faksana  ################################### faksana  ################################### faksana"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FUnsBV_oV_ly"
      },
      "source": [
        "# Q&A model now\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VuX5LS5WWCqe"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "from transformers import BertForQuestionAnswering\n",
        "import torch\n",
        "import textwrap\n",
        "from textblob import TextBlob\n",
        "import nltk"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3fWXvHxkGiVE"
      },
      "source": [
        "Load the Pretrained Model for Question Answering Bert"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGRSs5F2WFXW"
      },
      "source": [
        "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\n",
        "tokenizer1 = AutoTokenizer.from_pretrained(\"bert-large-uncased-whole-word-masking-finetuned-squad\")\n",
        "\n",
        "model1 = AutoModelForQuestionAnswering.from_pretrained(\"bert-large-uncased-whole-word-masking-finetuned-squad\")\n",
        "\n",
        "#AutoTokenizer.from_pretrained(\"deepset/bert-large-uncased-whole-word-masking-squad2\")"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NIahnOYgdlX7"
      },
      "source": [
        "wrapper = textwrap.TextWrapper(width=80)\n",
        "import concurrent.futures\n",
        "\n",
        "\n",
        "def answer_question(question, answer_text):\n",
        "    '''\n",
        "    Takes a `question` string and an `answer_text` string (which contains the\n",
        "    answer), and identifies the words within the `answer_text` that are the\n",
        "    answer. Prints them out.\n",
        "    '''\n",
        " \n",
        "\n",
        "    \n",
        "\n",
        "  \n",
        "\n",
        "   # print('ba3d el thread')\n",
        "\n",
        "\n",
        "\n",
        "    # ======== Tokenize ========\n",
        "    # Apply the tokenizer to the input text, treating them as a text-pair.\n",
        "    input_ids = tokenizer1.encode(question, answer_text)\n",
        "\n",
        "    # Report how long the input sequence is.\n",
        "   # print('Query has {:,} tokens.\\n'.format(len(input_ids)))\n",
        "\n",
        "    # ======== Set Segment IDs ========\n",
        "    # Search the input_ids for the first instance of the `[SEP]` token.\n",
        "    sep_index = input_ids.index(tokenizer1.sep_token_id)\n",
        "\n",
        "    # The number of segment A tokens includes the [SEP] token istelf.\n",
        "    num_seg_a = sep_index + 1\n",
        "\n",
        "    # The remainder are segment B.\n",
        "    num_seg_b = len(input_ids) - num_seg_a\n",
        "\n",
        "    # Construct the list of 0s and 1s.\n",
        "    segment_ids = [0] * num_seg_a + [1] * num_seg_b\n",
        "\n",
        "    # There should be a segment_id for every input token.\n",
        "    assert len(segment_ids) == len(input_ids)\n",
        "\n",
        "    # ======== Evaluate ========\n",
        "    # Run  question through the model.\n",
        "    start_scores, end_scores = model1(torch.tensor([input_ids]),  # The tokens representing our input text.\n",
        "                                     token_type_ids=torch.tensor(\n",
        "                                         [segment_ids]))  # The segment IDs to differentiate question from answer_text\n",
        "\n",
        "    # ======== Reconstruct Answer ========\n",
        "    # Find the tokens with the highest `start` and `end` scores.\n",
        "    answer_start = torch.argmax(start_scores)\n",
        "    answer_end = torch.argmax(end_scores)\n",
        "\n",
        "    # Get the string versions of the input tokens.\n",
        "    tokens = tokenizer1.convert_ids_to_tokens(input_ids)\n",
        "\n",
        "    # Start with the first token.\n",
        "    answer = tokens[answer_start]\n",
        "\n",
        "    # Select the remaining answer tokens and join them with whitespace.\n",
        "    for i in range(answer_start + 1, answer_end + 1):\n",
        "\n",
        "        # If it's a subword token, then recombine it with the previous token.\n",
        "        if tokens[i][0:2] == '##':\n",
        "            answer += tokens[i][2:]\n",
        "\n",
        "        # Otherwise, add a space then the token.\n",
        "        else:\n",
        "            answer += ' ' + tokens[i]\n",
        "    if \"CLS\" in answer:\n",
        "      print('You: '+ question )\n",
        "      print('Sorry i dont know')\n",
        "    else:\n",
        "      print('a5er questions')\n",
        "      print('You: '+ question )\n",
        "      print('Answer : '+ answer )\n",
        "     # print('oh i sense that you are '+emotion)\n"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yImsxDsKrSl6"
      },
      "source": [
        "def answer_this(question,y=1000):\n",
        "  print('awel answer')\n",
        "  if(y<200):\n",
        "    return\n",
        " # question = ['which types of vaccines available to cure covid']\n",
        "  global Selected_Category\n",
        "  global inputs\n",
        "  global model\n",
        "  global prediction\n",
        "  question1=[question]\n",
        "  inputs = bert_encode(string_list=list(question1), \n",
        "                     tokenizer=tokenizer, \n",
        "                     max_seq_length=63)\n",
        "  prediction = model.predict(inputs)\n",
        "  possible_categories=['Comparison','Economic_Effects','Having_COVID','Individual_Response','Nomenclature','Origin','Prevention','Reporting','Societal Effects','Societal Response','Speculation','Symptoms','Testing','Transmission','Treatment']\n",
        "  for i in range (len(possible_categories)):\n",
        "    if np.argmax(prediction)== i:\n",
        "\n",
        "      Selected_Category=possible_categories[i]\n",
        "\n",
        "\n",
        "\n",
        "  # if np.argmax(prediction)== 0:\n",
        "  #   Selected_Category='Comparison'\n",
        "    \n",
        "  # if np.argmax(prediction)== 1:\n",
        "  #   Selected_Category='Economic_Effects'\n",
        "    \n",
        "  # if np.argmax(prediction)== 2:\n",
        "  #   Selected_Category='Having_COVID'\n",
        "    \n",
        "  # if np.argmax(prediction)== 3:\n",
        "  #   Selected_Category='Individual_Response'\n",
        "    \n",
        "  # if np.argmax(prediction)== 4:\n",
        "  #   Selected_Category='Nomenclature'\n",
        "    \n",
        "  # if np.argmax(prediction)== 5:\n",
        "  #   Selected_Category='Origin'\n",
        "    \n",
        "  # if np.argmax(prediction)== 6:\n",
        "  #   Selected_Category='Prevention'\n",
        "    \n",
        "  # if np.argmax(prediction)== 7:\n",
        "  #   Selected_Category='Reporting'\n",
        "    \n",
        "  # if np.argmax(prediction)== 8:\n",
        "  #   Selected_Category='Societal Effects'\n",
        "    \n",
        "  # if np.argmax(prediction)== 9:\n",
        "  #   Selected_Category='Societal Response'\n",
        "    \n",
        "  # if np.argmax(prediction)== 10:\n",
        "  #   Selected_Category='Speculation'\n",
        "    \n",
        "  # if np.argmax(prediction)== 11:\n",
        "  #   Selected_Category='Symptoms'\n",
        "    \n",
        "  # if np.argmax(prediction)== 12:\n",
        "  #   Selected_Category='Testing'\n",
        "    \n",
        "  # if np.argmax(prediction)== 13:\n",
        "  #   Selected_Category='Transmission'\n",
        "    \n",
        "  # if np.argmax(prediction)== 14:\n",
        "  #   Selected_Category='Treatment'\n",
        "    \n",
        "\n",
        "  \n",
        "  flag_exception = False\n",
        "\n",
        "  try:\n",
        "    with open('/content/gdrive/My Drive/Categories/'+Selected_Category+'.txt') as f:\n",
        "      x=f.readlines()\n",
        "    Categorial_Abtract=x\n",
        "    bert_abstract=''.join(Categorial_Abtract)\n",
        "\n",
        "    bert_abstract\n",
        "  except Exception as e:\n",
        "    print('You: '+ question )\n",
        "    print(Selected_Category +' Category is still not available yet.')\n",
        "    return\n",
        "    flag_exception = True\n",
        "\n",
        "  if  flag_exception == True:\n",
        "    print(Selected_Category +' Category is still not available yet.')\n",
        "    return\n",
        "  else:\n",
        "    try:\n",
        "      \n",
        "      \n",
        "      #print(bert_abstract)\n",
        "      answer_question(question,bert_abstract[:y])\n",
        "    except Exception as e:\n",
        "      print ('ops an error occured  '+ str(e))\n",
        "      answer_this(question,y-100)\n",
        "      \n",
        "\n",
        "  "
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcW-y9H5CB9Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "06d0a4dd-b1dc-44ef-acdd-7d4f462d614f"
      },
      "source": [
        "# answer_this('what is covid cure')\n",
        "# answer_this('and where did covid start')\n",
        "# answer_this('what is the symptoms?')\n",
        "# answer_this('what is the incubation period?')\n",
        "# answer_this('when will corona end?')\n",
        "answer_this('what are type of vaccines to cure corona?')"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "You: what are type of vaccines to cure corona?\n",
            "Answer : mrna vaccines\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DNXVhRvH-2gM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 640
        },
        "outputId": "d1ec1334-a008-4501-9306-d6a5949d39af"
      },
      "source": [
        "  import concurrent.futures\n",
        "  \n",
        "  #print(read_face_emotion())\n",
        "  with concurrent.futures.ThreadPoolExecutor() as executor:\n",
        "      future = executor.submit(read_face_emotion)\n",
        "      future2=executor.submit(answer_this,'what is covid')\n",
        "      r=future.result()\n",
        "      print('i sense '+r)\n",
        "      "
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    var video;\n",
              "    var div = null;\n",
              "    var stream;\n",
              "    var captureCanvas;\n",
              "    var imgElement;\n",
              "    var labelElement;\n",
              "    \n",
              "    var pendingResolve = null;\n",
              "    var shutdown = false;\n",
              "    \n",
              "    function removeDom() {\n",
              "       stream.getVideoTracks()[0].stop();\n",
              "       video.remove();\n",
              "       div.remove();\n",
              "       video = null;\n",
              "       div = null;\n",
              "       stream = null;\n",
              "       imgElement = null;\n",
              "       captureCanvas = null;\n",
              "       labelElement = null;\n",
              "    }\n",
              "    \n",
              "    function onAnimationFrame() {\n",
              "      if (!shutdown) {\n",
              "        window.requestAnimationFrame(onAnimationFrame);\n",
              "      }\n",
              "      if (pendingResolve) {\n",
              "        var result = \"\";\n",
              "        if (!shutdown) {\n",
              "          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n",
              "          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n",
              "        }\n",
              "        var lp = pendingResolve;\n",
              "        pendingResolve = null;\n",
              "        lp(result);\n",
              "      }\n",
              "    }\n",
              "    \n",
              "    async function createDom() {\n",
              "      if (div !== null) {\n",
              "        return stream;\n",
              "      }\n",
              "\n",
              "      div = document.createElement('div');\n",
              "      div.style.border = '2px solid black';\n",
              "      div.style.padding = '3px';\n",
              "      div.style.width = '100%';\n",
              "      div.style.maxWidth = '600px';\n",
              "      document.body.appendChild(div);\n",
              "      \n",
              "      const modelOut = document.createElement('div');\n",
              "      modelOut.innerHTML = \"<span>Status:</span>\";\n",
              "      labelElement = document.createElement('span');\n",
              "      labelElement.innerText = 'No data';\n",
              "      labelElement.style.fontWeight = 'bold';\n",
              "      modelOut.appendChild(labelElement);\n",
              "      div.appendChild(modelOut);\n",
              "           \n",
              "      video = document.createElement('video');\n",
              "      video.style.display = 'block';\n",
              "      video.width = div.clientWidth - 6;\n",
              "      video.setAttribute('playsinline', '');\n",
              "      video.onclick = () => { shutdown = true; };\n",
              "      stream = await navigator.mediaDevices.getUserMedia(\n",
              "          {video: { facingMode: \"environment\"}});\n",
              "      div.appendChild(video);\n",
              "\n",
              "      imgElement = document.createElement('img');\n",
              "      imgElement.style.position = 'absolute';\n",
              "      imgElement.style.zIndex = 1;\n",
              "      imgElement.onclick = () => { shutdown = true; };\n",
              "      div.appendChild(imgElement);\n",
              "      \n",
              "      const instruction = document.createElement('div');\n",
              "      instruction.innerHTML = \n",
              "          '<span style=\"color: red; font-weight: bold;\">' +\n",
              "          'When finished, click here or on the video to stop this demo</span>';\n",
              "      div.appendChild(instruction);\n",
              "      instruction.onclick = () => { shutdown = true; };\n",
              "      \n",
              "      video.srcObject = stream;\n",
              "      await video.play();\n",
              "\n",
              "      captureCanvas = document.createElement('canvas');\n",
              "      captureCanvas.width = 640; //video.videoWidth;\n",
              "      captureCanvas.height = 480; //video.videoHeight;\n",
              "      window.requestAnimationFrame(onAnimationFrame);\n",
              "      \n",
              "      return stream;\n",
              "    }\n",
              "    async function stream_frame(label, imgData) {\n",
              "      if (shutdown) {\n",
              "        removeDom();\n",
              "        shutdown = false;\n",
              "        return '';\n",
              "      }\n",
              "\n",
              "      var preCreate = Date.now();\n",
              "      stream = await createDom();\n",
              "      \n",
              "      var preShow = Date.now();\n",
              "      if (label != \"\") {\n",
              "        labelElement.innerHTML = label;\n",
              "      }\n",
              "            \n",
              "      if (imgData != \"\") {\n",
              "        var videoRect = video.getClientRects()[0];\n",
              "        imgElement.style.top = videoRect.top + \"px\";\n",
              "        imgElement.style.left = videoRect.left + \"px\";\n",
              "        imgElement.style.width = videoRect.width + \"px\";\n",
              "        imgElement.style.height = videoRect.height + \"px\";\n",
              "        imgElement.src = imgData;\n",
              "      }\n",
              "      \n",
              "      var preCapture = Date.now();\n",
              "      var result = await new Promise(function(resolve, reject) {\n",
              "        pendingResolve = resolve;\n",
              "      });\n",
              "      shutdown = false;\n",
              "      \n",
              "      return {'create': preShow - preCreate, \n",
              "              'show': preCapture - preShow, \n",
              "              'capture': Date.now() - preCapture,\n",
              "              'img': result};\n",
              "    }\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "awel answer\n",
            "awel read\n",
            "a5er questions\n",
            "You: what is covid\n",
            "Answer : a new disease\n",
            "ana gowa\n",
            "i sense Happy\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "TW4YSTmRPvfh",
        "outputId": "17fc17f6-f8a6-438f-ced3-d7e774844638"
      },
      "source": [
        "video_stream()"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    var video;\n",
              "    var div = null;\n",
              "    var stream;\n",
              "    var captureCanvas;\n",
              "    var imgElement;\n",
              "    var labelElement;\n",
              "    \n",
              "    var pendingResolve = null;\n",
              "    var shutdown = false;\n",
              "    \n",
              "    function removeDom() {\n",
              "       stream.getVideoTracks()[0].stop();\n",
              "       video.remove();\n",
              "       div.remove();\n",
              "       video = null;\n",
              "       div = null;\n",
              "       stream = null;\n",
              "       imgElement = null;\n",
              "       captureCanvas = null;\n",
              "       labelElement = null;\n",
              "    }\n",
              "    \n",
              "    function onAnimationFrame() {\n",
              "      if (!shutdown) {\n",
              "        window.requestAnimationFrame(onAnimationFrame);\n",
              "      }\n",
              "      if (pendingResolve) {\n",
              "        var result = \"\";\n",
              "        if (!shutdown) {\n",
              "          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n",
              "          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n",
              "        }\n",
              "        var lp = pendingResolve;\n",
              "        pendingResolve = null;\n",
              "        lp(result);\n",
              "      }\n",
              "    }\n",
              "    \n",
              "    async function createDom() {\n",
              "      if (div !== null) {\n",
              "        return stream;\n",
              "      }\n",
              "\n",
              "      div = document.createElement('div');\n",
              "      div.style.border = '2px solid black';\n",
              "      div.style.padding = '3px';\n",
              "      div.style.width = '100%';\n",
              "      div.style.maxWidth = '600px';\n",
              "      document.body.appendChild(div);\n",
              "      \n",
              "      const modelOut = document.createElement('div');\n",
              "      modelOut.innerHTML = \"<span>Status:</span>\";\n",
              "      labelElement = document.createElement('span');\n",
              "      labelElement.innerText = 'No data';\n",
              "      labelElement.style.fontWeight = 'bold';\n",
              "      modelOut.appendChild(labelElement);\n",
              "      div.appendChild(modelOut);\n",
              "           \n",
              "      video = document.createElement('video');\n",
              "      video.style.display = 'block';\n",
              "      video.width = div.clientWidth - 6;\n",
              "      video.setAttribute('playsinline', '');\n",
              "      video.onclick = () => { shutdown = true; };\n",
              "      stream = await navigator.mediaDevices.getUserMedia(\n",
              "          {video: { facingMode: \"environment\"}});\n",
              "      div.appendChild(video);\n",
              "\n",
              "      imgElement = document.createElement('img');\n",
              "      imgElement.style.position = 'absolute';\n",
              "      imgElement.style.zIndex = 1;\n",
              "      imgElement.onclick = () => { shutdown = true; };\n",
              "      div.appendChild(imgElement);\n",
              "      \n",
              "      const instruction = document.createElement('div');\n",
              "      instruction.innerHTML = \n",
              "          '<span style=\"color: red; font-weight: bold;\">' +\n",
              "          'When finished, click here or on the video to stop this demo</span>';\n",
              "      div.appendChild(instruction);\n",
              "      instruction.onclick = () => { shutdown = true; };\n",
              "      \n",
              "      video.srcObject = stream;\n",
              "      await video.play();\n",
              "\n",
              "      captureCanvas = document.createElement('canvas');\n",
              "      captureCanvas.width = 640; //video.videoWidth;\n",
              "      captureCanvas.height = 480; //video.videoHeight;\n",
              "      window.requestAnimationFrame(onAnimationFrame);\n",
              "      \n",
              "      return stream;\n",
              "    }\n",
              "    async function stream_frame(label, imgData) {\n",
              "      if (shutdown) {\n",
              "        removeDom();\n",
              "        shutdown = false;\n",
              "        return '';\n",
              "      }\n",
              "\n",
              "      var preCreate = Date.now();\n",
              "      stream = await createDom();\n",
              "      \n",
              "      var preShow = Date.now();\n",
              "      if (label != \"\") {\n",
              "        labelElement.innerHTML = label;\n",
              "      }\n",
              "            \n",
              "      if (imgData != \"\") {\n",
              "        var videoRect = video.getClientRects()[0];\n",
              "        imgElement.style.top = videoRect.top + \"px\";\n",
              "        imgElement.style.left = videoRect.left + \"px\";\n",
              "        imgElement.style.width = videoRect.width + \"px\";\n",
              "        imgElement.style.height = videoRect.height + \"px\";\n",
              "        imgElement.src = imgData;\n",
              "      }\n",
              "      \n",
              "      var preCapture = Date.now();\n",
              "      var result = await new Promise(function(resolve, reject) {\n",
              "        pendingResolve = resolve;\n",
              "      });\n",
              "      shutdown = false;\n",
              "      \n",
              "      return {'create': preShow - preCreate, \n",
              "              'show': preCapture - preShow, \n",
              "              'capture': Date.now() - preCapture,\n",
              "              'img': result};\n",
              "    }\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MbpILxT0roah"
      },
      "source": [
        "#\n",
        " # what is  covid\n",
        "#video_stream()\n",
        "# process=[]\n",
        "# for _ in range(10)\n",
        "#   p1=Process(target=threaded_face_emotion)\n",
        "#   p1.start()\n",
        "\n",
        "# p2=Process(target=bert_answer)\n",
        "# p2.start()\n",
        "# p2.join()\n",
        "# p2.start()\n",
        "# p1.start()\n",
        "\n",
        "# p2.join()\n",
        "# p1.join()\n",
        "\n",
        "#face_emotion()\n",
        "# read_face_emotion()\n",
        "# t1=threading.Thread(target=face_emotion)\n",
        "# t2=threading.Thread(target=bert_answer)\n",
        "# t1.start()\n",
        "\n",
        "#t2.start()\n",
        "# time.sleep(500)\n",
        "#t1.join()\n",
        "#bert_answer()\n",
        "\n",
        "  #answer_this('what is covid cure')\n",
        "# answer_this('and where did covid start')\n",
        "# answer_this('what is the symptoms?')\n",
        "# answer_this('what is the incubation period?')\n",
        "# answer_this('how does corona spread?')\n"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5WdGJvqvzN32"
      },
      "source": [
        "\n",
        "\n",
        "# End of Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uDKzIEbgP5pv",
        "outputId": "b57445a5-d92a-463b-cc33-02e31bbd2e59"
      },
      "source": [
        "import concurrent.futures\n",
        "\n",
        "def foo():\n",
        "  return 2\n",
        "\n",
        "with concurrent.futures.ThreadPoolExecutor() as executor:\n",
        "    future = executor.submit(foo)\n",
        "    return_value = future.result()\n",
        "    print(return_value)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}