web_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "web_extraction.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amitsamantara/sentiment_analysis/blob/main/web_extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGTP9MtBA9HM"
      },
      "source": [
        "import requests\n",
        "import bs4\n",
        "from bs4 import BeautifulSoup\n",
        "from textblob import TextBlob"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZibY_N0mBKWA"
      },
      "source": [
        "response = requests.get(\"https://www.trustpilot.com/review/olacabs.com\")\n",
        "f = open(\"demofile.txt\", \"w\")"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EJs2jNgXCjNK"
      },
      "source": [
        "  i = 1\n",
        "  while True:\n",
        "      next_page = \"\"\n",
        "      \n",
        "      #getting the html script of the link save in \"response\"\n",
        "      soup = bs4.BeautifulSoup(response.content, \"html.parser\")\n",
        "      \n",
        "      #extracting the reviews from the html scripting\n",
        "      paras = soup.find_all('p', class_=\"review-content__text\")\n",
        "      \n",
        "      #print(paras)\n",
        "      data = []\n",
        "\n",
        "      #extracting individual review from list of reviews in a page\n",
        "      for para in paras:\n",
        "        buffer = \"\"\n",
        "        #removing <br/> tag from individual review script\n",
        "        for x in para:\n",
        "          if str(x) == \"<br/>\":\n",
        "            continue\n",
        "          buffer += str(x)\n",
        "        data.append(buffer)\n",
        "\n",
        "      for x in data:\n",
        "        f.write(x)\n",
        "        \n",
        "\n",
        "      #next page link\n",
        "      for link in soup.find_all('a',class_=\"button button--primary next-page\"):\n",
        "        next_page = \"https://www.trustpilot.com/\"+link.get('href')\n",
        "      \n",
        "      #break link search at the last page\n",
        "      if next_page == \"\":\n",
        "        break\n",
        "      \n",
        "      #updating response to the next page link\n",
        "      response = requests.get(str(next_page))\n",
        "      \n",
        "  #closing the file in which reviews are been stored    \n",
        "  f.close()   "
      ],
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gETy5KpswoGr"
      },
      "source": [
        "import spacy\n",
        "from spacytextblob.spacytextblob import SpacyTextBlob\n",
        "\n",
        "nlp=spacy.load('en_core_web_sm')\n",
        "nlp.add_pipe('spacytextblob')\n",
        "\n",
        "file = open(\"demofile.txt\", \"r\")\n",
        "\n",
        "data_line = file.readlines()\n",
        "\n",
        "output = []\n",
        "new_lis = []\n",
        "weight = []\n",
        "\n",
        "for element in data_line:\n",
        "  if element.strip():\n",
        "    new_lis.append(element)\n",
        "#extracting if the review is good or bad based on the ploarity by Spacy library\n",
        "for x in new_lis:\n",
        "  doc=nlp(x)\n",
        "  pred=doc._.polarity\n",
        "  weight.append(pred)\n",
        "  if pred<0.26:\n",
        "    output.append(\"Negative\")\n",
        "  elif pred>0.26:\n",
        "    output.append(\"Positive\")\n",
        "\"\"\"TO PRINT OUTPUT\"\"\"\n",
        "z=0\n",
        "for x, y in zip(new_lis, output):\n",
        "  print(z, y, x)\n",
        "  z+=1"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}