dataset_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "collections_df = pd.read_json('../data/data_search_e_collection.jsonl', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total of datasets = 46615\n"
     ]
    }
   ],
   "source": [
    "print(\"total of datasets = \"+str(len(collections_df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 46615 entries, 0 to 46614\n",
      "Data columns (total 7 columns):\n",
      "attribution    46615 non-null object\n",
      "data           46615 non-null object\n",
      "data_fields    46615 non-null object\n",
      "description    46615 non-null object\n",
      "id             46615 non-null object\n",
      "title          46615 non-null object\n",
      "url            46615 non-null object\n",
      "dtypes: object(7)\n",
      "memory usage: 2.5+ MB\n"
     ]
    }
   ],
   "source": [
    "collections_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TRAIN DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46615it [00:15, 2982.40it/s]\n"
     ]
    }
   ],
   "source": [
    "title_lenght = []\n",
    "description_lenght = []\n",
    "\n",
    "\n",
    "tknzr = TweetTokenizer()\n",
    "\n",
    "for i,row in tqdm(collections_df.iterrows()):\n",
    "    \n",
    "    title = str(row['title'])\n",
    "    description = str(row['description'])\n",
    "\n",
    "    lenght_title = len(tknzr.tokenize(title))\n",
    "    lenght_description = len(tknzr.tokenize(description))\n",
    "\n",
    "    title_lenght.append(lenght_title)\n",
    "    description_lenght.append(lenght_description)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "title analysis: - TRAIN DATA\n",
      "max words = 106\n",
      "min words = 1\n",
      "avg words = 13.803196396009868\n",
      "std words = 9.535812943518135\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('title analysis: - TRAIN DATA')\n",
    "print(\"max words = \"+str(np.max(title_lenght)))\n",
    "print(\"min words = \"+str(np.min(title_lenght)))\n",
    "print(\"avg words = \"+str(np.mean(title_lenght)))\n",
    "print(\"std words = \"+str(np.std(title_lenght)))\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHVBJREFUeJzt3XmUHXWZ//H3x7DK0ogJHAxLgEQWZZtpGJVVBQUhbMMWtjAjICqyKCh4+A04MyqbiIiAMOyHHQGJ7C4QFAdIkB0ZQyQksiRsgbAvz++P+l4omup7qztdXfd2f17n9Olb+3Orb9+nvkt9SxGBmZlZTx+pOwAzM2tPThBmZlbICcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgOpSk8yT9dz+33UHSTEnzJK030LH1IY5+vYcU9ypVxFTy+A9J2qzEeiFpbMWxbCzp0SbLx6Q4FqgyDhuanCAqIulxSZtXtf58OhE4MCIWj4i/DNIxB0yKe/pgHKsoiUXEpyLi1sE4fkE8H0g6EXF7RKyWWz4onyNJe6REPU/Sa5LezU3Py8XyWpr3dDqXixfs65j0vjboMX8fSX/s8d6ekbRYbt6+km7tJcZGcmzE9Yyk30jaopf1b5X0gqSFc/NuyG3/lqQ3c9Nn5NZbOZ2D0/pwGtueE8TwtBLw0GAdrFOvXiWNqDuGdhURF6VEvTiwFfBkYzrNaxifptcF1gOOzO9HkoC9gOeBiSUOvQBwcB/DXSrFsA5wC3C1pH16xDEG2BgIYNvG/IjYKveeLgKOz73PA3K72Bt4Adgtn2A6nRNEBSRdCKwITEpXGt9N87dN1RMvpquVNVqsf0W68porabKkT5U8/kckHSVphqTZki6Q1CVp4XR1NwK4T9JjBdv+QNLP0+sFJb0i6fg0vaik1yV9rNn7Scsel/Q9SfcDr0haQNJ6ku6R9LKky4BFcuuPTFd3L0p6XtLtkgo/n/mr6HRV+gtJ16X93ilp1SbnptdzmvZ1uqTrJb0CfBXYA/hu+rtMyr23zdPrEZK+L+mxdPypklYoOO7Ckk6U9ES6kj1D0qJ9ee+SJqeX96V4dpW0maRZaXnh56jHProknS3pKUn/kPTfg5EII+Jp4CayRJG3MfAJsi/93SQt1GJXJwCHSVqqPzFExM+AY4DjepzjvYH/Bc6jXKLqaW/gKOAtYHw/tm9LThAViIi9gCdIV08RcbykTwKXAIcAo4Dryf6RFypaP+3qBmAcsAxwD9kVTBn7pJ/PA6sAiwOnRsQbuau7dSKi6Iv0NmCz9Hp94Glg0zT9WeDRiHih2fvJ7WsCsDWwFNln7RrgQmBp4ArgX3PrfgeYlfa1LPB9squ5MiYAPwA+BkwDfthk3VbndPe0/RLABXzwqrHoH//b6fhfAZYE/h14tWC944BPkn1BjgVGA/+RlpV67xGxSXq5Tornsh7Le/sc5Z0PvJ1iWA/4ErBvwXoDStLyZCWNaT0WTQQmAY33sk2LXU0BbgUOm49wriL7+6+Wm7c32d/6IuDLkpYtuzNJGwPLA5cCl6d9DQlOEINnV+C6iLglIt4iawdYFPhcbxtExDkR8XJEvEF21bOOpK4Sx9oDOCkipkfEPLJi/W4lq3r+DIyT9HFgE+BsYLSyuuNNyRJI2fdzSkTMjIjXgM8ACwInR8RbEXElcHdu3beA5YCV0vLbo/xAYVdFxF0R8TbZP3jPq9T3lDinv46IP0XEuxHxeolj7wscFRGPRua+iHguv0KqRtkPODQino+Il4EfAbsNwHsvLX3pbQUcEhGvRMRs4Ke5OKpwjaSXgZnAbODoXDwfBXYGLk6foSspd/X+H8C3JI3qZ0xPpt9Lpzg2Iqt2vTwipgKPkV0olDURuCEiXgAuBraStEw/Y2srThCD5xPAjMZERLxL9k8zumjlVHVxbKq6eAl4PC0a2ddjpdcLkF2dNpW+zKeQJYNNyBLCHcCGfDBBlHk/M3vE9I8eX3z5GE8gu7q8WdJ0SUe0ijXn6dzrV8lKTB9S8pzO/PCWTa1A9oXSzCjgo8DUVI30InBjmg/z9977YiWyJP1ULo5fkl1Nf4hyjc6SVuznMbePiCXISqWr88FzvQNZaeb6NH0R2Zdr0y/+iHgQ+A3Q3/PU+Iw+n35PBG6OiGfT9MWUrGZK1YQ7k0qiEfFnslJcXxJM23KCqE7PK8Anyf5BgfeuKlcA/tHL+rsD2wGbA13AmMamJY79gWOR1Uu/DTxTYlvIksAXyKog7k7TXwY2ABr14K3eD3zwPT1FVhLJx//el066qv9ORKxCVof7bUlfLBlvWWXOac+/Q6sr+ZlAr20eybPAa8CnImKp9NPVqO4b4PfeLN6ZwBvAyFwcS0ZEYdtWvtE5Ip7oZzyNfd1GVr9/Ym72RLJk/oSkp8mqHRckq7Jr5WiyUlnhBVYLO5CVZh5NX/C7AJumtqmngUPJSpbrlNzXksBpue1HM0SqmZwgqvMMWf1/w+XA1pK+KGlBsnrnN8iuzovWXyItf47s6vNHfTj2JcChyrreLZ62vSxVwZRxG9kH/OGIeJOszndf4O8RMafk++npz2RJ6iBlDdY7kiUcACRtI2lsSiAvAe+kn4HUn3Pa8+/S0/8A/yVpnDJrp+q596TS1VnATxtVD5JGS/pyet2X994qnl6XR8RTwM3ATyQtqawzw6qSNi1avwInA1tIWlfSaOCLZG0O66afdcjaalpevUfENLJ2i4PKHlzSspIOJEsuR6a/y/Zk53rNXBxrALdT7kt+InAOsFZu+w2BdSWtVTa2duUEUZ0fA0elovxhEfEosCfwc7IryvFkjYlvFq1P1kA6g+yK/GGyHhZlnUPWGDwZ+DvwOvCtPmx/B1l7QqO08HDaR2OaEu/nA9L8Hckaz18ga8O4KrfKOOC3wDyyZHJaBfca9Oecng2smf4u1xQsP4ksWd5M9uV+Ntm56+l7ZNVI/5uqt37L+42kfXnvxwDnp3h2KVje83PU097AQmTv/wWyev/lejnWgEoXFxcA/4+sa+u9EXFz6l30dOrpdAqwtqRPl9jlfwKLtVwLXlTWK+0Bss4EO0fEOWnZRODciHiiRxynAns0a7fLJbmT89umdowb6V9vqLaiCtrCzMxsCHAJwszMCjlBmJlZIScIMzMr5ARhZmaFOnIQtYaRI0fGmDFj6g7DzKyjTJ069dmIaHknekcniDFjxjBlypS6wzAz6yiSZrRey1VMZmbWCycIMzMr1JEJQtJ4SWfOnTu37lDMzIasjkwQETEpIvbv6ioz8rWZmfVHRyYIMzOrnhOEmZkVcoIwM7NCThBmZlaoo2+Us+qMOeK6lus8fuzWgxCJmdWlI0sQ7uZqZla9jkwQ7uZqZla9jkwQZmZWPScIMzMr5ARhZmaFnCDMzKyQE4SZmRVygjAzs0IdmSB8H4SZWfU6MkH4Pggzs+p1ZIIwM7PqOUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMDOzQn6inA0ZrZ6C5yfgmfVNR5YgfCe1mVn1OjJB+E5qM7PqdWSCMDOz6jlBmJlZITdSW0do1QBtZgPPJQgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMDOzQu7FZMNGmZ5QHo7D7H0uQZiZWaGOTBAei8nMrHodWcUUEZOASd3d3fvVHctgchWJmQ2mjixBmJlZ9TqyBGFWFZfSzN7nEoSZmRVygjAzs0JOEGZmVshtEGZ95HYKGy5cgjAzs0JOEGZmVshVTFY7Py3OrD25BGFmZoWcIMzMrJCrmKzf3JvHbGhzgrBKuX3BrHO5isnMzAo5QZiZWSEnCDMzK9SRCcJPlDMzq15HJoiImBQR+3d1ddUdipnZkNW0F5OkRYBtgI2BTwCvAQ8C10XEQ9WHZ2Zmdek1QUg6BhgP3ArcCcwGFgE+CRybksd3IuL+6sM06yy+R8SGgmYliLsj4phelp0kaRlgxYEPyczM2kGvCSIiml4CRcRsslKFmZkNQS3vpJY0CYges+cCU4BfRsTrVQRmZmb1KtOLaTowDzgr/bwEPEPWFnFWdaGZmVmdyozFtF5EbJKbniRpckRsIsk9mTqQx0cyszLKlCBGSXqvMTq9Hpkm36wkKjMzq12ZEsR3gD9KegwQsDLwDUmLAedXGZyZmdWnZYKIiOsljQNWJ0sQf801TJ9cZXBmZlafMr2YFgS+BjTaIW6V9MuIeKvSyMzMrFZlqphOBxYETkvTe6V5+1YVlJmZ1a9Mglg/ItbJTf9e0n1VBWRmZu2hTIJ4R9KqEfEYgKRVgHeqDcts6PN4TdbuyiSIw4E/SJpO1ki9EvBvlUZlZma1K9OL6XepF9NqvN+L6Y3KIzMzs1o1G+57x14WrSqJiLiqopjMzKwNNCtBjG+yLAAnCDOzIazZcN9uZzAzG8Z6HYtJ0p6Smi1fVdJG1YRlZmZ1a1bF9HHgL5KmAlOBOWSPHB0LbAo8CxxReYRmZlaLZlVMP5N0KvAFYENgbeA14BFgr4h4YnBCNDOzOjTt5hoR7wC3pB8zMxtGyjwPwszMhiEnCDMzK9Q2CULSGpLOkHSlpK/XHY+Z2XDXMkFIOljSksqcLekeSV8qs3NJ50iaLenBHvO3lPSopGmSjgCIiEci4gBgF6C7P2/GzMwGTpkSxL9HxEvAl4BRZAP1HVty/+cBW+ZnSBoB/ALYClgTmCBpzbRsW+CPwO9K7t/MzCpSJkEo/f4KcG5E3Jeb11RETAae7zF7A2BaREyPiDeBS4Ht0vrXRsTngD16DUbaX9IUSVPmzJlTJgwzM+uHMsN9T5V0M7AycKSkJYB35+OYo4GZuelZwL9I2gzYEVgYuL63jSPiTOBMgO7u7piPONpKmWcD2PDjZ0ZYncokiK8C6wLTI+JVSR9n/p4HUVT6iIi4Fbh1PvZrZmYDqEwVU5C1FRyUphcjG3Kjv2YBK+SmlweenI/9mZlZBcokiNOAzwIT0vTLZI3M/XU3ME7SypIWAnYDrp2P/ZmZWQXKJIh/iYhvAq8DRMQLwEJldi7pEuDPwGqSZkn6akS8DRwI3EQ2rtPlEfFQX4KWNF7SmXPnzu3LZmZm1gdl2iDeSl1TA0DSKEo2UkfEhF7mX0+ThugS+50ETOru7t6vv/swM7PmypQgTgGuBpaR9EOy+xR+VGlUZmZWu5YliIi4KD0T4otkPZC2j4hHKo/M+sXdZc1soPSaICQtnZucDVySXxYRPW+AMzOzIaRZCWIqWbtD476Fxk1pSq9XqTCupiSNB8aPHTu2rhDMzIY8RXTuzcjd3d0xZcqUusMYEK4asir5bmvLkzQ1IloOilpmNNcdJHXlppeStP38BmhmZu2tTC+moyPivRsOIuJF4OjqQjIzs3ZQJkEUrVPm/gkzM+tgZRLEFEknSVpV0iqSfkrWgG1mZkNYmQTxLeBN4DLgCrIhN75ZZVCteKgNM7PqlblR7hXgiEGIpTQPtWFmVr1mN8qdHBGHSJrE+/dAvCcitq00MjMzq1WzEsSF6feJgxGImZm1l14TREQ0GqLXjYif5ZdJOhi4rcrAzGxw+fGm1lOZRuqJBfP2GeA4zMyszTRrg5gA7A6sIin/xLclgOeqDszMzOrVrA3iDuApYCTwk9z8l4H7qwyqFQ/WZ9Y3HuvL+qNZG8QMSbOAVyKirdob3M3VzKx6TdsgIuId4NX8YH1mZjY8lBlT6XXgAUm3AK80ZkbEQZVFZWZmtSuTIK5LP2ZmNoyUGWrj/MEIxMzM2kvLBCFpHPBjYE1gkcb8iKjtkaNmZla9MjfKnQucDrwNfB64gPeH4TAzsyGqTBvEohHxO0mKiBnAMZJup8anynXafRDug25mnahMCeJ1SR8B/ibpQEk7AMtUHFdTETEpIvbv6nLvWzOzqpRJEIcAHwUOAv4Z2JPi8ZnMzGwIKdOL6e70ch7wb9WGY2Zm7aJMCcLMzIYhJwgzMyvUa4KQdFz6vfPghWNmZu2iWQniK5IWBI4crGDMzKx9NGukvhF4FlhM0kuAgGj8joglByE+MzOrSa8liIg4PCK6gOsiYsmIWCL/exBj/BBJ4yWdOXfu3DrDMDMb0hQRrVeSlgXWT5N3RsScSqMqqbu7O6ZMmVJ3GC35TmobTh4/duu6Q7AWJE2NiO5W67XsxZQaqe8CdgZ2Ae6StNP8h2hmZu2szFhMRwHrR8RsAEmjgN8CV1YZmJmZ1avMfRAfaSSH5LmS25mZWQcrU4K4UdJNwCVpelfg+upCMjOzdlBmLKbDJe0IbETWxfXMiLi68sjMzKxWZUoQRMRVwFUVx2JmZm3EbQlmZlbICcLMzAo5QZiZWaFe2yAkPUA29tKHFpGNxbR2ZVGZmVntmjVSbzNoUZjZkFFmaBkPx9EZek0QETFjMAPpC0njgfFjx46tOxQzsyGrZTdXSZ8Bfg6sASwEjABeqXNE14iYBEzq7u7er64YGjwQn5kNVWUaqU8FJgB/AxYF9iVLGGZmNoSVvVFumqQREfEOcK6kOyqOy8zMalYmQbwqaSHgXknHA08Bi1UblpkNZW7I7gxlqpj2SusdCLwCrADsWGVQZmZWvzIJYvuIeD0iXoqIH0TEt3EXWDOzIa9MgphYMG+fAY7DzMzaTLM7qScAuwMrS7o2t2hJsocGmZnZENaskfoOsgbpkcBPcvNfBu6vMigzM6tfqzupZwCflbQssH5a9EhEvD0YwZmZWX1atkFI2hm4C9gZ2AW4U9JOVQdmZmb1KnMfxFHA+hExG0DSKOC3wJVVBmZmZvUq04vpI43kkDxXcjszM+tgZUoQN0q6CbgkTe8K3FBdSGZm1g5aJoiIOFzSjsBGZA8LOjMirq48MjMzq1WZ4b6Pi4jvAVcVzDMzsyGqTFvCFgXzthroQMzMrL00u5P668A3gFUk5W+MWwL4U9WBNeMnypkZeFTYqjUrQVwMjAeuTb8bP/8cEXsOQmy9iohJEbF/V1dXnWGYmQ1pze6kngvMJXuanJmZDTO+n8HMzAo5QZiZWSEnCDMzK+QEYWZmhcoMtWFmNujKdGG1arkEYWZmhZwgzMyskBOEmZkVchuEmQ17HrKjmEsQZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMDOzQk4QZmZWyAnCzMwKOUGYmVkhj8VkZkOanyvRfy5BmJlZIZcgzMzaTLuMLusShJmZFWqbEoSk7YGtgWWAX0TEzTWHZGb2nna5qh9MlZYgJJ0jabakB3vM31LSo5KmSToCICKuiYj9gH2AXauMy8zMWqu6iuk8YMv8DEkjgF8AWwFrAhMkrZlb5ai03MzMalRpgoiIycDzPWZvAEyLiOkR8SZwKbCdMscBN0TEPb3tU9L+kqZImjJnzpzqgjczG+bqaKQeDczMTc9K874FbA7sJOmA3jaOiDMjojsiukeNGlVtpGZmw1gdjdQqmBcRcQpwymAHY2ZmxeooQcwCVshNLw88WUMcZmbWRB0J4m5gnKSVJS0E7AZc25cdSBov6cy5c+dWEqCZmVVcxSTpEmAzYKSkWcDREXG2pAOBm4ARwDkR8VBf9hsRk4BJ3d3d+w10zGZm/TXU7pWoNEFExIRe5l8PXF/lsc3MbP54qA0zMyvkBGFmZoWcIMzMrFBHJgj3YjIzq15HJoiImBQR+3d1ddUdipnZkNU2w32bmQ0HnfQI1I4sQZiZWfWcIMzMrFBHJgg3UpuZVa8jE4Qbqc3MqteRCcLMzKrnXkxNdFJvAzOzgeYShJmZFXKCMDOzQk4QZmZWqCMThLu5mplVryMThLu5mplVryMThJmZVc8JwszMCjlBmJlZIScIMzMrpIioO4Z+kzQHmNHPzUcCzw5gOIPBMVev0+IFxzxYOi3mZvGuFBGjWu2goxPE/JA0JSK6646jLxxz9TotXnDMg6XTYh6IeF3FZGZmhZwgzMys0HBOEGfWHUA/OObqdVq84JgHS6fFPN/xDts2CDMza244lyDMzKwJJwgzMys0LBOEpC0lPSppmqQj6o6nDEmPS3pA0r2SptQdTxFJ50iaLenB3LylJd0i6W/p98fqjDGvl3iPkfSPdJ7vlfSVOmPsSdIKkv4g6RFJD0k6OM1vy/PcJN62Pc+SFpF0l6T7Usw/SPNXlnRnOseXSVqo7lgbmsR8nqS/587zun3a73Brg5A0Avg/YAtgFnA3MCEiHq41sBYkPQ50R0Tb3qgjaRNgHnBBRHw6zTseeD4ijk3J+GMR8b0642zoJd5jgHkRcWKdsfVG0nLAchFxj6QlgKnA9sA+tOF5bhLvLrTpeZYkYLGImCdpQeCPwMHAt4GrIuJSSWcA90XE6XXG2tAk5gOA30TElf3Z73AsQWwATIuI6RHxJnApsF3NMQ0JETEZeL7H7O2A89Pr88m+HNpCL/G2tYh4KiLuSa9fBh4BRtOm57lJvG0rMvPS5ILpJ4AvAI0v2rY5x9A05vkyHBPEaGBmbnoWbf6BTQK4WdJUSfvXHUwfLBsRT0H2ZQEsU3M8ZRwo6f5UBdUWVTVFJI0B1gPupAPOc494oY3Ps6QRku4FZgO3AI8BL0bE22mVtvve6BlzRDTO8w/Tef6ppIX7ss/hmCBUMK8T6tk2jIh/ArYCvpmqR2zgnQ6sCqwLPAX8pN5wiklaHPgVcEhEvFR3PK0UxNvW5zki3omIdYHlyWod1ihabXCjaq5nzJI+DRwJrA6sDywN9KnacTgmiFnACrnp5YEna4qltIh4Mv2eDVxN9qHtBM+keuhGffTsmuNpKiKeSf9o7wJn0YbnOdUx/wq4KCKuSrPb9jwXxdsJ5xkgIl4EbgU+AywlaYG0qG2/N3Ixb5mq+CIi3gDOpY/neTgmiLuBcalHwkLAbsC1NcfUlKTFUgMfkhYDvgQ82HyrtnEtMDG9ngj8usZYWmp8ySY70GbnOTVGng08EhEn5Ra15XnuLd52Ps+SRklaKr1eFNicrO3kD8BOabW2OcfQa8x/zV00iKzNpE/nedj1YgJIXepOBkYA50TED2sOqSlJq5CVGgAWAC5ux5glXQJsRjbM8DPA0cA1wOXAisATwM4R0RYNw73EuxlZtUcAjwNfa9TttwNJGwG3Aw8A76bZ3yer12+789wk3gm06XmWtDZZI/QIsovoyyPiP9P/4aVkVTV/AfZMV+a1axLz74FRZFXr9wIH5BqzW+93OCYIMzNrbThWMZmZWQlOEGZmVsgJwszMCjlBmJlZIScIMzMr5ARhQ4qkpSR9o8R6YyTtXnK9vvUdl05II2qe0Jft+iuN2LlT6zXN+sYJwoaapYCWCQIYA7RMEP30NeCfIuLwgd5x7k5es8o5QdhQcyywahr7/gRlTpD0oLLnaeyaW2/jtN6hqaRwu6R70s/nmh2kt/1KuhZYDLgzd6zGNg+kEo4kPSdp7zT/QkmbKxvT/9y03l8kfT4t30fSFZImkQ3YKEmnSnpY0nXkBuaTdGyaf7+kthtK2zqLr0ZsqDkC+HQatAxJ/0p2x+46ZHdM3y1pclrvsIjYJq33UWCLiHhd0jjgEqC7yXF2LNpvRGwraV7j+D38CdgQmAFMBzYGLiAb5+frwDcBImItSauTJYNPpm0/C6wdEc9L2hFYDVgLWBZ4GDhH0tJkw1asHhHRGHrBrL9cgrChbiPgkjQw3DPAbWQjW/a0IHCWpAeAK4A1B2i/ebcDm6Sf04G1JI0me9DPvLTPCwEi4q9kiaSRIG7JDZ2xSe7YTwK/T/NfAl4H/iclkVdbxGPWlBOEDXVFw7sXOZRsPKZ1yEoOrR4nWXa/eZPJSg0bk422OYds8LfbS+zzlR7THxojJz2rYAOykVO3B27sR4xm73GCsKHmZWCJ3PRkYFdlD1MZRXb1fVfBel3AU2n46b3IBj1rprf99ioiZpJVR42LiOlkj4U8jPcTxGRgD4BUtbQi8Ggvx94tHXs5oNFWsTjQFRHXA4eQVYGZ9ZvbIGxIiYjnJP0pdU29AfguWf39fWRX3d+NiKclPQe8Lek+4DzgNOBXknYmG9a55xV7T1cX7bdEiHfyfvK5HfgxWaIgxXBGquZ6G9gnIt7IRmr+0LG/QDZC6v+RVW9BlvB+LWkRstLIoSXiMeuVR3M1M7NCrmIyM7NCThBmZlbICcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgzMys0P8H2HUSrZF/ygkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(title_lenght, range(0,35))\n",
    "plt.title('total of words in articles title - TRAIN DATA')\n",
    "plt.xlabel('total of words')\n",
    "plt.ylabel('total of articles (log)')\n",
    "plt.yscale('log')\n",
    "plt.savefig('article_title1.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "description analysis: - TRAIN DATA\n",
      "max words = 3306\n",
      "min words = 0\n",
      "avg words = 117.69125817869785\n",
      "std words = 100.90804742928448\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('description analysis: - TRAIN DATA')\n",
    "print(\"max words = \"+str(np.max(description_lenght)))\n",
    "print(\"min words = \"+str(np.min(description_lenght)))\n",
    "print(\"avg words = \"+str(np.mean(description_lenght)))\n",
    "print(\"std words = \"+str(np.std(description_lenght)))\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XncXGV99/HPl7AKGMrmowEMkIiCBaw3KK6oqIDEBaESlUVR6kK1qCBU+lS7CdUqUlGMglQeCoKFSiAKFAyIUCBR2aUCZYksCVsQEDH4e/64riGHyblnztz3nJl7Zr7v12teM2f/XWdmzu+c6zqLIgIzM7Nmq/U7ADMzm5qcIMzMrJQThJmZlXKCMDOzUk4QZmZWygnCzMxKOUHUTNIpkv5hgtO+S9Ldkh6T9LJux9ZBHBMqQ457qzpiqrj8GyXtWmG8kDSr5lgWSvpQnctoWt4Wef1Pm+D075N0YbfjssEy8glC0h2Sdqtr/En6MnBoRKwXEb/o0TK7Jsd9ey+WVZbEImK7iFjYi+VPNRFxV17/T7cbV9LMnCRXL0x/WkS8pd4oV4mjkdQar5D0eKH7tfl7fip3PyTpIkkvLpnXrnn6I5r6P6useX4haefCOLMkjXuBWN4G/E7SbyU9IukKSR+RtMr2VNLnS+b/14UyPSnp6UL3jYXxJOl2STd1vja7Y+QTxBT3QuDGtmN1SXEDMUgmupc8rAb1eywktfUiYr3ce4dCv5/mfv+ch88AfgOcVDK7A4GH8ns7DwGdHiHPiYj1Sf/RY4DPNschScD+zXFExD8VyvgR4MpCGbcrzOJ1wKbAVpJ26jC+rhjpBCHpVGALYH7O3kfk/m/P1ROP5KqBl7QZ/yxJ90laLukySduNt8ym5a8m6WhJd0paKul7kqZLWkvSY8A04FpJt5VM+wVJ/5o/r5H3tP45d6+T90z+pFV58rA7JH1W0nXA45JWl/QyST/Pe0jfB9YujL+xpPPyvB6S9NOyPac87jNVN3lP7QRJ5+f5XiVp6xbrZtx1muf1TUkLJD0OHAy8Dzgify/zC2XbLX+elvfcbsvLXyxp85LlriXpy5LuknS/pBMlrTOBsr9Z0q9y/F8H1DT8g5JulvSwpAskvTD3l6Sv5t/DcknXSXpp4Xv9l/x7WS7p8tyvsVd8sKS7gEu06p7yQklflHR1nvaHkjbM4VyW3x/J628XSQdJurwQ76skXZOnvUbSqwrDFkr6e0k/y+v2Qkkbj/fddktE/A44E9ix2F/Sc4B9gI8DsyWNtZnVvwHbS3r9BGJYHhHnAu8BDmx8V9lrgRcAnwT2k7Rmh7M/EPghsIBqia77ImKkX8AdwG6F7hcBjwNvBtYAjgBuBdYsGz/3+yCwPrAWcBzwy8KwU4B/GGfZH8zz3gpYDzgbOLUwPIBZ40z7RuD6/PlVwG3AVYVh13ZQnl8CmwPrAGsCdwKH5fH3Af7QKAPwReDEPGwN0p9A48T4TPx5PTwE7AysDpwGnNHie2m3TpcDrybt5Kxdtp6L3xVwOHA9sA1pY70DsFFJnMcB5wIb5uXPB77YSdmBjYFH87pbI6/LFcCH8vB35u/gJXldHA1ckYe9FVgMbJDjfAnw/DzsBGAhac95Wv7e1wJm5jJ8D1g3f4+NfqvnaReS9rZfmsf5D+D/5WHPGjf3Owi4PH/eEHiYtDe8OjA3d29UmPdtpN/aOrn7mC78N1f5/Re/51yOU8m/9cI4+wP35nU0Hzi+MKx5vZxCOnr4RKG8s4Cous0o9L8L+Gih+yRSAlsDeBDYu2SaZ9ZzU//n5N/QnsC7gQfI/9levvq+ge73q/nLBv4GOLPQvVr+Y+3a6sdRGH+D/AOcXvwBjjPuxcDHCt3bkDbGjR9vqwSxDvAksBFwJPDXwBJSovlC409RsTwfLAx/HXAPhQ0fcEXhT/l3pL2a0riaYmxOEN8pDNsT+FXF76hsnX6vaZxV1jPPThC3AO9oFSdpg/w4sHVh2C7A/3ZSduAA4L8L3crfTSNB/Ag4uOk7eYJUXfFG4H+AVwKrNY3zO1KVS/PyZuYybFXSr5ggjikM3xZ4irQRfda4efhBrNxg7g9c3bTMK4GDCvM+ujDsY8CPu/DfHC9BPAk8AvwR+F9g+6Zx/gs4Ln+eCywD1hhnvZxCShBrkTbwezDxBPHfwOfy58YG/p25+1vAD0umeWY9N/V/f4579RzbI8C7JrtOO32NdBXTOF5A2oMGICL+CNxN2mtbRa66OCZXXTxK+vFA2ovsaFn58+rA89pNGOnwehHwetJG/VLShvzVud+lHZTn7qaYfhP5V1qIq+FLpL3fC5Ua0I5sF2vBfYXPT5CS2SoqrtO7V52ypc1Je7mtbEL6Yy/O1UiPAD/O/aF62V9QjC+vy2K8LwS+VljGQ6QkMiMiLgG+TjpauF/SPEnPJZV97TZlaLdOisPvJO3ZTuR32pi++Buq+t3+SCsbZN9XYdllvhwRG5A29r8j7Vg15r858AbSESqkhL428LZWM4yI3wN/n19qNW4LM0jfJcC7SEeNC3L3acAekjYpm7DEgaQduxU5trPpQzWTE0Tamyi6h/QHBp5paNqctNddNv57gXcAuwHTST9aqPYje9aySO0bK4D7K0wLKQm8EXgZcE3ufiupGqdRr9yuPPDsMt0LzMjjFeNKI0b8NiI+HRFbAXOAT0l6U8V4q6qyTpu/h+buZncD47Z5ZA+QNjjbRcQG+TU9coNpB2W/l7SOU9Ar13kxlr8oLGODiFgnIq7Iyzk+Il4ObEeqtjk8x/ZkmzK0WwfFGLYgHa0+UGG65t9pY/rflIzbUkTsESsbZE9rP0XLed1Fqt//WqOdiHS0sxqpnfA+4HZSgjigwiy/S/q9vavTWJQakWcAjXabA0lJ8q4cx1mkhDy3wrw2I/2v35/b4e4jVVfu2Yu2nSIniLQxLp6rfybwNklvkrQG8Gng96S987Lx18/DHyTtff5TB8s+HThM0paS1svTfj8iVlSc/lLSD/+miHiKdKj/IVKVyLKK5Wl2JSlJfUKpwXpvUsIBQNJeSqcBinQI/XR+ddNE1mnz99LsO8DfS5qdG4K3l7RRcYR8dPVt4KuSNgWQNEPSW/PnqmU/H9hO0t65kfgTwP8pDD8ROEq54V3pxIR98+edJL0if1ePk5LC0zm2k4GvSHpBPsraRdJaFdZNw/slbZsbcf8O+EGk02CXkaprxlt/C4AXSXpv/k28h1RFdV4Hy65FRFxESmCH5F4HkKpYdyy83k36D2xUOpOV81oBfJ50RlIlkp4raS/gDFKbzvWSZgBvAvYqxLADcCzVjgL2J1UzblOY/kWkasq2CaabnCBSw+PR+XD/MxFxC6n+719Je1dzSKe0PVU2Pqlh8E7S3tRNpHrIqk4mNbJdRqpLfRL4yw6mv4LUFtE4Wrgpz6PRTYXyPEvuvzepbvRh0tkZZxdGmU2q432MlEy+Ed2/1mAi6/QkYNv8vfxnyfCvkJLlhaSN+0mkddfss6RqpP/O1Vv/xcoqjEplj4gHgH1Jpz8+mKf7WWH4OaSNxRl5GTeQ6r4BnktKUg/ndfAg6XoYgM+QGtqvIVVlHEtn/+FTSXXu95H2qj+R43kC+EfgZ3n9vbKpPA+SNnafzvEcAeyVyzkVfIl0BtvrSUebJ0TEfYXXuaTvtMrG9XTSEWA78yX9lnQ0+DnS7+sDedj+pJMqLizGARxPOlvqpeWzfMaBpN/WfU3Tn0iPq5n07KpmMxtGkhaS9nC/0+9YbHD4CMLMzEo5QZiZWSlXMZmZWSkfQZiZWamBvKlXw8YbbxwzZ87sdxhmZgNl8eLFD0RE24v2BjpBzJw5k0WLFvU7DDOzgSKp+cr4Uq5iMjOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMyvlBGFmZqWcIMzMrJQThJmZlZpSCULSupIW5wdwWBfMPPL8fodgZgOq1gQh6WRJSyXd0NR/d0m3SLq16bm+nyU91MXMzPqs7iOIU4Ddiz0kTSM9kH0P0mML5+bHIO5GenpY1ecxm5lZjWq9F1NEXCZpZlPvnYFbI+J2AElnkB5Qvx6wLilp/E7Sgvwc3meRdAj5+bNbbLFFfcGbmY24ftysbwbpOa4NS4BXRMShAJIOAh4oSw4AETEPmAcwNjbmh1mYmdWkHwlCJf2e2dBHxCltZyDNAebMmjWri2GZmVlRP85iWgJsXujeDLinkxlExPyIOGT69OldDczMzFbqR4K4BpgtaUtJawL7Aef2IQ4zM2uh7tNcTweuBLaRtETSwRGxAjgUuAC4GTgzIm7scL5zJM1bvnx594M2MzOg/rOY5o7TfwGwYBLznQ/MHxsb+/BE52FmZq1NqSupzcxs6hjIBOEqJjOz+g1kgvBZTGZm9RvIBGFmZvUbyAThKiYzs/oNZIJwFZOZWf0GMkGYmVn9nCDMzKzUQCYIt0GYmdVvIBOE2yDMzOo3kAnCzMzq5wRhZmalnCDMzKzUQCYIN1KbmdVvIBOEG6nNzOo3kAnCzMzq5wRhZmalnCDMzKyUE4SZmZUayAThs5jMzOo3kAnCZzGZmdVvIBOEmZnVzwnCzMxKOUGYmVkpJwgzMyvlBGFmZqWcIMzMrNRAJghfB2FmVr+BTBC+DsLMrH4DmSDMzKx+ThBmZlbKCcLMzEo5QZiZWSknCDMzK+UEYWZmpZwgzMyslBOEmZmVcoIwM7NSThBmZlZqyiQISS+RdKKkH0j6aL/jMTMbdbUmCEknS1oq6Yam/rtLukXSrZKOBIiImyPiI8CfA2N1xmVmZu21TBCS1pa0j6SvSTpL0vckHSFpu4rzPwXYvWme04ATgD2AbYG5krbNw94OXA5c3GE5zMysy8ZNEJI+D/wM2AW4CvgWcCawAjhG0kWStm8184i4DHioqffOwK0RcXtEPAWcAbwjj39uRLwKeF+LuA6RtEjSomXLlrUrn5mZTdDqLYZdExGfH2fYVyRtCmwxgWXOAO4udC8BXiFpV2BvYC1gwXgTR8Q8YB7A2NhYTGD5ZmZWwbgJIiLObzVhRCwFlk5gmSqfXSwEFlaagTQHmDNr1qwJLN7MzKpodQQBgKT5QPOe+nJgEfCtiHiyw2UuATYvdG8G3NPJDCJiPjB/bGzswx0u28zMKqpyFtPtwGPAt/PrUeB+4EW5u1PXALMlbSlpTWA/4NwJzMfMzGrU9ggCeFlEvK7QPV/SZRHxOkk3tppQ0unArsDGkpYAfxsRJ0k6FLgAmAacHBEt51MyX1cxmZnVrEqC2ETSFhFxF4CkLYCN87CnWk0YEXPH6b+AFg3R7biKycysflUSxKeByyXdRmpg3hL4mKR1gX+rM7jx+AjCzKx+bRNERCyQNBt4MSlB/KrQMH1cncG1iMlHEGZmNatyFtMawF8AjXaIhZK+FRF/qDUyMzPrqypVTN8E1gC+kbv3z/0+VFdQZmbWf1USxE4RsUOh+xJJ19YVUBVugzAzq1+V6yCelrR1o0PSVsDT9YXUXkTMj4hDpk+f3s8wzMyGWpUEcTjwE0kLJV0KXEI6s8kKZh7Z8s4kZmYDp8pZTBfns5i2YeVZTL+vPTIzM+urcROEpL3HGbS1JCLi7JpiasttEGZm9Wt1BDGnxbAA+pYgfB2EmVn9Wt3u+wO9DMTMzKaWVk+Ue7+kVsO3lvSaesIyM7N+a1XFtBHwC0mLgcXAMmBtYBbweuAB4MjaIywxFdogZh55Pncc87a+Ld/MrG7jHiFExNeAPwNOBzYB3pS7fwPsHxHvjohf9yTKVWPzdRCT4FNyzayKlqe5RsTTwEX5ZWZmI6TKhXIjr1973N7TN7N+coLokDfaZjYqnCBacDIws1HWNkFI+qSk5yo5SdLPJb2lF8G1iGmOpHnLly+vfVlOEmY2qqocQXwwIh4F3kI6m+kDwDG1RtXGVD2LycnEzIZJlQSh/L4n8N2IuLbQz8zMhlSVBLFY0oWkBHGBpPWBP9Yb1tTkIwQzGyVVEsTBpCumd4qIJ4A1SdVMNgFOMmY2KKokiAC2BT6Ru9cl3XJjqE1mQz5sSWDYymNm1VRJEN8AdgHm5u7fAifUFtGA8cbTzIZVlQTxioj4OPAkQEQ8TKpmGilOBNV5XZkNhyoJ4g+SppGqmpC0CSPaSF0nb1TNbKqpkiCOB84BNpX0j8DlwD/VGlUbvbxQrpVB3qgPcuxm1httE0REnAYcAXwRuBd4Z0ScVXdgbWKakhfKTURdG+qpkACmQgxmNnGtnhi3YeMFLCU9F+LfgftzPxtizRv3dt1mNnxaHUEsBhbl98bnRYXP1gMzjzx/3I1xtzfSnc7PScJsuI37wKCI2LKXgQySVhvGQdpoFh+bWiXuxjh+1KrZaGj5RDkASe8CLomI5bl7A2DXiPjPuoOz8XUrEbWbj48qzEZXlbOY/raRHAAi4hHgb+sLyfqtk428E4LZ+Ab9/1ElQZSN0/bIY1BN5AvtZVtAL39wdSxr0P8wZqOkSoJYJOkrkraWtJWkr5IaqodK3RuuYaqqmcqxmVn3VEkQfwk8BXwfOIt0y42P1xlUv3VzA9irtoKpYJiSoJlVu1Du8Yg4MiLGIuLlEXFURDzei+BGRV1JpJfVZZ00dg9KYhiUOM3q0upCuePy+3xJ5za/6ghG0jslfVvSD/v93Otu6MUGprGMKhvofly1PYiJwcySVo3Np+b3L09mAZJOBvYClkbESwv9dwe+BkwDvhMRx+RTZ/9T0p/k5V44mWVPNf3aQPZjuU4GZoNv3COIiGg0RO8YEZcWX8COHSzjFGD3Yo98d9gTgD1IDyOaK2nbwihH04dnTvS77aFqNc2gbXw7uQjPzKaOKo3UB5b0O6jqAiLiMuChpt47A7dGxO0R8RRwBvAOJccCP4qIn1ddxlQ3lTZ+UyWWqRKHmY2vVRvEXEnzga2a2h9+Ajw4yeXOAO4udC/J/f4S2A3YR9JHxonrEEmLJC1atmzZJMPojzqvc/CGtxqvJ7P2WrVBXEG6vffGwL8U+v8WuG6Sy1VJv4iI40nPnxhXRMwD5gGMjY3FJOMYSt74mVk3tLpZ352SlgCP53aHbloCbF7o3gy4p+rEkuYAc2bNmtXlsHrLG/LuHDH55oFm9WjZBhERTwNPSOr2k3muAWZL2lLSmsB+QOVTZ4fhgUH9bhAfFYN+axKzfqrSSP0kcL2kkyQd33hVXYCk04ErgW0kLZF0cESsAA4FLgBuBs6MiBs7mOeUeOSoTR1+oJFZ91W56d75+TUhETF3nP4LgAUTnOd8YP7Y2NiHJxqXDSZXKZn1TpVbbfxb2asXwdlo6eTsrk6v0PYRhVnn2iYISbMl/UDSTZJub7x6EVyLmFzFNOS8QTfrvyptEN8FvgmsAN4AfI+Vt+Hoi2FopLbx9eoZ3BOJwWyUVEkQ60TExYAi4s6I+DzwxnrDMlupm3eYHe+mhU4IZquqdBaTpNWAX0s6ND+jetOa42rJVUwGk3/+RNUzn5w8bFRVSRB/BTwH+ATwcuD9lN+fqWe6XcXkDcDUNJW+l6kUi1mvtD3NNSKuyR8fAz5QbzhmZjZVVDmCMJtSuvlo0260b/jowobVQCYIt0FYmToe3eqNv42yVrf7Pja/79u7cKrxaa7WTZNJCE4gNsxaHUHsKWkN4KheBWPWDXVvtKvO30ciNuhaNVL/GHgAWFfSo6RnOETjPSKe24P4bMRM9QcmeUNvo6TVM6kPj4jpwPkR8dyIWL/43sMYa+U//NQ2TN/PMJXFRkOV01zfIel5wE6511UR0ddnfQ7LA4NssHmDb8Ouys369gWuBvYF/hy4WtI+dQfWihuprR96lRCceGyqqHKa69HAThFxYEQcAOwM/E29YZkNNycBGwRVEsRqEbG00P1gxenMRk7ZzQD9tDsbVFWeKPdjSRcAp+fu9zDBJ8GZmdngqNJIfbikvYHXkE5xnRcR59QemdkA8VGBDaMqRxBExNnA2TXHUpnPYrJB5URig2Qg2xJ8FpMNCt8M0AbZQCaIbvEfz8xsfCOdIMz6qewooR87Ld5RsvGM2wYh6XrSvZdWGUS6F9P2tUVlNkK8gbapqlUj9V49i8LMzKaccRNERNzZy0DMzGxqqXIvpldKukbSY5KekvR0vv23mdXIVU/Wb1Uaqb8OzAV+DawDfAj41zqDMjOz/qt0FlNE3ApMi4inI+K7wBvqDas1P5PaRpGPKKzXqiSIJyStCfxS0j9LOgxYt+a4WvKFcjaMqiSAiTzutBucnEZTlQSxfx7vUOBxYHNg7zqDMrNn62QD7Y25dUuVBPHOiHgyIh6NiC9ExKfwKbBmXTGIG/NBjNkmpkqCOLCk30FdjsNsqHgjasOg1ZXUc4H3AltKOrcw6LmkhwaZmdkQa3UEcQXwL8Cv8nvj9Slg9/pDM7MyrZ5QV+eRi4+KRs+4CSIi7oyIhRGxCylJrJ9fSyJiRa8CNLNVTWZjXXZjwEHe+E+1M7YGeV02q3Il9b7A1cC+wJ8DV0nap+7AzMysv6o0Uh8N7BQRB0bEAcDOwN/UG5aZtVPlCKCftxGfyrw+qqmSIFaLiKWF7gcrTmdmkzQo1z90MxG5HWXqqPJM6h9LugA4PXe/B/hRtwORtBXwOWB6RLgKy8ysz9oeCUTE4cC3gO2BHYB5EXFElZlLOlnSUkk3NPXfXdItkm6VdGRezu0RcXDnRTAz65121XnDpEoj9bERcXZEfCoiDouIcyQdW3H+p9B0SqykacAJwB7AtsBcSdt2GLeZmdWsSlvCm0v67VFl5hFxGfBQU++dgVvzEcNTwBnAO6rMD0DSIZIWSVq0bNmyqpOZWYcGcY94sm02ra4xGUXjJghJH83Ppd5G0nWF1/8C101imTOAuwvdS4AZkjaSdCLwMklHjTdxRMyLiLGIGNtkk00mEYaZmbXS6gji34E5wLn5vfF6eUS8fxLLVEm/iIgHI+IjEbF1RHyx5Qz8PAizyspOh2131lG3bj3e6R74eHH1+nRdH0kkra6kXh4Rd0TE3HxVdePVXGXUqSWkW4Y3bAbc08kM/DwIM7P69eN6hmuA2ZK2zA8i2o90lGJmZlNIrQlC0unAlaR2jCWSDs73cToUuAC4GTgzIm7scL6uYjLrULtqkplHnv/Mq7lfu2mqLKvT6qxeV+tUWT+jpsqFchMWEXPH6b8AWDCJ+c4H5o+NjX14ovMwM7PWfMsMMzMrNZAJwlVMZkm76p+psryJxjKZKqxhuFFhv2MfyAThs5jMzOo3kAnCzMzqN5AJwlVMZlPXRKpFujlNuzOvJqOXVT5T4eyugUwQrmIyM6vfQCYIMzOrnxOEmZmVGsgE4TYIs6mv7DTTVjfBq1q33s06+LKrx3ul36ewVjGQCcJtEGZm9RvIBGFmZvVzgjAzs1JOEGZmVmogE4Qbqc36azINrN1uZK463mTuB9Xp8Ik0vncybq8auAcyQbiR2sysfgOZIMzMrH5OEGZmVsoJwszMSjlBmJlZqYFMED6LyWzwdXomTrduCT7VzhSaKsstM5AJwmcxmZnVbyAThJmZ1c8JwszMSjlBmJlZKScIMzMr5QRhZmalnCDMzKzUQCaIblwHMZXONTYbNd3+//X7moVO7uha9rlV/MU70fa6nAOZIHwdhJlZ/QYyQZiZWf2cIMzMrJQThJmZlXKCMDOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMyvlBGFmZqWcIMzMrNTq/Q6gQdK6wDeAp4CFEXFan0MyMxtptR5BSDpZ0lJJNzT1313SLZJulXRk7r038IOI+DDw9jrjMjOz9uquYjoF2L3YQ9I04ARgD2BbYK6kbYHNgLvzaE/XHJeZmbWhiKh3AdJM4LyIeGnu3gX4fES8NXcflUddAjwcEedJOiMi9htnfocAhwBsscUWL7/zzjsnFJdv921mU8kdx7yto+3SHce8bcLLkrQ4IsbajdePRuoZrDxSgJQYZgBnA++W9E1g/ngTR8S8iBiLiLFNNtmk3kjNzEZYPxqpVdIvIuJx4AOVZiDNAebMmjWrq4GZmdlK/TiCWAJsXujeDLinkxn4gUFmZvXrR4K4BpgtaUtJawL7Aef2IQ4zM2uh7tNcTweuBLaRtETSwRGxAjgUuAC4GTgzIm7scL6Tfia1mZm1VmsbRETMHaf/AmDBJOY7H5g/Njb24YnOw8zMWhvIW234CMLMrH4DmSDcSG1mVr+BTBBmZla/2q+krpOkZcDELqWGjYEHuhjOIHCZR4PLPBomU+YXRkTbK40HOkFMhqRFVS41HyYu82hwmUdDL8rsKiYzMyvlBGFmZqVGOUHM63cAfeAyjwaXeTTUXuaRbYMwM7PWRvkIwszMWnCCMDOzUiOZIMZ5JvbAK3sGuKQNJV0k6df5/U9yf0k6Pq+D6yT9Wf8inzhJm0v6iaSbJd0o6ZO5/9CWW9Lakq6WdG0u8xdy/y0lXZXL/P18t2QkrZW7b83DZ/Yz/omSNE3SLySdl7uHurwAku6QdL2kX0palPv17Lc9cgmixTOxh8EpND0DHDgSuDgiZgMX525I5Z+dX4cA3+xRjN22Avh0RLwEeCXw8fx9DnO5fw+8MSJ2AHYEdpf0SuBY4Ku5zA8DB+fxDyY9zncW8NU83iD6JOkO0A3DXt6GN0TEjoVrHnr3246IkXoBuwAXFLqPAo7qd1xdLN9M4IZC9y3A8/Pn5wO35M/fAuaWjTfIL+CHwJtHpdzAc4CfA68gXVW7eu7/zO+cdGv9XfLn1fN46nfsHZZzs7wxfCNwHunJlENb3kK57wA2burXs9/2yB1BMP4zsYfV8yLiXoD8vmnuP3TrIVclvAy4iiEvd65u+SWwFLgIuA14JNLzVuDZ5XqmzHn4cmCj3kY8accBRwB/zN0bMdzlbQjgQkmLJR2S+/Xst92PZ1L3W+kzsXseRf8N1XqQtB7wH8BfRcSjUlnx0qgl/Qau3BHxNLCjpA2Ac4CXlI2W3we6zJL2ApZGxGJJuzZ6l4w6FOVt8uqIuEfSpsBFkn7VYtyul3sUjyAm/UzsAXO/pOcD5Peluf/QrAdJa5CSw2kRcXbuPfTlBoiIR4CFpPaXDSQ1dvqK5XqmzHn4dOCh3kZTAdNCAAAD+klEQVQ6Ka8G3i7pDuAMUjXTcQxveZ8REffk96WkHYGd6eFvexQTxKg9E/tc4MD8+UBSHX2j/wH5zIdXAssbh62DROlQ4STg5oj4SmHQ0JZb0ib5yAFJ6wC7kRpvfwLsk0drLnNjXewDXBK5knoQRMRREbFZRMwk/V8viYj3MaTlbZC0rqT1G5+BtwA30Mvfdr8bYfrU8LMn8D+ketvP9TueLpbrdOBe4A+kvYmDSXWvFwO/zu8b5nFFOpvrNuB6YKzf8U+wzK8hHUZfB/wyv/Yc5nID2wO/yGW+Afi/uf9WwNXArcBZwFq5/9q5+9Y8fKt+l2ESZd8VOG8UypvLd21+3djYVvXyt+1bbZiZWalRrGIyM7MKnCDMzKyUE4SZmZVygjAzs1JOEGZmVsoJwoaKpA0kfazCeDMlvbfieDe0G69pmi/lu6x+qZPpJkrSKZL2aT+mWWecIGzYbAC0TRCkmxq2TRAT9BfAn0XE4d2eceHKYbPaOUHYsDkG2DrfP/9L+arSL0m6Id9X/z2F8V6bxzssHyn8VNLP8+tVrRYy3nwlnQusC1xVWFZjmuvzEY4kPSjpgNz/VEm7KT3n4bt5vF9IekMefpCksyTNJ924TZK+LukmSeez8mZtSDom979O0pe7tE5tRHlvxIbNkcBLI2JHAEnvJj0zYQdgY+AaSZfl8T4TEXvl8Z4DvDkinpQ0m3RV+ljZArK9y+YbEW+X9Fhj+U1+Rrqv0J3A7cBrge+R7qP0UeDjABHxp5JeTEoGL8rT7gJsHxEPSdob2Ab4U+B5wE3AyZI2BN4FvDgionE7DrOJ8hGEDbvXAKdHxNMRcT9wKbBTyXhrAN+WdD3pNg3tHiJVdb5FPwVel1/fBP5U0gzgoYh4LM/zVICI+BUpkTQSxEUR0bjh3OsKy74HuCT3fxR4EvhOTiJPtInHrCUnCBt24973u8lhwP2kI4IxYM0uzbfoMtJRw2tJd2BdRrqZ3E8rzPPxpu5V7pET6dkHO5PubPtO4McTiNHsGU4QNmx+C6xf6L4MeI/SA3Y2Ie19X10y3nTg3oj4I7A/MK3Ncsab77gi4m5SddTsiLgduBz4DCsTxGXA+wBy1dIWpKeClS17v7zs5wONtor1gOkRsQD4K1IVmNmEuQ3ChkpEPCjpZ/nU1B+RnkK2C+mOmAEcERH3SXoQWCHpWtKzvL8B/IekfUm3kW7eY292Ttl8K4R4FSuTz0+BL5ISBTmGE3M11wrgoIj4vVZ9+NE5pGciXE+6K/Gluf/6wA8lrU06GjmsQjxm4/LdXM3MrJSrmMzMrJQThJmZlXKCMDOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMyv1/wE8taHq39aWqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(description_lenght, range(0,500))\n",
    "plt.title('total of words in articles description - TRAIN DATA')\n",
    "plt.xlabel('total of words')\n",
    "plt.ylabel('total of articles (log)')\n",
    "plt.yscale('log')\n",
    "plt.savefig('article_title1.pdf')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}