Scrap_e_corpus.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dcatrib/tesePython/blob/main/Scrap_e_corpus.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "biEumS-QyEDv"
      },
      "source": [
        "Instala os pacotes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "KLlzG5xH9C2Q",
        "outputId": "6a00c00a-918d-4d30-c8e7-c71487651227"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting PyMuPDF\n",
            "  Downloading pymupdf-1.25.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (3.4 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (2.32.3)\n",
            "Collecting selenium\n",
            "  Downloading selenium-4.28.0-py3-none-any.whl.metadata (7.1 kB)\n",
            "Collecting chromedriver-autoinstaller\n",
            "  Downloading chromedriver_autoinstaller-0.6.4-py3-none-any.whl.metadata (2.1 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.2)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.11/dist-packages (3.9.1)\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.11/dist-packages (4.3.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.26.4)\n",
            "Requirement already satisfied: xlrd in /usr/local/lib/python3.11/dist-packages (2.0.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests) (2024.12.14)\n",
            "Collecting trio~=0.17 (from selenium)\n",
            "  Downloading trio-0.28.0-py3-none-any.whl.metadata (8.5 kB)\n",
            "Collecting trio-websocket~=0.9 (from selenium)\n",
            "  Downloading trio_websocket-0.11.1-py3-none-any.whl.metadata (4.7 kB)\n",
            "Requirement already satisfied: typing_extensions~=4.9 in /usr/local/lib/python3.11/dist-packages (from selenium) (4.12.2)\n",
            "Requirement already satisfied: websocket-client~=1.8 in /usr/local/lib/python3.11/dist-packages (from selenium) (1.8.0)\n",
            "Requirement already satisfied: packaging>=23.1 in /usr/local/lib/python3.11/dist-packages (from chromedriver-autoinstaller) (24.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from nltk) (8.1.8)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (from nltk) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.11/dist-packages (from nltk) (2024.11.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from nltk) (4.67.1)\n",
            "Requirement already satisfied: scipy<1.14.0,>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from gensim) (1.13.1)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.11/dist-packages (from gensim) (7.1.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.11/dist-packages (from smart-open>=1.8.1->gensim) (1.17.0)\n",
            "Requirement already satisfied: attrs>=23.2.0 in /usr/local/lib/python3.11/dist-packages (from trio~=0.17->selenium) (24.3.0)\n",
            "Collecting sortedcontainers (from trio~=0.17->selenium)\n",
            "  Downloading sortedcontainers-2.4.0-py2.py3-none-any.whl.metadata (10 kB)\n",
            "Collecting outcome (from trio~=0.17->selenium)\n",
            "  Downloading outcome-1.3.0.post0-py2.py3-none-any.whl.metadata (2.6 kB)\n",
            "Requirement already satisfied: sniffio>=1.3.0 in /usr/local/lib/python3.11/dist-packages (from trio~=0.17->selenium) (1.3.1)\n",
            "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium)\n",
            "  Downloading wsproto-1.2.0-py3-none-any.whl.metadata (5.6 kB)\n",
            "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.11/dist-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
            "Requirement already satisfied: h11<1,>=0.9.0 in /usr/local/lib/python3.11/dist-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.14.0)\n",
            "Downloading pymupdf-1.25.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (20.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.0/20.0 MB\u001b[0m \u001b[31m24.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading selenium-4.28.0-py3-none-any.whl (9.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.5/9.5 MB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading chromedriver_autoinstaller-0.6.4-py3-none-any.whl (7.6 kB)\n",
            "Downloading trio-0.28.0-py3-none-any.whl (486 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m486.3/486.3 kB\u001b[0m \u001b[31m18.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n",
            "Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
            "Downloading outcome-1.3.0.post0-py2.py3-none-any.whl (10 kB)\n",
            "Downloading sortedcontainers-2.4.0-py2.py3-none-any.whl (29 kB)\n",
            "Installing collected packages: sortedcontainers, wsproto, PyMuPDF, outcome, chromedriver-autoinstaller, trio, trio-websocket, selenium\n",
            "Successfully installed PyMuPDF-1.25.2 chromedriver-autoinstaller-0.6.4 outcome-1.3.0.post0 selenium-4.28.0 sortedcontainers-2.4.0 trio-0.28.0 trio-websocket-0.11.1 wsproto-1.2.0\n",
            "Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB]\n",
            "Get:4 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "Get:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease [1,581 B]\n",
            "Get:6 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n",
            "Hit:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Hit:8 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Hit:9 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Get:10 https://r2u.stat.illinois.edu/ubuntu jammy InRelease [6,555 B]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,860 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,519 kB]\n",
            "Get:13 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ Packages [62.5 kB]\n",
            "Get:14 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  Packages [1,234 kB]\n",
            "Get:15 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2,561 kB]\n",
            "Get:16 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,228 kB]\n",
            "Get:17 https://r2u.stat.illinois.edu/ubuntu jammy/main all Packages [8,606 kB]\n",
            "Get:18 https://r2u.stat.illinois.edu/ubuntu jammy/main amd64 Packages [2,641 kB]\n",
            "Fetched 21.1 MB in 8s (2,790 kB/s)\n",
            "Reading package lists... Done\n",
            "W: Skipping acquire of configured file 'main/source/Sources' as repository 'https://r2u.stat.illinois.edu/ubuntu jammy InRelease' does not seem to provide it (sources.list entry misspelt?)\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "wget is already the newest version (1.21.2-2ubuntu1.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 65 not upgraded.\n",
            "--2025-01-21 13:45:03--  https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\n",
            "Resolving dl.google.com (dl.google.com)... 74.125.143.190, 74.125.143.93, 74.125.143.91, ...\n",
            "Connecting to dl.google.com (dl.google.com)|74.125.143.190|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 112253548 (107M) [application/x-debian-package]\n",
            "Saving to: ‘google-chrome-stable_current_amd64.deb’\n",
            "\n",
            "google-chrome-stabl 100%[===================>] 107.05M   313MB/s    in 0.3s    \n",
            "\n",
            "2025-01-21 13:45:03 (313 MB/s) - ‘google-chrome-stable_current_amd64.deb’ saved [112253548/112253548]\n",
            "\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Note, selecting 'google-chrome-stable' instead of './google-chrome-stable_current_amd64.deb'\n",
            "The following additional packages will be installed:\n",
            "  libvulkan1 mesa-vulkan-drivers\n",
            "The following NEW packages will be installed:\n",
            "  google-chrome-stable libvulkan1 mesa-vulkan-drivers\n",
            "0 upgraded, 3 newly installed, 0 to remove and 65 not upgraded.\n",
            "Need to get 10.9 MB/123 MB of archives.\n",
            "After this operation, 416 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 libvulkan1 amd64 1.3.204.1-2 [128 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 mesa-vulkan-drivers amd64 23.2.1-1ubuntu3.1~22.04.3 [10.7 MB]\n",
            "Get:3 /content/google-chrome-stable_current_amd64.deb google-chrome-stable amd64 132.0.6834.83-1 [112 MB]\n",
            "Fetched 10.9 MB in 2s (6,181 kB/s)\n",
            "Selecting previously unselected package libvulkan1:amd64.\n",
            "(Reading database ... 124565 files and directories currently installed.)\n",
            "Preparing to unpack .../libvulkan1_1.3.204.1-2_amd64.deb ...\n",
            "Unpacking libvulkan1:amd64 (1.3.204.1-2) ...\n",
            "Selecting previously unselected package google-chrome-stable.\n",
            "Preparing to unpack .../google-chrome-stable_current_amd64.deb ...\n",
            "Unpacking google-chrome-stable (132.0.6834.83-1) ...\n",
            "Selecting previously unselected package mesa-vulkan-drivers:amd64.\n",
            "Preparing to unpack .../mesa-vulkan-drivers_23.2.1-1ubuntu3.1~22.04.3_amd64.deb ...\n",
            "Unpacking mesa-vulkan-drivers:amd64 (23.2.1-1ubuntu3.1~22.04.3) ...\n",
            "Setting up libvulkan1:amd64 (1.3.204.1-2) ...\n",
            "Setting up mesa-vulkan-drivers:amd64 (23.2.1-1ubuntu3.1~22.04.3) ...\n",
            "Setting up google-chrome-stable (132.0.6834.83-1) ...\n",
            "update-alternatives: using /usr/bin/google-chrome-stable to provide /usr/bin/x-www-browser (x-www-browser) in auto mode\n",
            "update-alternatives: using /usr/bin/google-chrome-stable to provide /usr/bin/gnome-www-browser (gnome-www-browser) in auto mode\n",
            "update-alternatives: using /usr/bin/google-chrome-stable to provide /usr/bin/google-chrome (google-chrome) in auto mode\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for mailcap (3.70+nmu1ubuntu1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtcm.so.1 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_opencl.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_loader.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtcm_debug.so.1 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libumf.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libhwloc.so.15 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!pip install PyMuPDF requests selenium chromedriver-autoinstaller pandas nltk gensim numpy xlrd\n",
        "\n",
        "\n",
        "!apt-get update\n",
        "!apt-get install -y wget\n",
        "!wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\n",
        "!apt-get install -y ./google-chrome-stable_current_amd64.deb\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4KJ1jIQayInW"
      },
      "source": [
        "Importa os pacotes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "AyVkOEFexu7A"
      },
      "outputs": [],
      "source": [
        "from selenium import webdriver\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.support.ui import WebDriverWait\n",
        "from selenium.webdriver.support import expected_conditions as EC\n",
        "from selenium.common.exceptions import NoSuchElementException, TimeoutException, StaleElementReferenceException\n",
        "import chromedriver_autoinstaller\n",
        "import requests\n",
        "import fitz\n",
        "import os\n",
        "import pandas as pd\n",
        "import re\n",
        "import string\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from google.colab import files\n",
        "import numpy as np\n",
        "from gensim.models import KeyedVectors, Word2Vec\n",
        "import gensim\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.model_selection import train_test_split, cross_val_score\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from statsmodels.tsa.ar_model import AutoReg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3KTjmaLPyOK2"
      },
      "source": [
        "Configura o webdriver"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "collapsed": true,
        "id": "MqPHOhrtxvNR",
        "outputId": "985a4a1b-c06a-4625-b34a-fb608b131266"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/usr/local/lib/python3.11/dist-packages/chromedriver_autoinstaller/132/chromedriver'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "# Configurando o WebDriver\n",
        "chrome_options = webdriver.ChromeOptions()\n",
        "chrome_options.add_argument('--headless')\n",
        "chrome_options.add_argument('--no-sandbox')\n",
        "chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "\n",
        "# Instala automaticamente a versão correta do ChromeDriver\n",
        "chromedriver_autoinstaller.install()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ykV6aFA4yVGG"
      },
      "source": [
        "Entra no url, espera os seletores e armazena os links"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N6MM2g-p7BdE"
      },
      "outputs": [],
      "source": [
        "# URL para scrape\n",
        "url = \"https://www.bcb.gov.br/en/publications/copomminutes/cronologicos\"\n",
        "\n",
        "# Criar uma nova instância do Chrome driver\n",
        "driver = webdriver.Chrome(options=chrome_options)\n",
        "driver.get(url)\n",
        "\n",
        "# Esperar o seletor específico estar presente\n",
        "WebDriverWait(driver, 15).until(\n",
        "    EC.presence_of_element_located((By.CSS_SELECTOR, 'body > app-root > app-root > main > dynamic-comp > div > div > bcb-publicacao > div > div > bcb-ultimaspublicacoes > div'))\n",
        ")\n",
        "\n",
        "# Definir o seletor\n",
        "selector = 'body > app-root > app-root > main > dynamic-comp > div > div > bcb-publicacao > div > div > bcb-ultimaspublicacoes > div'\n",
        "\n",
        "# Lista para armazenar hrefs, textos e links de download\n",
        "hrefs_and_texts = []\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6q7Gc8Ceyed0"
      },
      "source": [
        "Entra no link e baixa o conteudo do pdf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uXBHUR-O7HRi",
        "outputId": "4a8e8300-afc6-4162-a4a8-c01014ee629b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "4\n",
            "5\n",
            "6\n",
            "7\n",
            "8\n",
            "9\n",
            "10\n",
            "11\n",
            "12\n",
            "13\n",
            "14\n",
            "15\n",
            "16\n",
            "17\n",
            "18\n",
            "19\n",
            "20\n",
            "21\n",
            "22\n",
            "23\n",
            "24\n",
            "25\n",
            "26\n",
            "27\n",
            "28\n",
            "29\n",
            "30\n",
            "31\n",
            "32\n",
            "33\n",
            "34\n",
            "35\n",
            "36\n",
            "37\n",
            "38\n",
            "39\n",
            "40\n",
            "41\n",
            "42\n",
            "43\n",
            "44\n",
            "45\n",
            "46\n",
            "47\n",
            "48\n",
            "49\n",
            "50\n",
            "51\n",
            "52\n",
            "53\n",
            "54\n",
            "55\n",
            "56\n",
            "57\n",
            "58\n",
            "59\n",
            "60\n",
            "61\n",
            "62\n",
            "63\n",
            "64\n",
            "65\n",
            "66\n",
            "67\n",
            "68\n",
            "69\n",
            "70\n",
            "71\n",
            "72\n",
            "73\n",
            "74\n",
            "75\n",
            "76\n",
            "77\n",
            "78\n",
            "79\n",
            "80\n",
            "81\n",
            "82\n",
            "83\n",
            "84\n",
            "85\n",
            "86\n",
            "87\n",
            "88\n",
            "89\n",
            "90\n",
            "91\n",
            "92\n",
            "93\n",
            "94\n",
            "95\n",
            "96\n",
            "97\n",
            "98\n",
            "99\n",
            "100\n",
            "101\n",
            "102\n",
            "103\n",
            "104\n",
            "105\n",
            "106\n",
            "107\n",
            "108\n",
            "109\n",
            "110\n",
            "111\n",
            "112\n",
            "113\n",
            "114\n",
            "115\n",
            "116\n",
            "117\n",
            "118\n",
            "119\n",
            "120\n",
            "121\n",
            "122\n",
            "123\n",
            "124\n",
            "125\n",
            "126\n",
            "127\n",
            "128\n",
            "129\n",
            "130\n",
            "131\n",
            "132\n",
            "133\n",
            "134\n",
            "135\n",
            "136\n",
            "137\n",
            "138\n",
            "139\n",
            "140\n",
            "141\n",
            "142\n",
            "143\n",
            "144\n",
            "145\n",
            "146\n",
            "147\n",
            "148\n",
            "149\n",
            "150\n",
            "151\n",
            "152\n",
            "153\n",
            "154\n",
            "155\n",
            "156\n",
            "157\n",
            "158\n",
            "159\n",
            "160\n",
            "161\n",
            "162\n",
            "163\n",
            "164\n",
            "165\n",
            "166\n",
            "167\n",
            "168\n",
            "169\n",
            "170\n",
            "171\n",
            "172\n",
            "173\n",
            "174\n",
            "175\n",
            "176\n",
            "177\n",
            "178\n",
            "179\n",
            "180\n",
            "181\n",
            "182\n"
          ]
        }
      ],
      "source": [
        "# Função para recuperar links e botões de download\n",
        "def retrieve_links():\n",
        "    try:\n",
        "        elements = driver.find_elements(By.CSS_SELECTOR, selector)\n",
        "        link_count = 0  # Inicializa um contador para os links\n",
        "        for element in elements:\n",
        "            links = element.find_elements(By.TAG_NAME, 'a')\n",
        "            for link in links:\n",
        "                if link_count >= 183:  # Limita a 50 links\n",
        "                    return  # Sai da função se 50 links foram processados\n",
        "                href = link.get_attribute('href')\n",
        "                text = link.text\n",
        "                if href and text and text[0].isdigit():\n",
        "                    driver.get(href)\n",
        "                    # Esperar o botão de download\n",
        "                    try:\n",
        "                        download_button = WebDriverWait(driver, 65).until(\n",
        "                            EC.presence_of_element_located((By.CSS_SELECTOR, '#publicacao > div.col-lg-9.d-flex.flex-column > div > div > div > div.d-flex.flex-column-reverse.flex-sm-row.justify-content-sm-between.align-items-sm-center > div.d-flex.align-items-center.mb-3.mb-sm-0.d-print-none > download > div > div > a'))\n",
        "                        )\n",
        "                        download_href = download_button.get_attribute('href')\n",
        "\n",
        "                        # Fazer download do PDF\n",
        "                        pdf_response = requests.get(download_href)\n",
        "                        pdf_filename = f\"{text}.pdf\"\n",
        "                        with open(pdf_filename, 'wb') as pdf_file:\n",
        "                            pdf_file.write(pdf_response.content)\n",
        "\n",
        "                        # Extrair texto do PDF usando PyMuPDF\n",
        "                        pdf_text = ''\n",
        "                        with fitz.open(pdf_filename) as pdf_file:\n",
        "                            for page in pdf_file:\n",
        "                                page_text = page.get_text()\n",
        "                                pdf_text += page_text + '\\n'\n",
        "\n",
        "                        # Limpeza do texto\n",
        "                        pdf_text = pdf_text.replace('\\n', ' ')  # Remove quebras de linha\n",
        "                        pdf_text = re.sub(r'\\s+', ' ', pdf_text)  # Substitui múltiplos espaços por um único\n",
        "                        pdf_text = pdf_text.strip()  # Remove espaços no início e no fim\n",
        "\n",
        "                        # Print da primeira linha extraída\n",
        "                        first_line = pdf_text.split('.')[0]  # Considera a primeira frase\n",
        "                        #print(f'Primeira linha extraída: {first_line.strip()}')\n",
        "                        print(link_count)\n",
        "                    except (NoSuchElementException, TimeoutException):\n",
        "                        download_href = None\n",
        "                        pdf_text = None\n",
        "\n",
        "                    hrefs_and_texts.append((href, text, download_href, pdf_text))\n",
        "                    link_count += 1  # Incrementa o contador\n",
        "                    driver.back()\n",
        "                    WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.CSS_SELECTOR, selector)))\n",
        "    except StaleElementReferenceException:\n",
        "        retrieve_links()\n",
        "\n",
        "# Iniciar a recuperação de links\n",
        "retrieve_links()\n",
        "\n",
        "# Fechar o driver\n",
        "driver.quit()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XvLYZn946Hlr"
      },
      "source": [
        "Minhas stopwords e expressoes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D7_zXjpX6HUz"
      },
      "outputs": [],
      "source": [
        "# Minhas stopwords\n",
        "my_stopwords = {\n",
        "    'January', 'February', 'March', 'April', 'May', 'June', 'July',\n",
        "    'August', 'September', 'October', 'November', 'December','january', 'february', 'march', 'april', 'may', 'june', 'july',\n",
        "    'august', 'september', 'october', 'november', 'december', 'pp',\n",
        "    'minutes', 'bcb', 'th', 'copom', 'bcbgovbr', 'brasilia', 'pm','roberto', \"twelve\", 'month', 'monthonmonth', 'monthly',\n",
        "    \"i\", \"me\", \"my\", \"myself\", \"we\", \"our\", \"ours\", \"ourselves\", \"us\", \"you\", \"your\", \"yours\", \"yourself\", \"yourselves\", \"he\", \"him\", \"his\", \"himself\", \"she\", \"her\", \"hers\", \"herself\", \"it\", \"its\", \"itself\", \"they\", \"them\", \"their\", \"theirs\", \"themselves\", \"what\", \"which\", \"who\", \"whom\", \"this\", \"that\", \"these\", \"those\", \"am\", \"is\", \"are\", \"was\", \"were\", \"be\", \"been\", \"being\", \"have\", \"has\", \"had\", \"having\", \"do\", \"does\", \"did\", \"doing\", \"would\", \"could\", \"should\", \"ought\", \"not\", \"a\", \"an\", \"the\", \"and\", \"but\", \"if\", \"or\", \"because\", \"as\", \"until\", \"while\", \"of\", \"at\", \"by\", \"for\", \"with\", \"about\", \"against\", \"between\", \"into\", \"through\", \"during\", \"before\", \"after\", \"above\", \"below\", \"to\", \"from\", \"up\", \"down\", \"in\", \"out\", \"on\", \"off\", \"over\", \"under\", \"again\", \"further\", \"then\", \"once\", \"here\", \"there\", \"when\", \"where\", \"why\", \"how\", \"all\", \"any\", \"both\", \"each\", \"few\", \"more\", \"most\", \"other\", \"some\", \"such\", \"no\", \"nor\", \"not\", \"only\", \"own\", \"same\", \"so\", \"than\", \"too\", \"very\", \"one\", \"every\", \"least\", \"less\", \"many\", \"now\", \"ever\", \"never\", \"say\", \"says\", \"said\", \"also\", \"get\", \"go\", \"goes\", \"just\", \"made\", \"make\", \"put\", \"see\", \"seen\", \"whether\", \"like\", \"well\", \"back\", \"even\", \"still\", \"way\", \"take\", \"since\", \"another\", \"however\", \"two\", \"three\", \"four\", \"five\", \"first\", \"second\", \"new\", \"old\", \"high\", \"long\",\n",
        "    \"aa\", \"abcr\", \"afonso\", \"ailton\",\n",
        "    \"adriana\", \"afonso\", \"aguiar\", \"ailton\", \"alagoas\", \"alberto\", \"alegre\", \"alexandre\", \"almeida\", \"altamir\", \"alves\", \"alvir\", \"amariz\", \"andré\", \"ângelo\", \"angola\", \"anthero\", \"antonio\", \"antônio\", \"antunes\", \"aquino\", \"arantes\", \"araujo\", \"araújo\", \"ariosto\", \"arquete\", \"arraes\", \"assis\", \"awazu\", \"ayrosa\", \"azevedo\", \"barbosa\", \"barreira\", \"barreto\", \"barros\", \"beatriz\", \"belo\", \"berardinelli\", \"bevilaqua\", \"bezerra\", \"brito\", \"bruno\", \"cadastro\", \"caetano\", \"campos\", \"cardoso\", \"carioca\", \"carla\", \"carlos\", \"carolina\", \"carvalho\", \"cassava\", \"cavalheiro\", \"celina\", \"césar\", \"chukin\", \"coelho\", \"correa\", \"corrêa\", \"costa\", \"coutinho\", \"couto\", \"cunha\", \"dias\", \"diogo\", \"donizete\", \"duarte\", \"edilson\", \"edson\", \"eduardo\", \"emanuel\", \"ênio\", \"enrico\", \"eugênio\", \"fabio\", \"fábio\", \"falcão\", \"feitosa\", \"felício\", \"feltrim\", \"fenabrave\", \"fernanda\", \"fernandes\", \"fernando\", \"ferreira\", \"filgueiras\", \"filho\", \"flávio\", \"florido\", \"fonseca\", \"francisco\", \"franco\", \"freire\", \"freitas\", \"gabriel\", \"gaglianone\", \"galípolo\", \"geraldo\", \"getulio\", \"getúlio\", \"goldfajn\", \"gomes\", \"gonçalves\", \"goreth\", \"guardado\", \"guerra\", \"gustavo\", \"hamilton\", \"hennings\", \"henrique\", \"hoffmann\", \"ilan\", \"isaac\", \"isabela\", \"ivan\", \"janeiro\", \"jansson\", \"joão\", \"jocimar\", \"jose\", \"josé\", \"júnior\", \"kanczuk\", \"katherine\", \"katrina\", \"katrinas\", \"kurrle\", \"leonardo\", \"leopoldo\", \"lilian\", \"lima\", \"linaldo\", \"lopes\", \"lourenço\", \"luis\", \"luís\", \"luiz\", \"maciel\", \"magalhães\", \"maia\", \"manoel\", \"marciano\", \"marcio\", \"márcio\", \"maria\", \"mário\", \"marques\", \"martins\", \"matos\", \"maurício\", \"meirelles\", \"mello\", \"melo\", \"mendes\", \"mendonça\", \"menezes\", \"mesquita\", \"michel\", \"minella\", \"miranda\", \"moraes\", \"moreira\", \"moura\", \"muricca\", \"nechio\", \"nelson\", \"neto\", \"neves\", \"niña\", \"niño\", \"nogueira\", \"oliveira\", \"otávio\", \"pacceli\", \"paes\", \"paul\", \"paula\", \"paulo\", \"pereira\", \"piazza\", \"picchetti\", \"pinheiro\", \"pinho\", \"rabello\", \"rego\", \"reinaldo\", \"reis\", \"renato\", \"ribamar\", \"ribeiro\", \"ricardo\", \"rocha\", \"rodrigo\", \"rodrigues\", \"santanna\", \"santo\", \"santos\", \"saraiva\", \"sergio\", \"sérgio\", \"serra\", \"sidnei\", \"silva\", \"simão\", \"sobrinho\", \"sonia\", \"sousa\", \"souza\", \"stefano\", \"teixeira\", \"thomaz\", \"thomson\", \"tiago\", \"torós\", \"tulio\", \"túlio\", \"valderez\", \"vanderléia\", \"vargas\", \"vasconcelos\", \"vera\", \"viana\", \"vieira\", \"vilela\", \"villa\", \"volpon\", \"walter\", \"ximenes\", \"yukos\",\n",
        "    \"aa\", \"abastecimento\", \"acre\", \"africa\", \"amazonas\", \"america\", \"american\", \"angra\", \"aprilmay\", \"aprils\", \"arabia\", \"argentina\", \"argentinean\", \"asia\", \"asian\", \"augustoctober\", \"augustseptember\", \"australia\", \"bahia\", \"banco\", \"belém\", \"boston\", \"brasil\", \"brasília\", \"brazil\", \"brazilian\", \"brazilians\", \"brazils\", \"canada\", \"centro\", \"chicago\", \"china\", \"chinas\", \"chinese\", \"coimbra\", \"colombia\", \"comércio\", \"companhia\", \"curitiba\", \"czech\", \"da\", \"de\", \"denmark\", \"dos\", \"dubai\", \"elétrica\", \"embargo\", \"endapril\", \"endaugust\", \"endjuly\", \"endjune\", \"endmarch\", \"endmay\", \"endoctober\", \"endofnovember\", \"endseptember\", \"energia\", \"england\", \"englands\", \"espírito\", \"estado\", \"estadual\", \"estudos\", \"feb\", \"februaryapril\", \"februarymarch\", \"februarymay\", \"febuary\", \"federação\", \"florianópolis\", \"fortaleza\", \"franc\", \"france\", \"fukushima\", \"fundação\", \"gerais\", \"geral\", \"german\", \"germany\", \"goiânia\", \"grande\", \"grazie\", \"greece\", \"grosso\", \"hong\", \"horizonte\", \"india\", \"indonesia\", \"iran\", \"iranian\", \"iraq\", \"iraqs\", \"ireland\", \"israel\", \"istanbul\", \"italian\", \"italy\", \"iv\", \"jan\", \"januaryapril\", \"januaryaugust\", \"januaryfebruary\", \"januaryjuly\", \"januaryjune\", \"januarymarch\", \"januarymay\", \"januarynovember\", \"januaryoctober\", \"januarys\", \"japan\", \"japanese\", \"japans\", \"jp\", \"julyaugust\", \"julynovember\", \"julys\", \"juneaugust\", \"junejuly\", \"juneoctober\", \"juneseptember\", \"kong\", \"korea\", \"la\", \"latejanuary\", \"latemarch\", \"legumes\", \"lehman\", \"madrid\", \"malaysia\", \"maranhão\", \"marchmay\", \"mato\", \"mayjuly\", \"mayjune\", \"mercosur\", \"mexican\", \"mexico\", \"michigan\", \"michiganthomson\", \"midapril\", \"midaugust\", \"middecember\", \"midfebruary\", \"midjuly\", \"midjune\", \"midmarch\", \"midmay\", \"midoctober\", \"midseptember\", \"minas\", \"nacional\", \"nasdaq\", \"netherlands\", \"niger\", \"nigeria\", \"nikkei\", \"norte\", \"novemberapril\", \"novo\", \"oceania\", \"octobernovember\", \"octobers\", \"Palavras\", \"paraná\", \"pernambuco\", \"peru\", \"philippines\", \"pia\", \"piauí\", \"poland\", \"portugal\", \"presidente\", \"reais\", \"real\", \"realdollar\", \"rio\", \"rj\", \"romania\", \"rondônia\", \"roraima\", \"rural\", \"russia\", \"russian\", \"sa\", \"salvador\", \"sao\", \"são\", \"saudi\", \"sergipe\", \"shangai\", \"shanghai\", \"sidney\", \"singapore\", \"sp\", \"spain\", \"spains\", \"sul\", \"switzerland\", \"syria\", \"taiwan\", \"tocantins\", \"tokyo\", \"turkey\", \"uk\", \"ukraine\", \"uruguay\", \"usa\", \"venezuela\", \"zealand\",\n",
        "    \"df\", \"di\", \"el\", \"es\", \"eu\", \"ha\", \"ii\", \"iic\", \"iif\", \"iii\", \"st\"\n",
        "    }\n",
        "\n",
        "# Array de expressões a serem removidas\n",
        "expressions_to_remove = [\n",
        "    'bcb.gov.br Minutes of the Meeting of the Monetary Policy Committee — Copom',\n",
        "]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Is8XaMx7-BIN"
      },
      "source": [
        "fazer o preprocessamento e criar o df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "jRdPCx--8pb-",
        "outputId": "cca7859c-3849-4a32-f575-835f392b46d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                                  Link  \\\n",
            "0    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "1    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "2    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "3    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "4    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "..                                                 ...   \n",
            "178  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "179  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "180  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "181  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "182  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "\n",
            "                                     Texto  \\\n",
            "0     267th Meeting - December 10-11, 2024   \n",
            "1       266th Meeting - November 5-6, 2024   \n",
            "2    265th Meeting - September 17-18, 2024   \n",
            "3         264th Meeting - July 30-31, 2024   \n",
            "4         263rd Meeting - June 18-19, 2024   \n",
            "..                                     ...   \n",
            "178      89th Copom minutes - October 2003   \n",
            "179    88th Copom minutes - September 2003   \n",
            "180       87th Copom minutes - August 2003   \n",
            "181         86th Copom minutes - July 2003   \n",
            "182         85th Copom minutes - June 2003   \n",
            "\n",
            "                                      Link de Download  \\\n",
            "0    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "1    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "2    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "3    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "4    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "..                                                 ...   \n",
            "178  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "179  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "180  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "181  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "182  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "\n",
            "                                          Texto do PDF  \\\n",
            "0    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "1    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "2    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "3    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "4    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "..                                                 ...   \n",
            "178  1 Minutes of the 89th Meeting of the Monetary ...   \n",
            "179  1 Minutes of the 88th Meeting of the Monetary ...   \n",
            "180  1 Minutes of the 87th Meeting of the Monetary ...   \n",
            "181  1 Minutes of the 86th Meeting of the Monetary ...   \n",
            "182  Minutes of the 85th Meeting of the Monetary Po...   \n",
            "\n",
            "                                      Texto Processado  \n",
            "0    meeting monetary policy committee meeting date...  \n",
            "1    meeting monetary policy committee meeting date...  \n",
            "2    meeting monetary policy committee meeting date...  \n",
            "3    meeting monetary policy committee meeting date...  \n",
            "4    meeting monetary policy committee rd rd meetin...  \n",
            "..                                                 ...  \n",
            "178  meeting monetary policy committee (copom) date...  \n",
            "179  meeting monetary policy committee (copom) date...  \n",
            "180  meeting monetary policy committee (copom) date...  \n",
            "181  meeting monetary policy committee (copom) date...  \n",
            "182  meeting monetary policy committee (copom) date...  \n",
            "\n",
            "[183 rows x 5 columns]\n"
          ]
        }
      ],
      "source": [
        "# Baixar as stopwords se ainda não tiver feito\n",
        "nltk.download('stopwords')\n",
        "\n",
        "# Criar um DataFrame com os links e textos extraídos\n",
        "df = pd.DataFrame(hrefs_and_texts, columns=['Link', 'Texto', 'Link de Download', 'Texto do PDF'])\n",
        "\n",
        "# Carregar stopwords em inglês\n",
        "stop_words = set(stopwords.words('english'))\n",
        "stop_words.update(my_stopwords)\n",
        "\n",
        "# Função para preprocessar o texto\n",
        "def preprocess_text(text):\n",
        "    if text:\n",
        "        # Remover números, pontuação e caracteres especiais\n",
        "        text = re.sub(r'[0-9]+', '', text)  # Remove números\n",
        "        # Remover pontuação, exceto parênteses\n",
        "        text = re.sub(r'[^\\w\\s()]+', '', text)  # Mantém letras, números, espaços e parênteses\n",
        "        text = re.sub(r'\\s+', ' ', text)  # Substituir múltiplos espaços por um único espaço\n",
        "        text = text.strip().lower()  # Retorna texto em minúsculas\n",
        "\n",
        "        # Remover expressões específicas\n",
        "        for expr in expressions_to_remove:\n",
        "            text = text.replace(expr, '')\n",
        "\n",
        "        # Remover stopwords\n",
        "        text = ' '.join([word for word in text.split() if word not in stop_words])\n",
        "        return text\n",
        "    return ''\n",
        "\n",
        "# Criar a nova coluna 'Texto Processado'\n",
        "df['Texto Processado'] = df['Texto do PDF'].apply(preprocess_text)\n",
        "\n",
        "# Imprimir o DataFrame\n",
        "print(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WzPHCbt0ynDM"
      },
      "source": [
        "Extrai do texto a data da reuniao, formata a data e coloca como uma variavel no df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "pd-A0QrjI-xe",
        "outputId": "a22756b3-d4a8-419e-dae2-05b96861726d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                                  Link  \\\n",
            "0    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "1    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "2    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "3    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "4    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "..                                                 ...   \n",
            "178  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "179  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "180  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "181  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "182  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "\n",
            "                                     Texto  \\\n",
            "0     267th Meeting - December 10-11, 2024   \n",
            "1       266th Meeting - November 5-6, 2024   \n",
            "2    265th Meeting - September 17-18, 2024   \n",
            "3         264th Meeting - July 30-31, 2024   \n",
            "4         263rd Meeting - June 18-19, 2024   \n",
            "..                                     ...   \n",
            "178      89th Copom minutes - October 2003   \n",
            "179    88th Copom minutes - September 2003   \n",
            "180       87th Copom minutes - August 2003   \n",
            "181         86th Copom minutes - July 2003   \n",
            "182         85th Copom minutes - June 2003   \n",
            "\n",
            "                                      Link de Download  \\\n",
            "0    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "1    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "2    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "3    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "4    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "..                                                 ...   \n",
            "178  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "179  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "180  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "181  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "182  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "\n",
            "                                          Texto do PDF  \\\n",
            "0    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "1    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "2    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "3    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "4    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "..                                                 ...   \n",
            "178  1 Minutes of the 89th Meeting of the Monetary ...   \n",
            "179  1 Minutes of the 88th Meeting of the Monetary ...   \n",
            "180  1 Minutes of the 87th Meeting of the Monetary ...   \n",
            "181  1 Minutes of the 86th Meeting of the Monetary ...   \n",
            "182  Minutes of the 85th Meeting of the Monetary Po...   \n",
            "\n",
            "                                      Texto Processado  Ano_Mes  \n",
            "0    meeting monetary policy committee meeting date...  2024-12  \n",
            "1    meeting monetary policy committee meeting date...  2024-11  \n",
            "2    meeting monetary policy committee meeting date...  2024-09  \n",
            "3    meeting monetary policy committee meeting date...  2024-07  \n",
            "4    meeting monetary policy committee rd rd meetin...  2024-06  \n",
            "..                                                 ...      ...  \n",
            "178  meeting monetary policy committee (copom) date...  2003-10  \n",
            "179  meeting monetary policy committee (copom) date...  2003-09  \n",
            "180  meeting monetary policy committee (copom) date...  2003-08  \n",
            "181  meeting monetary policy committee (copom) date...  2003-07  \n",
            "182  meeting monetary policy committee (copom) date...  2003-06  \n",
            "\n",
            "[183 rows x 6 columns]\n"
          ]
        }
      ],
      "source": [
        "# Função para extrair ano e mês\n",
        "def extract_year_month(text):\n",
        "    # Caso 1: Formato com dia e mês\n",
        "    match_full = re.search(r'(\\w+)\\s+\\d{1,2}[- ]\\d{1,2},\\s*(\\d{4})', text)\n",
        "    # Caso 2: Formato apenas com mês e ano\n",
        "    match_month_year = re.search(r'(\\w+)\\s+(\\d{4})', text)\n",
        "    # Caso 3: Formato com intervalo de datas\n",
        "    match_interval = re.search(r'(\\w+)\\s+\\d{1,2}\\s*-\\s*(\\w+)\\s+\\d{1,2},\\s*(\\d{4})', text)\n",
        "\n",
        "    if match_full:\n",
        "        month_str = match_full.group(1)\n",
        "        year = match_full.group(2)\n",
        "    elif match_month_year:\n",
        "        month_str = match_month_year.group(1)\n",
        "        year = match_month_year.group(2)\n",
        "    elif match_interval:\n",
        "        month_str = match_interval.group(2)  # Captura o mês da última data\n",
        "        year = match_interval.group(3)\n",
        "    else:\n",
        "        return None\n",
        "\n",
        "    # Converter mês para número\n",
        "    month_num = pd.to_datetime(month_str, format='%B').month\n",
        "    return f\"{year}-{month_num:02d}\"  # Formatar como YYYY-MM\n",
        "\n",
        "# Criar nova coluna com ano e mês\n",
        "df['Ano_Mes'] = df['Texto'].apply(extract_year_month)\n",
        "\n",
        "# Exibir o DataFrame resultante\n",
        "print(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8PHBGhVVyt0L"
      },
      "source": [
        "Usa o regex para encontrar essas expressoes, q sao expressoes que comecam a primeira secao (modelo de ata novo e antigo) e extrai o conteudo a partir dai. Alem disso cria novas variaveis com as secoes novas (A,B,C,D)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nFcN5ZtWbdxE",
        "outputId": "07091736-57ef-44e8-e701-2d65e6336b81"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                      Texto Processado\n",
            "0    a) update economic outlook copoms scenario glo...\n",
            "1    a) update economic outlook copoms scenario glo...\n",
            "2    a) update economic outlook copoms scenario glo...\n",
            "3    a) update economic outlook copoms scenario glo...\n",
            "4    a) update economic outlook copoms scenario glo...\n",
            "..                                                 ...\n",
            "178  recent evolution inflation main price indices ...\n",
            "179  recent evolution inflation ipca increased incr...\n",
            "180  recent evolution inflation inflation rates ref...\n",
            "181  recent evolution inflation inflation continued...\n",
            "182  recent evolution inflation measures inflation ...\n",
            "\n",
            "[183 rows x 1 columns]\n",
            "                                                     A  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                                     B  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                                     C  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                                     D  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                             Footnotes  \n",
            "0    a) update economic outlook copoms scenario glo...  \n",
            "1    a) update economic outlook copoms scenario glo...  \n",
            "2    a) update economic outlook copoms scenario glo...  \n",
            "3    a) update economic outlook copoms scenario glo...  \n",
            "4    a) update economic outlook copoms scenario glo...  \n",
            "..                                                 ...  \n",
            "178  recent evolution inflation main price indices ...  \n",
            "179  recent evolution inflation ipca increased incr...  \n",
            "180  recent evolution inflation inflation rates ref...  \n",
            "181  recent evolution inflation inflation continued...  \n",
            "182  recent evolution inflation measures inflation ...  \n",
            "\n",
            "[183 rows x 5 columns]\n"
          ]
        }
      ],
      "source": [
        "# Definir as expressões que você deseja encontrar\n",
        "expressions = [\n",
        "    r'(?i)(A\\)\\s*update (?:e conomic|economic) outlook (?:c opoms|copoms) (?:reference|baseline|)\\s*scenario)(.*)',\n",
        "    r'(?i)(Recent Economic Developments)(.*)',\n",
        "    r'(?i)(Recent Evolution Inflation)(.*)',\n",
        "]\n",
        "\n",
        "# Função para extrair texto a partir de várias expressões\n",
        "def extract_from_expressions(text):\n",
        "    for pattern in expressions:\n",
        "        match = re.search(pattern, text)\n",
        "        if match:\n",
        "            # Retorna o texto a partir da expressão encontrada até o final\n",
        "            return text[match.start():]  # A partir do início da expressão até o final do texto\n",
        "    return text  # Retorna todo o texto se nenhuma expressão for encontrada\n",
        "\n",
        "df['Texto Processado'] = df['Texto Processado'].apply(extract_from_expressions)\n",
        "\n",
        "\n",
        "# Definir os títulos das seções, permitindo variações com ou sem espaço\n",
        "section_titles = [\n",
        "    r'(?i)A\\)\\s*update (?:e conomic|economic) outlook (?:c opoms|copoms) (?:reference|baseline|)\\s*scenario.*?(?=B\\)\\s*risks around|B\\)\\s*scenarios risk analysis|$)',\n",
        "    r'(?i)(B\\)\\s*risks around|B\\)\\s*scenarios risk analysis).*?(?=C\\)\\s*discussion conduct monetary policy|$)',\n",
        "    r'(?i)C\\)\\s*discussion conduct monetary policy.*?(?=D\\)\\s*monetary policy decision|$)',\n",
        "    r'(?i)D\\)\\s*monetary policy decision.*?(?=Footnotes|$)',\n",
        "    r'(?i)Footnotes.*'  # Captura tudo até o final do texto\n",
        "]\n",
        "\n",
        "\n",
        "# Criar variáveis para cada seção no DataFrame\n",
        "for title in section_titles:\n",
        "    # Usar str.extract para capturar as seções\n",
        "    section_name = re.search(r'A|B|C|D|Footnotes', title).group(0)  # Captura A, B, C, D ou Footnotes\n",
        "    df[section_name] = df['Texto Processado']#.str.extract(f'({title})')[0] mudou 16/01/2025\n",
        "\n",
        "print (df[['Texto Processado']])\n",
        "# Exibir o DataFrame resultante\n",
        "print(df[['A', 'B', 'C', 'D', 'Footnotes']])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Função para preprocessar o texto\n",
        "def preprocess_text(text):\n",
        "    if text:\n",
        "        # Remove números\n",
        "        text = re.sub(r'[0-9]+', '', text)\n",
        "\n",
        "        # Remove pontuação e caracteres especiais, incluindo parênteses\n",
        "        text = re.sub(r'[^\\w\\s]+', '', text)  # Remove toda pontuação, exceto letras e números\n",
        "\n",
        "        # Substituir múltiplos espaços por um único espaço\n",
        "        text = re.sub(r'\\s+', ' ', text)\n",
        "\n",
        "        # Retorna texto em minúsculas\n",
        "        text = text.strip().lower()\n",
        "\n",
        "        # Remove palavras com 1 letra\n",
        "        text = ' '.join([word for word in text.split() if len(word) > 1])\n",
        "\n",
        "        # Remove expressões específicas\n",
        "        for expr in expressions_to_remove:\n",
        "            text = text.replace(expr, '')\n",
        "\n",
        "        # Remove stopwords\n",
        "        text = ' '.join([word for word in text.split() if word not in stop_words])\n",
        "\n",
        "        return text\n",
        "\n",
        "    return ''\n",
        "\n",
        "    # Criar a nova coluna 'Texto Processado'\n",
        "df['Texto Processado2'] = df['Texto Processado'].apply(preprocess_text)"
      ],
      "metadata": {
        "id": "mkozfio0wolU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Com STEMMER\n",
        "from nltk.stem import PorterStemmer\n",
        "\n",
        "# Função para preprocessar o texto\n",
        "def preprocess_text(text, stemmer=PorterStemmer()):\n",
        "    if text:\n",
        "        # Remove números\n",
        "        text = re.sub(r'[0-9]+', '', text)\n",
        "\n",
        "        # Remove pontuação e caracteres especiais, incluindo parênteses\n",
        "        text = re.sub(r'[^\\w\\s]+', '', text)  # Remove toda pontuação, exceto letras e números\n",
        "\n",
        "        # Substituir múltiplos espaços por um único espaço\n",
        "        text = re.sub(r'\\s+', ' ', text)\n",
        "\n",
        "        # Retorna texto em minúsculas\n",
        "        text = text.strip().lower()\n",
        "\n",
        "        # Remove palavras com 1 letra\n",
        "        text = ' '.join([word for word in text.split() if len(word) > 1])\n",
        "\n",
        "        # Remove expressões específicas\n",
        "        for expr in expressions_to_remove:\n",
        "            text = text.replace(expr, '')\n",
        "\n",
        "        # Remove stopwords\n",
        "        text = ' '.join([word for word in text.split() if word not in stop_words])\n",
        "\n",
        "        # Tokenizar\n",
        "        words = text.split()\n",
        "\n",
        "        # Stemming\n",
        "        stemmer = PorterStemmer()\n",
        "        words = [stemmer.stem(word) for word in words]\n",
        "\n",
        "        return ' '.join(words)\n",
        "\n",
        "    return ''\n",
        "\n",
        "    # Criar a nova coluna 'Texto Processado'\n",
        "df['Texto Processado3'] = df['Texto Processado'].apply(preprocess_text)"
      ],
      "metadata": {
        "id": "6qv6GIgradTG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Função para extrair palavras únicas de uma coluna\n",
        "def palavras_unicas(df, coluna):\n",
        "    \"\"\"\n",
        "    Extrai todas as palavras únicas de uma coluna de um DataFrame.\n",
        "\n",
        "    Args:\n",
        "        df: DataFrame.\n",
        "        coluna: Nome da coluna a ser analisada.\n",
        "\n",
        "    Returns:\n",
        "        Lista de palavras únicas.\n",
        "    \"\"\"\n",
        "\n",
        "    # Separar as palavras de cada texto em uma lista\n",
        "    palavras_por_texto = df[coluna].str.split(expand=True).stack()\n",
        "\n",
        "    # Remover valores nulos e duplicados\n",
        "    palavras_unicas = palavras_por_texto.dropna().unique()\n",
        "\n",
        "    return palavras_unicas\n",
        "\n",
        "# Aplicar a função à coluna \"Texto do PDF\"\n",
        "palavras_unicas = palavras_unicas(df, 'Texto Processado2')\n",
        "\n",
        "# Imprimir as palavras únicas\n",
        "print(palavras_unicas)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "3rtSq7luCiGz",
        "outputId": "fbb8f7d9-eec5-40bf-fce9-705fc739cb11"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['update' 'economic' 'outlook' ... 'defaults' 'recessionary' 'averse']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = pd.DataFrame({'Palavras': palavras_unicas})\n",
        "df2.to_csv('palavras.csv', index=False)\n",
        "# Download no Google Colab\n",
        "files.download('palavras.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "bVSTm3gsFT2z",
        "outputId": "b89e9713-11e8-46a4-9534-105e07bcda1d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_3a6f203e-02f7-48e6-a341-77445f04eeb5\", \"palavras.csv\", 54931)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Agrupa pelo trimestre"
      ],
      "metadata": {
        "id": "W-h1e27eiKXM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Cria a variavel trimestre\n",
        "print(df)\n",
        "\n",
        "df['trimestre'] = df['Ano_Mes'].apply(lambda x: pd.to_datetime(x).to_period('Q'))\n",
        "#print(df)\n",
        "\n",
        "#Agrupando pelo trimestre\n",
        "cols_to_join = [col for col in df.columns if col not in ['trimestre']]\n",
        "agg_dict = {}\n",
        "for col in cols_to_join:\n",
        "    agg_dict[col] = lambda x: ' '.join(x.astype(str))\n",
        "\n",
        "df_agrupado = df.groupby('trimestre').agg(agg_dict)\n",
        "df_agrupado = df_agrupado.reset_index()\n",
        "\n",
        "print(df_agrupado)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2xL0AhXgiIpi",
        "outputId": "64840c03-2b6a-4134-bfa8-e800c166e140"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                                  Link  \\\n",
            "0    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "1    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "2    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "3    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "4    https://www.bcb.gov.br/en/publications/copommi...   \n",
            "..                                                 ...   \n",
            "178  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "179  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "180  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "181  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "182  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "\n",
            "                                     Texto  \\\n",
            "0     267th Meeting - December 10-11, 2024   \n",
            "1       266th Meeting - November 5-6, 2024   \n",
            "2    265th Meeting - September 17-18, 2024   \n",
            "3         264th Meeting - July 30-31, 2024   \n",
            "4         263rd Meeting - June 18-19, 2024   \n",
            "..                                     ...   \n",
            "178      89th Copom minutes - October 2003   \n",
            "179    88th Copom minutes - September 2003   \n",
            "180       87th Copom minutes - August 2003   \n",
            "181         86th Copom minutes - July 2003   \n",
            "182         85th Copom minutes - June 2003   \n",
            "\n",
            "                                      Link de Download  \\\n",
            "0    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "1    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "2    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "3    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "4    https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "..                                                 ...   \n",
            "178  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "179  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "180  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "181  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "182  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "\n",
            "                                          Texto do PDF  \\\n",
            "0    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "1    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "2    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "3    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "4    1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "..                                                 ...   \n",
            "178  1 Minutes of the 89th Meeting of the Monetary ...   \n",
            "179  1 Minutes of the 88th Meeting of the Monetary ...   \n",
            "180  1 Minutes of the 87th Meeting of the Monetary ...   \n",
            "181  1 Minutes of the 86th Meeting of the Monetary ...   \n",
            "182  Minutes of the 85th Meeting of the Monetary Po...   \n",
            "\n",
            "                                      Texto Processado  Ano_Mes  \\\n",
            "0    a) update economic outlook copoms scenario glo...  2024-12   \n",
            "1    a) update economic outlook copoms scenario glo...  2024-11   \n",
            "2    a) update economic outlook copoms scenario glo...  2024-09   \n",
            "3    a) update economic outlook copoms scenario glo...  2024-07   \n",
            "4    a) update economic outlook copoms scenario glo...  2024-06   \n",
            "..                                                 ...      ...   \n",
            "178  recent evolution inflation main price indices ...  2003-10   \n",
            "179  recent evolution inflation ipca increased incr...  2003-09   \n",
            "180  recent evolution inflation inflation rates ref...  2003-08   \n",
            "181  recent evolution inflation inflation continued...  2003-07   \n",
            "182  recent evolution inflation measures inflation ...  2003-06   \n",
            "\n",
            "                                                     A  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                                     B  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                                     C  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                                     D  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                             Footnotes  \\\n",
            "0    a) update economic outlook copoms scenario glo...   \n",
            "1    a) update economic outlook copoms scenario glo...   \n",
            "2    a) update economic outlook copoms scenario glo...   \n",
            "3    a) update economic outlook copoms scenario glo...   \n",
            "4    a) update economic outlook copoms scenario glo...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                     Texto Processado2  \\\n",
            "0    update economic outlook copoms scenario global...   \n",
            "1    update economic outlook copoms scenario global...   \n",
            "2    update economic outlook copoms scenario global...   \n",
            "3    update economic outlook copoms scenario global...   \n",
            "4    update economic outlook copoms scenario global...   \n",
            "..                                                 ...   \n",
            "178  recent evolution inflation main price indices ...   \n",
            "179  recent evolution inflation ipca increased incr...   \n",
            "180  recent evolution inflation inflation rates ref...   \n",
            "181  recent evolution inflation inflation continued...   \n",
            "182  recent evolution inflation measures inflation ...   \n",
            "\n",
            "                                     Texto Processado3  \n",
            "0    updat econom outlook copom scenario global env...  \n",
            "1    updat econom outlook copom scenario global env...  \n",
            "2    updat econom outlook copom scenario global env...  \n",
            "3    updat econom outlook copom scenario global env...  \n",
            "4    updat econom outlook copom scenario global env...  \n",
            "..                                                 ...  \n",
            "178  recent evolut inflat main price indic show hig...  \n",
            "179  recent evolut inflat ipca increas increas prev...  \n",
            "180  recent evolut inflat inflat rate reflect lack ...  \n",
            "181  recent evolut inflat inflat continu downward t...  \n",
            "182  recent evolut inflat measur inflat continu dow...  \n",
            "\n",
            "[183 rows x 13 columns]\n",
            "   trimestre                                               Link  \\\n",
            "0     2003Q2  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "1     2003Q3  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "2     2003Q4  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "3     2004Q1  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "4     2004Q2  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "..       ...                                                ...   \n",
            "82    2023Q4  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "83    2024Q1  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "84    2024Q2  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "85    2024Q3  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "86    2024Q4  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "\n",
            "                                                Texto  \\\n",
            "0                      85th Copom minutes - June 2003   \n",
            "1   88th Copom minutes - September 2003 87th Copom...   \n",
            "2   91st Copom minutes - December 2003 90th Copom ...   \n",
            "3   94th Copom minutes - March 2004 93rd Copom min...   \n",
            "4   97th Copom minutes - June 2004 96th Copom minu...   \n",
            "..                                                ...   \n",
            "82  259th Meeting - December 12-13, 2023 258th Mee...   \n",
            "83  261st Meeting - March 19-20, 2024 260th Meetin...   \n",
            "84  263rd Meeting - June 18-19, 2024 262nd Meeting...   \n",
            "85  265th Meeting - September 17-18, 2024 264th Me...   \n",
            "86  267th Meeting - December 10-11, 2024 266th Mee...   \n",
            "\n",
            "                                     Link de Download  \\\n",
            "0   https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "1   https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "2   https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "3   https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "4   https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "..                                                ...   \n",
            "82  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "83  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "84  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "85  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "86  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "\n",
            "                                         Texto do PDF  \\\n",
            "0   Minutes of the 85th Meeting of the Monetary Po...   \n",
            "1   1 Minutes of the 88th Meeting of the Monetary ...   \n",
            "2   Minutes of the 91st Meeting of the Monetary Po...   \n",
            "3   Minutes of the 94th Meeting of the Monetary Po...   \n",
            "4   Minutes of the 97th Meeting of the Monetary Po...   \n",
            "..                                                ...   \n",
            "82  1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "83  1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "84  1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "85  1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "86  1 bcb.gov.br Minutes of the Meeting of the Mon...   \n",
            "\n",
            "                                     Texto Processado  \\\n",
            "0   recent evolution inflation measures inflation ...   \n",
            "1   recent evolution inflation ipca increased incr...   \n",
            "2   recent evolution inflation main inflation indi...   \n",
            "3   recent evolution inflation price deceleration ...   \n",
            "4   recent evolution inflation main inflation indi...   \n",
            "..                                                ...   \n",
            "82  a) update economic outlook copoms scenario glo...   \n",
            "83  a) update economic outlook copoms scenario glo...   \n",
            "84  a) update economic outlook copoms scenario glo...   \n",
            "85  a) update economic outlook copoms scenario glo...   \n",
            "86  a) update economic outlook copoms scenario glo...   \n",
            "\n",
            "                    Ano_Mes  \\\n",
            "0                   2003-06   \n",
            "1   2003-09 2003-08 2003-07   \n",
            "2   2003-12 2003-11 2003-10   \n",
            "3   2004-03 2004-02 2004-01   \n",
            "4   2004-06 2004-05 2004-04   \n",
            "..                      ...   \n",
            "82          2023-12 2023-11   \n",
            "83          2024-03 2024-01   \n",
            "84          2024-06 2024-05   \n",
            "85          2024-09 2024-07   \n",
            "86          2024-12 2024-11   \n",
            "\n",
            "                                                    A  \\\n",
            "0   recent evolution inflation measures inflation ...   \n",
            "1   recent evolution inflation ipca increased incr...   \n",
            "2   recent evolution inflation main inflation indi...   \n",
            "3   recent evolution inflation price deceleration ...   \n",
            "4   recent evolution inflation main inflation indi...   \n",
            "..                                                ...   \n",
            "82  a) update economic outlook copoms scenario glo...   \n",
            "83  a) update economic outlook copoms scenario glo...   \n",
            "84  a) update economic outlook copoms scenario glo...   \n",
            "85  a) update economic outlook copoms scenario glo...   \n",
            "86  a) update economic outlook copoms scenario glo...   \n",
            "\n",
            "                                                    B  \\\n",
            "0   recent evolution inflation measures inflation ...   \n",
            "1   recent evolution inflation ipca increased incr...   \n",
            "2   recent evolution inflation main inflation indi...   \n",
            "3   recent evolution inflation price deceleration ...   \n",
            "4   recent evolution inflation main inflation indi...   \n",
            "..                                                ...   \n",
            "82  a) update economic outlook copoms scenario glo...   \n",
            "83  a) update economic outlook copoms scenario glo...   \n",
            "84  a) update economic outlook copoms scenario glo...   \n",
            "85  a) update economic outlook copoms scenario glo...   \n",
            "86  a) update economic outlook copoms scenario glo...   \n",
            "\n",
            "                                                    C  \\\n",
            "0   recent evolution inflation measures inflation ...   \n",
            "1   recent evolution inflation ipca increased incr...   \n",
            "2   recent evolution inflation main inflation indi...   \n",
            "3   recent evolution inflation price deceleration ...   \n",
            "4   recent evolution inflation main inflation indi...   \n",
            "..                                                ...   \n",
            "82  a) update economic outlook copoms scenario glo...   \n",
            "83  a) update economic outlook copoms scenario glo...   \n",
            "84  a) update economic outlook copoms scenario glo...   \n",
            "85  a) update economic outlook copoms scenario glo...   \n",
            "86  a) update economic outlook copoms scenario glo...   \n",
            "\n",
            "                                                    D  \\\n",
            "0   recent evolution inflation measures inflation ...   \n",
            "1   recent evolution inflation ipca increased incr...   \n",
            "2   recent evolution inflation main inflation indi...   \n",
            "3   recent evolution inflation price deceleration ...   \n",
            "4   recent evolution inflation main inflation indi...   \n",
            "..                                                ...   \n",
            "82  a) update economic outlook copoms scenario glo...   \n",
            "83  a) update economic outlook copoms scenario glo...   \n",
            "84  a) update economic outlook copoms scenario glo...   \n",
            "85  a) update economic outlook copoms scenario glo...   \n",
            "86  a) update economic outlook copoms scenario glo...   \n",
            "\n",
            "                                            Footnotes  \\\n",
            "0   recent evolution inflation measures inflation ...   \n",
            "1   recent evolution inflation ipca increased incr...   \n",
            "2   recent evolution inflation main inflation indi...   \n",
            "3   recent evolution inflation price deceleration ...   \n",
            "4   recent evolution inflation main inflation indi...   \n",
            "..                                                ...   \n",
            "82  a) update economic outlook copoms scenario glo...   \n",
            "83  a) update economic outlook copoms scenario glo...   \n",
            "84  a) update economic outlook copoms scenario glo...   \n",
            "85  a) update economic outlook copoms scenario glo...   \n",
            "86  a) update economic outlook copoms scenario glo...   \n",
            "\n",
            "                                    Texto Processado2  \\\n",
            "0   recent evolution inflation measures inflation ...   \n",
            "1   recent evolution inflation ipca increased incr...   \n",
            "2   recent evolution inflation main inflation indi...   \n",
            "3   recent evolution inflation price deceleration ...   \n",
            "4   recent evolution inflation main inflation indi...   \n",
            "..                                                ...   \n",
            "82  update economic outlook copoms scenario global...   \n",
            "83  update economic outlook copoms scenario global...   \n",
            "84  update economic outlook copoms scenario global...   \n",
            "85  update economic outlook copoms scenario global...   \n",
            "86  update economic outlook copoms scenario global...   \n",
            "\n",
            "                                    Texto Processado3  \n",
            "0   recent evolut inflat measur inflat continu dow...  \n",
            "1   recent evolut inflat ipca increas increas prev...  \n",
            "2   recent evolut inflat main inflat indic remain ...  \n",
            "3   recent evolut inflat price deceler part consum...  \n",
            "4   recent evolut inflat main inflat indic increas...  \n",
            "..                                                ...  \n",
            "82  updat econom outlook copom scenario global env...  \n",
            "83  updat econom outlook copom scenario global env...  \n",
            "84  updat econom outlook copom scenario global env...  \n",
            "85  updat econom outlook copom scenario global env...  \n",
            "86  updat econom outlook copom scenario global env...  \n",
            "\n",
            "[87 rows x 14 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#https://www.ibge.gov.br/estatisticas/economicas/contas-nacionais/9300-contas-nacionais-trimestrais.html?=&t=resultados\n",
        "\n",
        "# Fazendo upload do arquivo XLS\n",
        "uploaded = files.upload()\n",
        "\n",
        "# Carregar as abas, iniciando da 5ª linha e sem considerar a 5ª linha como cabeçalho\n",
        "df_taxa_trimestral = pd.read_excel('Tab_Compl_CNT.xls', sheet_name='Taxa Trimestral', engine='xlrd', skiprows=4, header=None)\n",
        "df_trimestre_contra_trimestre = pd.read_excel('Tab_Compl_CNT.xls', sheet_name='Trimestre contra Trimestre Ant.', engine='xlrd', skiprows=4, header=None)\n",
        "\n",
        "# Realizar o merge pela coluna A\n",
        "df_trimestres_merged = pd.merge(df_taxa_trimestral, df_trimestre_contra_trimestre, on=0, how='outer')\n",
        "\n",
        "# Selecionar as colunas Q de ambas as abas\n",
        "df_trimestres_merged = df_trimestres_merged[[0, '17_x', '16_y']]\n",
        "\n",
        "# Renomear as colunas 0, 17_x e 16_y para nomes mais descritivos\n",
        "df_trimestres_merged = df_trimestres_merged.rename(columns={0: 'trimestre', '17_x': 'PIB_Taxa_Trimestral', '16_y': 'PIB_Trimestre_Contra_Trimestre'})\n",
        "\n",
        "# Dicionário para converter números romanos para números árabes\n",
        "romano_para_árabe = {\n",
        "    'I': 1,\n",
        "    'II': 2,\n",
        "    'III': 3,\n",
        "    'IV': 4\n",
        "}\n",
        "\n",
        "# Função para converter os trimestres romanos em formato 'YYYYQn'\n",
        "def converter_trimestre(trimestre):\n",
        "    if isinstance(trimestre, str):\n",
        "        ano, trimestre_romano = trimestre.split('.')  # Divide o ano e o número romano\n",
        "        trimestre_normal = romano_para_árabe.get(trimestre_romano, None)  # Converte o número romano para número árabe\n",
        "        if trimestre_normal:\n",
        "            return f\"{ano}Q{trimestre_normal}\"  # Formata como 'YYYYQn'\n",
        "    return trimestre\n",
        "\n",
        "# Aplicar a conversão à coluna 'trimestre'\n",
        "df_trimestres_merged['trimestre'] = df_trimestres_merged['trimestre'].apply(converter_trimestre)\n",
        "\n",
        "# Converter a coluna 'trimestre' para o tipo 'period[Q]'\n",
        "df_trimestres_merged['trimestre'] = pd.to_datetime(df_trimestres_merged['trimestre']).dt.to_period('Q')\n",
        "\n",
        "# Criar colunas com defasagens\n",
        "df_trimestres_merged['PIB_Taxa_Trimestral_lag1'] = df_trimestres_merged['PIB_Taxa_Trimestral'].shift(1)\n",
        "df_trimestres_merged['PIB_Taxa_Trimestral_lag2'] = df_trimestres_merged['PIB_Taxa_Trimestral'].shift(2)\n",
        "df_trimestres_merged['PIB_Trimestre_Contra_Trimestre_lag1'] = df_trimestres_merged['PIB_Trimestre_Contra_Trimestre'].shift(1)\n",
        "df_trimestres_merged['PIB_Trimestre_Contra_Trimestre_lag2'] = df_trimestres_merged['PIB_Trimestre_Contra_Trimestre'].shift(2)\n",
        "\n",
        "print(df_trimestres_merged)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 822
        },
        "id": "gsH0JsPGNjJd",
        "outputId": "984e771d-d1f0-42dd-b219-bc3e31bbded3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8cf300ca-133b-4095-b633-f16078a4113e\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-8cf300ca-133b-4095-b633-f16078a4113e\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving Tab_Compl_CNT.xls to Tab_Compl_CNT.xls\n",
            "    trimestre  PIB_Taxa_Trimestral  PIB_Trimestre_Contra_Trimestre  \\\n",
            "0      1996Q1            -0.579873                             NaN   \n",
            "1      1996Q2             1.347193                        1.273736   \n",
            "2      1996Q3             5.647213                        3.465120   \n",
            "3      1996Q4             2.274484                       -0.741284   \n",
            "4      1997Q1             3.400572                        1.010250   \n",
            "..        ...                  ...                             ...   \n",
            "110    2023Q3             2.355027                        0.196443   \n",
            "111    2023Q4             2.380569                        0.150298   \n",
            "112    2024Q1             2.559744                        1.054323   \n",
            "113    2024Q2             3.334259                        1.384627   \n",
            "114    2024Q3             4.047541                        0.913178   \n",
            "\n",
            "     PIB_Taxa_Trimestral_lag1  PIB_Taxa_Trimestral_lag2  \\\n",
            "0                         NaN                       NaN   \n",
            "1                   -0.579873                       NaN   \n",
            "2                    1.347193                 -0.579873   \n",
            "3                    5.647213                  1.347193   \n",
            "4                    2.274484                  5.647213   \n",
            "..                        ...                       ...   \n",
            "110                  3.850270                  4.444340   \n",
            "111                  2.355027                  3.850270   \n",
            "112                  2.380569                  2.355027   \n",
            "113                  2.559744                  2.380569   \n",
            "114                  3.334259                  2.559744   \n",
            "\n",
            "     PIB_Trimestre_Contra_Trimestre_lag1  PIB_Trimestre_Contra_Trimestre_lag2  \n",
            "0                                    NaN                                  NaN  \n",
            "1                                    NaN                                  NaN  \n",
            "2                               1.273736                                  NaN  \n",
            "3                               3.465120                             1.273736  \n",
            "4                              -0.741284                             3.465120  \n",
            "..                                   ...                                  ...  \n",
            "110                             0.800524                             1.376710  \n",
            "111                             0.196443                             0.800524  \n",
            "112                             0.150298                             0.196443  \n",
            "113                             1.054323                             0.150298  \n",
            "114                             1.384627                             1.054323  \n",
            "\n",
            "[115 rows x 7 columns]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-23-ee21e64f4acb>:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
            "  df_trimestres_merged['trimestre'] = pd.to_datetime(df_trimestres_merged['trimestre']).dt.to_period('Q')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Agora, você pode fazer o merge\n",
        "df_final = pd.merge(df_agrupado, df_trimestres_merged[['trimestre', 'PIB_Taxa_Trimestral', 'PIB_Trimestre_Contra_Trimestre','PIB_Taxa_Trimestral_lag1','PIB_Taxa_Trimestral_lag2','PIB_Trimestre_Contra_Trimestre_lag1','PIB_Trimestre_Contra_Trimestre_lag2']], on='trimestre', how='outer')\n",
        "\n",
        "#drop de 2003-03 para tras\n",
        "# Filtra para manter apenas os registros a partir de 2003-03\n",
        "df_final = df_final[df_final['trimestre'] >= '2003Q2']\n",
        "\n",
        "# Verifique o resultado final\n",
        "print(df_final.head())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9tXgcINQmzaX",
        "outputId": "fdb2f4d0-9ae4-4570-90bd-dfda31c1a81d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   trimestre                                               Link  \\\n",
            "29    2003Q2  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "30    2003Q3  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "31    2003Q4  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "32    2004Q1  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "33    2004Q2  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "\n",
            "                                                Texto  \\\n",
            "29                     85th Copom minutes - June 2003   \n",
            "30  88th Copom minutes - September 2003 87th Copom...   \n",
            "31  91st Copom minutes - December 2003 90th Copom ...   \n",
            "32  94th Copom minutes - March 2004 93rd Copom min...   \n",
            "33  97th Copom minutes - June 2004 96th Copom minu...   \n",
            "\n",
            "                                     Link de Download  \\\n",
            "29  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "30  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "31  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "32  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "33  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "\n",
            "                                         Texto do PDF  \\\n",
            "29  Minutes of the 85th Meeting of the Monetary Po...   \n",
            "30  1 Minutes of the 88th Meeting of the Monetary ...   \n",
            "31  Minutes of the 91st Meeting of the Monetary Po...   \n",
            "32  Minutes of the 94th Meeting of the Monetary Po...   \n",
            "33  Minutes of the 97th Meeting of the Monetary Po...   \n",
            "\n",
            "                                     Texto Processado  \\\n",
            "29  recent evolution inflation measures inflation ...   \n",
            "30  recent evolution inflation ipca increased incr...   \n",
            "31  recent evolution inflation main inflation indi...   \n",
            "32  recent evolution inflation price deceleration ...   \n",
            "33  recent evolution inflation main inflation indi...   \n",
            "\n",
            "                    Ano_Mes  \\\n",
            "29                  2003-06   \n",
            "30  2003-09 2003-08 2003-07   \n",
            "31  2003-12 2003-11 2003-10   \n",
            "32  2004-03 2004-02 2004-01   \n",
            "33  2004-06 2004-05 2004-04   \n",
            "\n",
            "                                                    A  \\\n",
            "29  recent evolution inflation measures inflation ...   \n",
            "30  recent evolution inflation ipca increased incr...   \n",
            "31  recent evolution inflation main inflation indi...   \n",
            "32  recent evolution inflation price deceleration ...   \n",
            "33  recent evolution inflation main inflation indi...   \n",
            "\n",
            "                                                    B  \\\n",
            "29  recent evolution inflation measures inflation ...   \n",
            "30  recent evolution inflation ipca increased incr...   \n",
            "31  recent evolution inflation main inflation indi...   \n",
            "32  recent evolution inflation price deceleration ...   \n",
            "33  recent evolution inflation main inflation indi...   \n",
            "\n",
            "                                                    C  \\\n",
            "29  recent evolution inflation measures inflation ...   \n",
            "30  recent evolution inflation ipca increased incr...   \n",
            "31  recent evolution inflation main inflation indi...   \n",
            "32  recent evolution inflation price deceleration ...   \n",
            "33  recent evolution inflation main inflation indi...   \n",
            "\n",
            "                                                    D  \\\n",
            "29  recent evolution inflation measures inflation ...   \n",
            "30  recent evolution inflation ipca increased incr...   \n",
            "31  recent evolution inflation main inflation indi...   \n",
            "32  recent evolution inflation price deceleration ...   \n",
            "33  recent evolution inflation main inflation indi...   \n",
            "\n",
            "                                            Footnotes  \\\n",
            "29  recent evolution inflation measures inflation ...   \n",
            "30  recent evolution inflation ipca increased incr...   \n",
            "31  recent evolution inflation main inflation indi...   \n",
            "32  recent evolution inflation price deceleration ...   \n",
            "33  recent evolution inflation main inflation indi...   \n",
            "\n",
            "                                    Texto Processado2  \\\n",
            "29  recent evolution inflation measures inflation ...   \n",
            "30  recent evolution inflation ipca increased incr...   \n",
            "31  recent evolution inflation main inflation indi...   \n",
            "32  recent evolution inflation price deceleration ...   \n",
            "33  recent evolution inflation main inflation indi...   \n",
            "\n",
            "                                    Texto Processado3  PIB_Taxa_Trimestral  \\\n",
            "29  recent evolut inflat measur inflat continu dow...             0.781242   \n",
            "30  recent evolut inflat ipca increas increas prev...             0.626181   \n",
            "31  recent evolut inflat main inflat indic remain ...             0.588712   \n",
            "32  recent evolut inflat price deceler part consum...             3.878082   \n",
            "33  recent evolut inflat main inflat indic increas...             6.308257   \n",
            "\n",
            "    PIB_Trimestre_Contra_Trimestre  PIB_Taxa_Trimestral_lag1  \\\n",
            "29                       -0.836466                  2.667028   \n",
            "30                        0.839986                  0.781242   \n",
            "31                        1.035187                  0.626181   \n",
            "32                        1.488637                  0.588712   \n",
            "33                        2.690959                  3.878082   \n",
            "\n",
            "    PIB_Taxa_Trimestral_lag2  PIB_Trimestre_Contra_Trimestre_lag1  \\\n",
            "29                  5.172628                            -0.321290   \n",
            "30                  2.667028                            -0.836466   \n",
            "31                  0.781242                             0.839986   \n",
            "32                  0.626181                             1.035187   \n",
            "33                  0.588712                             1.488637   \n",
            "\n",
            "    PIB_Trimestre_Contra_Trimestre_lag2  \n",
            "29                             0.981180  \n",
            "30                            -0.321290  \n",
            "31                            -0.836466  \n",
            "32                             0.839986  \n",
            "33                             1.035187  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QqBKgooUzozy"
      },
      "source": [
        "Exportando para um arquivo xlsx ou csv (csv com problema de configuracao)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "YUrlrSVAKIAv",
        "outputId": "d7cce9a3-dc53-4474-8482-5600507322d4"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_642425e2-5df8-48f8-be1d-8ea4f6ddf26b\", \"dados_extraidos.xlsx\", 3591667)"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "#exportando para um arquivo csv\n",
        "#print(df_final.head())\n",
        "#output_csv = 'dados_extraidos.csv'  # Nome do arquivo CSV\n",
        "#df_final.to_csv(output_csv, index=False)\n",
        "\n",
        "# Exportando para Excel\n",
        "output_excel = 'dados_extraidos.xlsx'\n",
        "df_final.to_excel(output_excel, index=False)\n",
        "\n",
        "# Download no Google Colab\n",
        "files.download(output_excel)\n",
        "\n",
        "# Se você estiver usando Google Colab, faça o download do arquivo CSV\n",
        "#try:\n",
        "    #from google.colab import files\n",
        "    #files.download(output_csv)\n",
        "#except ImportError:\n",
        "    #print(\"Não é um ambiente Colab. O arquivo está salvo como:\", output_csv)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C5GbV214Re_H"
      },
      "source": [
        "--------------------------------------------------------------------------------"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Dados pib antigo - Nao estou usando mais\n"
      ],
      "metadata": {
        "id": "-SI1VqamffCs"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "jDDlDELndUq1",
        "outputId": "71aec8c2-8e72-4434-af43-f2f7554dc072"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-7aea01f9-8136-448f-bad0-2a955bd9e940\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-7aea01f9-8136-448f-bad0-2a955bd9e940\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving pib_trimestral.csv to pib_trimestral.csv\n",
            "DataFrame após a leitura:\n",
            "   1º trimestre 1996  2º trimestre 1996  3º trimestre 1996  4º trimestre 1996  \\\n",
            "0                2.5                2.1                2.3                2.2   \n",
            "\n",
            "   1º trimestre 1997  2º trimestre 1997  3º trimestre 1997  4º trimestre 1997  \\\n",
            "0                3.2                4.0                3.0                3.4   \n",
            "\n",
            "   1º trimestre 1998  2º trimestre 1998  ...  1º trimestre 2022  \\\n",
            "0                2.8                2.0  ...                4.7   \n",
            "\n",
            "   2º trimestre 2022  3º trimestre 2022  4º trimestre 2022  1º trimestre 2023  \\\n",
            "0                2.7                2.7                3.0                3.7   \n",
            "\n",
            "   2º trimestre 2023  3º trimestre 2023  4º trimestre 2023  1º trimestre 2024  \\\n",
            "0                3.7                3.1                2.9                2.5   \n",
            "\n",
            "   2º trimestre 2024  \n",
            "0                2.5  \n",
            "\n",
            "[1 rows x 114 columns]\n",
            "DataFrame derretido:\n",
            "           Trimestre  Valor\n",
            "0  1º trimestre 1996    2.5\n",
            "1  2º trimestre 1996    2.1\n",
            "2  3º trimestre 1996    2.3\n",
            "3  4º trimestre 1996    2.2\n",
            "4  1º trimestre 1997    3.2\n",
            "DataFrame com o mapeamento:\n",
            "            Trimestre  Ano_Mes\n",
            "0   1º trimestre 1996  1996-03\n",
            "1   2º trimestre 1996  1996-06\n",
            "2   3º trimestre 1996  1996-09\n",
            "3   4º trimestre 1996  1996-12\n",
            "4   1º trimestre 1997  1997-03\n",
            "5   2º trimestre 1997  1997-06\n",
            "6   3º trimestre 1997  1997-09\n",
            "7   4º trimestre 1997  1997-12\n",
            "8   1º trimestre 1998  1998-03\n",
            "9   2º trimestre 1998  1998-06\n",
            "10  3º trimestre 1998  1998-09\n",
            "11  4º trimestre 1998  1998-12\n",
            "12  1º trimestre 1999  1999-03\n",
            "13  2º trimestre 1999  1999-06\n",
            "14  3º trimestre 1999  1999-09\n",
            "15  4º trimestre 1999  1999-12\n",
            "16  1º trimestre 2000  2000-03\n",
            "17  2º trimestre 2000  2000-06\n",
            "18  3º trimestre 2000  2000-09\n",
            "19  4º trimestre 2000  2000-12\n",
            "Número de valores ausentes em 'Ano_Mes': 0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-09549212-51c8-4305-a9ff-57207d4fe4e9\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-09549212-51c8-4305-a9ff-57207d4fe4e9\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving pib_trimestral2.csv to pib_trimestral2.csv\n",
            "                                                 Link  \\\n",
            "29  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "30  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "31  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "32  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "33  https://www.bcb.gov.br/en/publications/copommi...   \n",
            "\n",
            "                                  Texto  \\\n",
            "29        84th Copom minutes - May 2003   \n",
            "30       85th Copom minutes - June 2003   \n",
            "31       86th Copom minutes - July 2003   \n",
            "32     87th Copom minutes - August 2003   \n",
            "33  88th Copom minutes - September 2003   \n",
            "\n",
            "                                     Link de Download  \\\n",
            "29  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "30  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "31  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "32  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "33  https://www.bcb.gov.br/content/copom/copomminu...   \n",
            "\n",
            "                                         Texto do PDF  \\\n",
            "29  Minutes of the 84th Meeting of the Monetary Po...   \n",
            "30  Minutes of the 85th Meeting of the Monetary Po...   \n",
            "31  1 Minutes of the 86th Meeting of the Monetary ...   \n",
            "32  1 Minutes of the 87th Meeting of the Monetary ...   \n",
            "33  1 Minutes of the 88th Meeting of the Monetary ...   \n",
            "\n",
            "                                     Texto Processado  Ano_Mes    A    B    C  \\\n",
            "29  recent evolution inflation measures inflation ...  2003-05  NaN  NaN  NaN   \n",
            "30  recent evolution inflation measures inflation ...  2003-06  NaN  NaN  NaN   \n",
            "31  recent evolution inflation inflation continued...  2003-07  NaN  NaN  NaN   \n",
            "32  recent evolution inflation inflation rates ref...  2003-08  NaN  NaN  NaN   \n",
            "33  recent evolution inflation ipca increased incr...  2003-09  NaN  NaN  NaN   \n",
            "\n",
            "      D Footnotes                                  Texto Processado2  Valor  \\\n",
            "29  NaN       NaN  recent evolution inflation measures inflation ...    NaN   \n",
            "30  NaN       NaN  recent evolution inflation measures inflation ...    3.2   \n",
            "31  NaN       NaN  recent evolution inflation inflation continued...    NaN   \n",
            "32  NaN       NaN  recent evolution inflation inflation rates ref...    NaN   \n",
            "33  NaN       NaN  recent evolution inflation ipca increased incr...    2.3   \n",
            "\n",
            "      Valor2 trimestre  \n",
            "29       NaN    2003Q2  \n",
            "30  418987.0    2003Q2  \n",
            "31       NaN    2003Q3  \n",
            "32       NaN    2003Q3  \n",
            "33  439350.0    2003Q3  \n"
          ]
        }
      ],
      "source": [
        "# https://www.ibge.gov.br/estatisticas/economicas/contas-nacionais/9300-contas-nacionais-trimestrais.html?=&t=series-historicas&utm_source=landing&utm_medium=explica&utm_campaign=pib#evolucao-taxa\n",
        "\n",
        "# Fazendo upload do arquivo CSV\n",
        "uploaded = files.upload()\n",
        "\n",
        "# Carregar o arquivo CSV usando o delimitador correto\n",
        "df_trimestres = pd.read_csv('pib_trimestral.csv', delimiter=';', header=1)  # O cabeçalho correto\n",
        "\n",
        "# Excluir a primeira coluna se necessário (baseado no que você descreveu)\n",
        "df_trimestres = df_trimestres.iloc[:1, 1:]  # Remove a primeira coluna\n",
        "\n",
        "\n",
        "# Verifique o DataFrame após as alterações\n",
        "print(\"DataFrame após a leitura:\")\n",
        "print(df_trimestres.head())\n",
        "\n",
        "# Agora, precisamos reorganizar os dados\n",
        "# Vamos usar melt para transformar os dados\n",
        "df_trimestres_melted = df_trimestres.melt(var_name='Trimestre', value_name='Valor')\n",
        "\n",
        "# Verifique o DataFrame derretido\n",
        "print(\"DataFrame derretido:\")\n",
        "print(df_trimestres_melted.head())\n",
        "\n",
        "# Função para gerar mapeamento de trimestres\n",
        "def gerar_mapeamento_trimestres(inicio_ano, fim_ano):\n",
        "    mapeamento = {}\n",
        "    for ano in range(inicio_ano, fim_ano + 1):\n",
        "        for trimestre in range(1, 5):\n",
        "            mes = trimestre * 3  # 1: Março, 2: Junho, 3: Setembro, 4: Dezembro\n",
        "            ano_mes = f\"{ano}-{mes:02d}\"\n",
        "            mapeamento[f\"{trimestre}º trimestre {ano}\"] = ano_mes\n",
        "    return mapeamento\n",
        "\n",
        "# Gerar mapeamento de 1996 a 2024\n",
        "mapeamento_trimestre = gerar_mapeamento_trimestres(1996, 2024)\n",
        "df_trimestres_melted['Ano_Mes'] = df_trimestres_melted['Trimestre'].map(mapeamento_trimestre)\n",
        "\n",
        "print(\"DataFrame com o mapeamento:\")\n",
        "print(df_trimestres_melted[['Trimestre', 'Ano_Mes']].head(20))  # Exibir as primeiras 20 linhas\n",
        "\n",
        "# Verificar se há valores ausentes na coluna Ano_Mes\n",
        "valores_ausentes = df_trimestres_melted['Ano_Mes'].isna().sum()\n",
        "print(f\"Número de valores ausentes em 'Ano_Mes': {valores_ausentes}\")\n",
        "\n",
        "\n",
        "# Fazer upload do segundo arquivo CSV\n",
        "uploaded2 = files.upload()\n",
        "\n",
        "# Carregar o segundo arquivo CSV\n",
        "df_trimestres2 = pd.read_csv('pib_trimestral2.csv', delimiter=';', header=1)  # Ajuste o delimitador e o cabeçalho conforme necessário\n",
        "\n",
        "# Excluir a primeira coluna no segundo DataFrame, se necessário\n",
        "df_trimestres2 = df_trimestres2.iloc[:1, 1:]  # Remove a primeira coluna\n",
        "\n",
        "# Transformar os dados do segundo DataFrame usando melt\n",
        "df_trimestres2_melted = df_trimestres2.melt(var_name='Trimestre', value_name='Valor2')\n",
        "\n",
        "# Gerar novamente o mapeamento de trimestres\n",
        "df_trimestres2_melted['Ano_Mes'] = df_trimestres2_melted['Trimestre'].map(mapeamento_trimestre)\n",
        "\n",
        "\n",
        "\n",
        "# Exibir trimestres e seus respectivos Ano_Mes\n",
        "#for trimestre in df_trimestres_melted['Trimestre'].unique():\n",
        "#    print(f\"{trimestre}: {df_trimestres_melted[df_trimestres_melted['Trimestre'] == trimestre]['Ano_Mes'].values}\")\n",
        "\n",
        "\n",
        "# Verifique o resultado do mapeamento\n",
        "#print(\"Trimestres únicos:\")\n",
        "#print(df_trimestres_melted['Trimestre'].unique())\n",
        "#print(df_trimestres_melted[['Ano_Mes', 'Valor']])\n",
        "\n",
        "\n",
        "# Agora, mescle os dois dataframes (supondo que o df original esteja carregado em df)\n",
        "df_trimestres_prov = pd.merge(df_trimestres_melted[['Ano_Mes', 'Valor']], df_trimestres2_melted[['Ano_Mes', 'Valor2']], on='Ano_Mes', how='outer')\n",
        "df_final = pd.merge(df, df_trimestres_prov[['Ano_Mes', 'Valor', 'Valor2']], on='Ano_Mes', how='outer')\n",
        "#df_final = df_final.dropna(subset=['Texto Processado'])\n",
        "\n",
        "\n",
        "#drop de 2003-03 para tras\n",
        "# Filtra para manter apenas os registros a partir de 2003-03\n",
        "df_final = df_final[df_final['Ano_Mes'] >= '2003-04']\n",
        "\n",
        "df_final['trimestre'] = df_final['Ano_Mes'].apply(lambda x: pd.to_datetime(x).to_period('Q'))\n",
        "\n",
        "# Verifique o resultado final\n",
        "print(df_final.head())"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}