hipe2022-datasets-stats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a5be4e76",
   "metadata": {},
   "source": [
    "# HIPE 2022 dataset statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3a7b60b",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "39ac9869",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "from typing import List, Dict\n",
    "from hipe_commons.helpers.tsv import parse_tsv, ENTITY_TYPES, HipeDocument, HipeEntity\n",
    "from hipe_commons.stats import describe_dataset, compute_entities_stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e16b1a3",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a7f403d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_entities(documents: List[HipeDocument]) -> Dict[str, List[HipeEntity]]:\n",
    "    \"\"\"Simple function to gather all entities from documents in a dataset, divided by type.\n",
    "\n",
    "    :param documents: Input documents in HIPE format\n",
    "    :type documents: List[HipeDocument]\n",
    "    :return: A list of `HipeEntity` objects\n",
    "    :rtype: Dict[str, List[HipeEntity]]\n",
    "    \"\"\"\n",
    "    all_entities = {}\n",
    "\n",
    "    for doc in ajmc_sample_en_docs:\n",
    "        for e_type in ENTITY_TYPES:\n",
    "            \n",
    "            if e_type in doc.entities:\n",
    "\n",
    "                if e_type not in all_entities:\n",
    "                    all_entities[e_type] = []\n",
    "\n",
    "                all_entities[e_type] += doc.entities[e_type]\n",
    "    return all_entities"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86eee725",
   "metadata": {},
   "source": [
    "# HIPE 2022 Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0f6a13c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "RELEASE_VERSION = \"v2.1\"\n",
    "HIPE2022_data_path = f\"./data/{RELEASE_VERSION}/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fc6a3e6",
   "metadata": {},
   "source": [
    "## ajmc\n",
    "\n",
    "See the [README file](./documentation/README-ajmc.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23ea36b7",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f1d7980b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# EN\n",
    "ajmc_train_en_path = os.path.join(HIPE2022_data_path, f\"ajmc/en/HIPE-2022-{RELEASE_VERSION}-ajmc-train-en.tsv\")\n",
    "ajmc_dev_en_path = os.path.join(HIPE2022_data_path, f\"ajmc/en/HIPE-2022-{RELEASE_VERSION}-ajmc-dev-en.tsv\")\n",
    "\n",
    "# DE\n",
    "ajmc_train_de_path = os.path.join(HIPE2022_data_path, f\"ajmc/de/HIPE-2022-{RELEASE_VERSION}-ajmc-train-de.tsv\")\n",
    "ajmc_dev_de_path = os.path.join(HIPE2022_data_path, f\"ajmc/de/HIPE-2022-{RELEASE_VERSION}-ajmc-dev-de.tsv\")\n",
    "\n",
    "# FR\n",
    "ajmc_train_fr_path = os.path.join(HIPE2022_data_path, f\"ajmc/fr/HIPE-2022-{RELEASE_VERSION}-ajmc-train-fr.tsv\")\n",
    "ajmc_dev_fr_path = os.path.join(HIPE2022_data_path, f\"ajmc/fr/HIPE-2022-{RELEASE_VERSION}-ajmc-dev-fr.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98069843",
   "metadata": {},
   "source": [
    "### ajmc EN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fe36fe5",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d7d48341",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_dev_en_docs = parse_tsv(file_path=ajmc_dev_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d4d2e96e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/ajmc/en/HIPE-2022-v2.1-ajmc-dev-en.tsv \n",
      "Number of documents: 14 \n",
      "Number of entities: {'coarse_lit': 416, 'fine_lit': 416} \n",
      "Number of tokens: 6506 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-------+---------+\n",
      "|       |   count |\n",
      "+=======+=========+\n",
      "| date  |       5 |\n",
      "+-------+---------+\n",
      "| loc   |       3 |\n",
      "+-------+---------+\n",
      "| pers  |     130 |\n",
      "+-------+---------+\n",
      "| scope |     162 |\n",
      "+-------+---------+\n",
      "| work  |     116 |\n",
      "+-------+---------+\n",
      "fine_lit\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| date         |       5 |\n",
      "+--------------+---------+\n",
      "| loc          |       3 |\n",
      "+--------------+---------+\n",
      "| pers.author  |      69 |\n",
      "+--------------+---------+\n",
      "| pers.editor  |       9 |\n",
      "+--------------+---------+\n",
      "| pers.myth    |      50 |\n",
      "+--------------+---------+\n",
      "| pers.other   |       2 |\n",
      "+--------------+---------+\n",
      "| scope        |     162 |\n",
      "+--------------+---------+\n",
      "| work.fragm   |       1 |\n",
      "+--------------+---------+\n",
      "| work.primlit |     103 |\n",
      "+--------------+---------+\n",
      "| work.seclit  |      12 |\n",
      "+--------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_dev_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cae8b1f1",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9c59de36",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_train_en_docs = parse_tsv(file_path=ajmc_train_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "27ed577b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/ajmc/en/HIPE-2022-v2.1-ajmc-train-en.tsv \n",
      "Number of documents: 60 \n",
      "Number of entities: {'coarse_lit': 1823, 'fine_lit': 1823, 'nested': 4} \n",
      "Number of tokens: 30932 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+--------+---------+\n",
      "|        |   count |\n",
      "+========+=========+\n",
      "| date   |      12 |\n",
      "+--------+---------+\n",
      "| loc    |      39 |\n",
      "+--------+---------+\n",
      "| object |       3 |\n",
      "+--------+---------+\n",
      "| pers   |     618 |\n",
      "+--------+---------+\n",
      "| scope  |     684 |\n",
      "+--------+---------+\n",
      "| work   |     467 |\n",
      "+--------+---------+\n",
      "fine_lit\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| date           |      12 |\n",
      "+----------------+---------+\n",
      "| loc            |      39 |\n",
      "+----------------+---------+\n",
      "| object.manuscr |       3 |\n",
      "+----------------+---------+\n",
      "| pers.author    |     237 |\n",
      "+----------------+---------+\n",
      "| pers.editor    |      28 |\n",
      "+----------------+---------+\n",
      "| pers.myth      |     331 |\n",
      "+----------------+---------+\n",
      "| pers.other     |      22 |\n",
      "+----------------+---------+\n",
      "| scope          |     684 |\n",
      "+----------------+---------+\n",
      "| work.fragm     |       2 |\n",
      "+----------------+---------+\n",
      "| work.journal   |       1 |\n",
      "+----------------+---------+\n",
      "| work.other     |       3 |\n",
      "+----------------+---------+\n",
      "| work.primlit   |     414 |\n",
      "+----------------+---------+\n",
      "| work.seclit    |      47 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| pers.author  |       1 |\n",
      "+--------------+---------+\n",
      "| pers.myth    |       1 |\n",
      "+--------------+---------+\n",
      "| scope        |       1 |\n",
      "+--------------+---------+\n",
      "| work.primlit |       1 |\n",
      "+--------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_train_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85a78a1a",
   "metadata": {},
   "source": [
    "### ajmc DE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2c54bf0",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e10a9e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_dev_de_docs = parse_tsv(file_path=ajmc_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7b3fd754",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/ajmc/de/HIPE-2022-v2.1-ajmc-dev-de.tsv \n",
      "Number of documents: 14 \n",
      "Number of entities: {'coarse_lit': 403, 'fine_lit': 403, 'nested': 2} \n",
      "Number of tokens: 4702 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+--------+---------+\n",
      "|        |   count |\n",
      "+========+=========+\n",
      "| loc    |      10 |\n",
      "+--------+---------+\n",
      "| object |       4 |\n",
      "+--------+---------+\n",
      "| pers   |     162 |\n",
      "+--------+---------+\n",
      "| scope  |     157 |\n",
      "+--------+---------+\n",
      "| work   |      70 |\n",
      "+--------+---------+\n",
      "fine_lit\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc            |      10 |\n",
      "+----------------+---------+\n",
      "| object.manuscr |       4 |\n",
      "+----------------+---------+\n",
      "| pers.author    |      65 |\n",
      "+----------------+---------+\n",
      "| pers.editor    |       7 |\n",
      "+----------------+---------+\n",
      "| pers.myth      |      87 |\n",
      "+----------------+---------+\n",
      "| pers.other     |       3 |\n",
      "+----------------+---------+\n",
      "| scope          |     157 |\n",
      "+----------------+---------+\n",
      "| work.primlit   |      70 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| loc       |       1 |\n",
      "+-----------+---------+\n",
      "| pers.myth |       1 |\n",
      "+-----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_dev_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67c3f97c",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8da40406",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_train_de_docs = parse_tsv(file_path=ajmc_train_de_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "af8ae6f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/ajmc/de/HIPE-2022-v2.1-ajmc-train-de.tsv \n",
      "Number of documents: 76 \n",
      "Number of entities: {'coarse_lit': 1738, 'fine_lit': 1738, 'nested': 11} \n",
      "Number of tokens: 22695 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+--------+---------+\n",
      "|        |   count |\n",
      "+========+=========+\n",
      "| date   |       2 |\n",
      "+--------+---------+\n",
      "| loc    |      31 |\n",
      "+--------+---------+\n",
      "| object |       6 |\n",
      "+--------+---------+\n",
      "| pers   |     620 |\n",
      "+--------+---------+\n",
      "| scope  |     758 |\n",
      "+--------+---------+\n",
      "| work   |     321 |\n",
      "+--------+---------+\n",
      "fine_lit\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| date           |       2 |\n",
      "+----------------+---------+\n",
      "| loc            |      31 |\n",
      "+----------------+---------+\n",
      "| object.manuscr |       6 |\n",
      "+----------------+---------+\n",
      "| pers.author    |     354 |\n",
      "+----------------+---------+\n",
      "| pers.editor    |      16 |\n",
      "+----------------+---------+\n",
      "| pers.myth      |     234 |\n",
      "+----------------+---------+\n",
      "| pers.other     |      16 |\n",
      "+----------------+---------+\n",
      "| scope          |     758 |\n",
      "+----------------+---------+\n",
      "| work.fragm     |       2 |\n",
      "+----------------+---------+\n",
      "| work.primlit   |     312 |\n",
      "+----------------+---------+\n",
      "| work.seclit    |       7 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| loc          |       3 |\n",
      "+--------------+---------+\n",
      "| pers.author  |       2 |\n",
      "+--------------+---------+\n",
      "| pers.editor  |       1 |\n",
      "+--------------+---------+\n",
      "| scope        |       3 |\n",
      "+--------------+---------+\n",
      "| work.primlit |       1 |\n",
      "+--------------+---------+\n",
      "| work.seclit  |       1 |\n",
      "+--------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_train_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f498af21",
   "metadata": {},
   "source": [
    "### ajmc FR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e2eb3ec",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cc683514",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_dev_fr_docs = parse_tsv(file_path=ajmc_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0644243e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/ajmc/fr/HIPE-2022-v2.1-ajmc-dev-fr.tsv \n",
      "Number of documents: 17 \n",
      "Number of entities: {'coarse_lit': 391, 'fine_lit': 391} \n",
      "Number of tokens: 5425 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-------+---------+\n",
      "|       |   count |\n",
      "+=======+=========+\n",
      "| pers  |     123 |\n",
      "+-------+---------+\n",
      "| scope |     169 |\n",
      "+-------+---------+\n",
      "| work  |      99 |\n",
      "+-------+---------+\n",
      "fine_lit\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| pers.author  |      52 |\n",
      "+--------------+---------+\n",
      "| pers.editor  |      13 |\n",
      "+--------------+---------+\n",
      "| pers.myth    |      23 |\n",
      "+--------------+---------+\n",
      "| pers.other   |      35 |\n",
      "+--------------+---------+\n",
      "| scope        |     169 |\n",
      "+--------------+---------+\n",
      "| work.primlit |      99 |\n",
      "+--------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2020856",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "de15545d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "ajmc_train_fr_docs = parse_tsv(file_path=ajmc_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "786a76a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/ajmc/fr/HIPE-2022-v2.1-ajmc-train-fr.tsv \n",
      "Number of documents: 72 \n",
      "Number of entities: {'coarse_lit': 1621, 'fine_lit': 1621, 'nested': 9} \n",
      "Number of tokens: 24669 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+--------+---------+\n",
      "|        |   count |\n",
      "+========+=========+\n",
      "| date   |       2 |\n",
      "+--------+---------+\n",
      "| loc    |      15 |\n",
      "+--------+---------+\n",
      "| object |      10 |\n",
      "+--------+---------+\n",
      "| pers   |     577 |\n",
      "+--------+---------+\n",
      "| scope  |     639 |\n",
      "+--------+---------+\n",
      "| work   |     378 |\n",
      "+--------+---------+\n",
      "fine_lit\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| date           |       2 |\n",
      "+----------------+---------+\n",
      "| loc            |      15 |\n",
      "+----------------+---------+\n",
      "| object.manuscr |      10 |\n",
      "+----------------+---------+\n",
      "| pers.author    |     294 |\n",
      "+----------------+---------+\n",
      "| pers.editor    |      57 |\n",
      "+----------------+---------+\n",
      "| pers.myth      |      83 |\n",
      "+----------------+---------+\n",
      "| pers.other     |     143 |\n",
      "+----------------+---------+\n",
      "| scope          |     639 |\n",
      "+----------------+---------+\n",
      "| work.fragm     |       4 |\n",
      "+----------------+---------+\n",
      "| work.primlit   |     368 |\n",
      "+----------------+---------+\n",
      "| work.seclit    |       6 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| pers.author  |       1 |\n",
      "+--------------+---------+\n",
      "| pers.editor  |       1 |\n",
      "+--------------+---------+\n",
      "| pers.other   |       1 |\n",
      "+--------------+---------+\n",
      "| scope        |       4 |\n",
      "+--------------+---------+\n",
      "| work.primlit |       2 |\n",
      "+--------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=ajmc_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d2a85c1",
   "metadata": {},
   "source": [
    "## hipe2020\n",
    "\n",
    "See the [README file](./documentation/README-hipe2020.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbd8cb25",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f992992b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# EN\n",
    "hipe2020_dev_en_path = os.path.join(HIPE2022_data_path, f\"hipe2020/en/HIPE-2022-{RELEASE_VERSION}-hipe2020-dev-en.tsv\")\n",
    "\n",
    "# DE\n",
    "hipe2020_train_de_path = os.path.join(HIPE2022_data_path, f\"hipe2020/de/HIPE-2022-{RELEASE_VERSION}-hipe2020-train-de.tsv\")\n",
    "hipe2020_dev_de_path = os.path.join(HIPE2022_data_path, f\"hipe2020/de/HIPE-2022-{RELEASE_VERSION}-hipe2020-dev-de.tsv\")\n",
    "\n",
    "# FR\n",
    "hipe2020_dev_fr_path = os.path.join(HIPE2022_data_path, f\"hipe2020/fr/HIPE-2022-{RELEASE_VERSION}-hipe2020-dev-fr.tsv\")\n",
    "hipe2020_train_fr_path = os.path.join(HIPE2022_data_path, f\"hipe2020/fr/HIPE-2022-{RELEASE_VERSION}-hipe2020-train-fr.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d13fb06",
   "metadata": {},
   "source": [
    "### hipe2020 EN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6037598d",
   "metadata": {},
   "source": [
    "#### Dev\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5980486f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_dev_en_docs = parse_tsv(file_path=hipe2020_dev_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e1ebf584",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/hipe2020/en/HIPE-2022-v2.1-hipe2020-dev-en.tsv \n",
      "Number of documents: 80 \n",
      "Number of entities: {'coarse_lit': 966, 'coarse_meto': 15} \n",
      "Number of tokens: 29063 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |     384 |\n",
      "+------+---------+\n",
      "| org  |     118 |\n",
      "+------+---------+\n",
      "| pers |     402 |\n",
      "+------+---------+\n",
      "| prod |      33 |\n",
      "+------+---------+\n",
      "| time |      29 |\n",
      "+------+---------+\n",
      "coarse_meto\n",
      "+-----+---------+\n",
      "|     |   count |\n",
      "+=====+=========+\n",
      "| loc |       6 |\n",
      "+-----+---------+\n",
      "| org |       9 |\n",
      "+-----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_dev_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "374f4ccb",
   "metadata": {},
   "source": [
    "### hipe2020 DE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5d064d1",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ee918688",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_dev_de_docs = parse_tsv(file_path=hipe2020_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ca1b34aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/hipe2020/de/HIPE-2022-v2.1-hipe2020-dev-de.tsv \n",
      "Number of documents: 33 \n",
      "Number of entities: {'coarse_lit': 1242, 'coarse_meto': 91, 'fine_lit': 1242, 'fine_meto': 91, 'fine_comp': 468, 'nested': 67} \n",
      "Number of tokens: 32671 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |     588 |\n",
      "+------+---------+\n",
      "| org  |     164 |\n",
      "+------+---------+\n",
      "| pers |     372 |\n",
      "+------+---------+\n",
      "| prod |      49 |\n",
      "+------+---------+\n",
      "| time |      69 |\n",
      "+------+---------+\n",
      "coarse_meto\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |      12 |\n",
      "+------+---------+\n",
      "| org  |      78 |\n",
      "+------+---------+\n",
      "| prod |       1 |\n",
      "+------+---------+\n",
      "fine_lit\n",
      "+---------------------+---------+\n",
      "|                     |   count |\n",
      "+=====================+=========+\n",
      "| loc.adm.nat         |     135 |\n",
      "+---------------------+---------+\n",
      "| loc.adm.reg         |      53 |\n",
      "+---------------------+---------+\n",
      "| loc.adm.sup         |       8 |\n",
      "+---------------------+---------+\n",
      "| loc.adm.town        |     339 |\n",
      "+---------------------+---------+\n",
      "| loc.fac             |      15 |\n",
      "+---------------------+---------+\n",
      "| loc.oro             |      24 |\n",
      "+---------------------+---------+\n",
      "| loc.phys.geo        |       7 |\n",
      "+---------------------+---------+\n",
      "| loc.phys.hydro      |       7 |\n",
      "+---------------------+---------+\n",
      "| org.adm             |      53 |\n",
      "+---------------------+---------+\n",
      "| org.ent             |     100 |\n",
      "+---------------------+---------+\n",
      "| org.ent.pressagency |      11 |\n",
      "+---------------------+---------+\n",
      "| pers.coll           |       1 |\n",
      "+---------------------+---------+\n",
      "| pers.ind            |     371 |\n",
      "+---------------------+---------+\n",
      "| prod.doctr          |       3 |\n",
      "+---------------------+---------+\n",
      "| prod.media          |      46 |\n",
      "+---------------------+---------+\n",
      "| time.date.abs       |      69 |\n",
      "+---------------------+---------+\n",
      "fine_meto\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| loc.adm.reg  |       1 |\n",
      "+--------------+---------+\n",
      "| loc.adm.town |       1 |\n",
      "+--------------+---------+\n",
      "| loc.fac      |      10 |\n",
      "+--------------+---------+\n",
      "| org.adm      |      75 |\n",
      "+--------------+---------+\n",
      "| org.ent      |       3 |\n",
      "+--------------+---------+\n",
      "| prod.media   |       1 |\n",
      "+--------------+---------+\n",
      "fine_comp\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| comp.demonym   |      16 |\n",
      "+----------------+---------+\n",
      "| comp.function  |      83 |\n",
      "+----------------+---------+\n",
      "| comp.name      |     225 |\n",
      "+----------------+---------+\n",
      "| comp.qualifier |       5 |\n",
      "+----------------+---------+\n",
      "| comp.title     |     139 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc.adm.nat    |      10 |\n",
      "+----------------+---------+\n",
      "| loc.adm.reg    |       5 |\n",
      "+----------------+---------+\n",
      "| loc.adm.town   |      26 |\n",
      "+----------------+---------+\n",
      "| loc.phys.geo   |       2 |\n",
      "+----------------+---------+\n",
      "| loc.phys.hydro |       2 |\n",
      "+----------------+---------+\n",
      "| org.adm        |       3 |\n",
      "+----------------+---------+\n",
      "| org.ent        |       7 |\n",
      "+----------------+---------+\n",
      "| pers.ind       |       9 |\n",
      "+----------------+---------+\n",
      "| time.date.abs  |       3 |\n",
      "+----------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_dev_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9066d1a",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3f07e60",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_train_de_docs = parse_tsv(file_path=hipe2020_train_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "681ff82c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/hipe2020/de/HIPE-2022-v2.1-hipe2020-train-de.tsv \n",
      "Number of documents: 103 \n",
      "Number of entities: {'coarse_lit': 3494, 'coarse_meto': 325, 'fine_lit': 3494, 'fine_meto': 325, 'fine_comp': 1436, 'nested': 158} \n",
      "Number of tokens: 86445 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |    1740 |\n",
      "+------+---------+\n",
      "| org  |     358 |\n",
      "+------+---------+\n",
      "| pers |    1166 |\n",
      "+------+---------+\n",
      "| prod |     112 |\n",
      "+------+---------+\n",
      "| time |     118 |\n",
      "+------+---------+\n",
      "coarse_meto\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |      17 |\n",
      "+------+---------+\n",
      "| org  |     306 |\n",
      "+------+---------+\n",
      "| pers |       2 |\n",
      "+------+---------+\n",
      "fine_lit\n",
      "+---------------------+---------+\n",
      "|                     |   count |\n",
      "+=====================+=========+\n",
      "| loc.add.phys        |       2 |\n",
      "+---------------------+---------+\n",
      "| loc.adm.nat         |     563 |\n",
      "+---------------------+---------+\n",
      "| loc.adm.reg         |     199 |\n",
      "+---------------------+---------+\n",
      "| loc.adm.sup         |      42 |\n",
      "+---------------------+---------+\n",
      "| loc.adm.town        |     687 |\n",
      "+---------------------+---------+\n",
      "| loc.fac             |      46 |\n",
      "+---------------------+---------+\n",
      "| loc.oro             |      52 |\n",
      "+---------------------+---------+\n",
      "| loc.phys.astro      |       2 |\n",
      "+---------------------+---------+\n",
      "| loc.phys.geo        |      58 |\n",
      "+---------------------+---------+\n",
      "| loc.phys.hydro      |      88 |\n",
      "+---------------------+---------+\n",
      "| loc.unk             |       1 |\n",
      "+---------------------+---------+\n",
      "| org.adm             |     136 |\n",
      "+---------------------+---------+\n",
      "| org.ent             |     196 |\n",
      "+---------------------+---------+\n",
      "| org.ent.pressagency |      26 |\n",
      "+---------------------+---------+\n",
      "| pers.coll           |      14 |\n",
      "+---------------------+---------+\n",
      "| pers.ind            |    1152 |\n",
      "+---------------------+---------+\n",
      "| prod.doctr          |      19 |\n",
      "+---------------------+---------+\n",
      "| prod.media          |      93 |\n",
      "+---------------------+---------+\n",
      "| time.date.abs       |     118 |\n",
      "+---------------------+---------+\n",
      "fine_meto\n",
      "+-------------+---------+\n",
      "|             |   count |\n",
      "+=============+=========+\n",
      "| loc.adm.reg |      17 |\n",
      "+-------------+---------+\n",
      "| org.adm     |     294 |\n",
      "+-------------+---------+\n",
      "| org.ent     |      12 |\n",
      "+-------------+---------+\n",
      "| pers.coll   |       1 |\n",
      "+-------------+---------+\n",
      "| pers.ind    |       1 |\n",
      "+-------------+---------+\n",
      "fine_comp\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| comp.demonym   |      30 |\n",
      "+----------------+---------+\n",
      "| comp.function  |     280 |\n",
      "+----------------+---------+\n",
      "| comp.name      |     703 |\n",
      "+----------------+---------+\n",
      "| comp.qualifier |      35 |\n",
      "+----------------+---------+\n",
      "| comp.title     |     388 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc.adm.nat    |      13 |\n",
      "+----------------+---------+\n",
      "| loc.adm.reg    |       3 |\n",
      "+----------------+---------+\n",
      "| loc.adm.sup    |       4 |\n",
      "+----------------+---------+\n",
      "| loc.adm.town   |      57 |\n",
      "+----------------+---------+\n",
      "| loc.fac        |       3 |\n",
      "+----------------+---------+\n",
      "| loc.oro        |       1 |\n",
      "+----------------+---------+\n",
      "| loc.phys.geo   |       2 |\n",
      "+----------------+---------+\n",
      "| loc.phys.hydro |      10 |\n",
      "+----------------+---------+\n",
      "| org.adm        |       9 |\n",
      "+----------------+---------+\n",
      "| org.ent        |      35 |\n",
      "+----------------+---------+\n",
      "| pers.coll      |       1 |\n",
      "+----------------+---------+\n",
      "| pers.ind       |      16 |\n",
      "+----------------+---------+\n",
      "| prod.media     |       1 |\n",
      "+----------------+---------+\n",
      "| time.date.abs  |       3 |\n",
      "+----------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_train_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d98272e",
   "metadata": {},
   "source": [
    "### hipe2020 FR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "070880bf",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "eb508bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_dev_fr_docs = parse_tsv(file_path=hipe2020_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1eac079a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/hipe2020/fr/HIPE-2022-v2.1-hipe2020-dev-fr.tsv \n",
      "Number of documents: 43 \n",
      "Number of entities: {'coarse_lit': 1729, 'coarse_meto': 108, 'fine_lit': 1729, 'fine_meto': 108, 'fine_comp': 724, 'nested': 91} \n",
      "Number of tokens: 37953 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |     774 |\n",
      "+------+---------+\n",
      "| org  |     159 |\n",
      "+------+---------+\n",
      "| pers |     679 |\n",
      "+------+---------+\n",
      "| prod |      49 |\n",
      "+------+---------+\n",
      "| time |      68 |\n",
      "+------+---------+\n",
      "coarse_meto\n",
      "+-----+---------+\n",
      "|     |   count |\n",
      "+=====+=========+\n",
      "| loc |       3 |\n",
      "+-----+---------+\n",
      "| org |     105 |\n",
      "+-----+---------+\n",
      "fine_lit\n",
      "+------------------------+---------+\n",
      "|                        |   count |\n",
      "+========================+=========+\n",
      "| loc.add.phys           |       1 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.nat            |     258 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.reg            |      73 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.sup            |      27 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.town           |     352 |\n",
      "+------------------------+---------+\n",
      "| loc.fac                |      11 |\n",
      "+------------------------+---------+\n",
      "| loc.oro                |      10 |\n",
      "+------------------------+---------+\n",
      "| loc.phys.geo           |      21 |\n",
      "+------------------------+---------+\n",
      "| loc.phys.hydro         |      18 |\n",
      "+------------------------+---------+\n",
      "| loc.unk                |       3 |\n",
      "+------------------------+---------+\n",
      "| org.adm                |      54 |\n",
      "+------------------------+---------+\n",
      "| org.ent                |      81 |\n",
      "+------------------------+---------+\n",
      "| org.ent.pressagency    |      24 |\n",
      "+------------------------+---------+\n",
      "| pers.coll              |      10 |\n",
      "+------------------------+---------+\n",
      "| pers.ind               |     655 |\n",
      "+------------------------+---------+\n",
      "| pers.ind.articleauthor |      14 |\n",
      "+------------------------+---------+\n",
      "| prod.doctr             |       2 |\n",
      "+------------------------+---------+\n",
      "| prod.media             |      47 |\n",
      "+------------------------+---------+\n",
      "| time.date.abs          |      68 |\n",
      "+------------------------+---------+\n",
      "fine_meto\n",
      "+---------+---------+\n",
      "|         |   count |\n",
      "+=========+=========+\n",
      "| loc.fac |       2 |\n",
      "+---------+---------+\n",
      "| loc.oro |       1 |\n",
      "+---------+---------+\n",
      "| org.adm |      79 |\n",
      "+---------+---------+\n",
      "| org.ent |      26 |\n",
      "+---------+---------+\n",
      "fine_comp\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| comp.demonym   |       7 |\n",
      "+----------------+---------+\n",
      "| comp.function  |     117 |\n",
      "+----------------+---------+\n",
      "| comp.name      |     343 |\n",
      "+----------------+---------+\n",
      "| comp.qualifier |       9 |\n",
      "+----------------+---------+\n",
      "| comp.title     |     248 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc.adm.nat    |      15 |\n",
      "+----------------+---------+\n",
      "| loc.adm.reg    |       5 |\n",
      "+----------------+---------+\n",
      "| loc.adm.town   |      37 |\n",
      "+----------------+---------+\n",
      "| loc.fac        |       1 |\n",
      "+----------------+---------+\n",
      "| loc.phys.geo   |       2 |\n",
      "+----------------+---------+\n",
      "| loc.phys.hydro |       5 |\n",
      "+----------------+---------+\n",
      "| org.adm        |       3 |\n",
      "+----------------+---------+\n",
      "| org.ent        |       8 |\n",
      "+----------------+---------+\n",
      "| pers.ind       |      14 |\n",
      "+----------------+---------+\n",
      "| prod.media     |       1 |\n",
      "+----------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8dbd691",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e21ac6f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "hipe2020_train_fr_docs = parse_tsv(file_path=hipe2020_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5824fc8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/hipe2020/fr/HIPE-2022-v2.1-hipe2020-train-fr.tsv \n",
      "Number of documents: 158 \n",
      "Number of entities: {'coarse_lit': 6926, 'coarse_meto': 450, 'fine_lit': 6926, 'fine_meto': 450, 'fine_comp': 3050, 'nested': 473} \n",
      "Number of tokens: 166220 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |    3089 |\n",
      "+------+---------+\n",
      "| org  |     836 |\n",
      "+------+---------+\n",
      "| pers |    2525 |\n",
      "+------+---------+\n",
      "| prod |     200 |\n",
      "+------+---------+\n",
      "| time |     276 |\n",
      "+------+---------+\n",
      "coarse_meto\n",
      "+-----+---------+\n",
      "|     |   count |\n",
      "+=====+=========+\n",
      "| loc |       7 |\n",
      "+-----+---------+\n",
      "| org |     443 |\n",
      "+-----+---------+\n",
      "fine_lit\n",
      "+------------------------+---------+\n",
      "|                        |   count |\n",
      "+========================+=========+\n",
      "| loc.add.elec           |       1 |\n",
      "+------------------------+---------+\n",
      "| loc.add.phys           |       3 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.nat            |     648 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.reg            |     376 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.sup            |      63 |\n",
      "+------------------------+---------+\n",
      "| loc.adm.town           |    1688 |\n",
      "+------------------------+---------+\n",
      "| loc.fac                |      59 |\n",
      "+------------------------+---------+\n",
      "| loc.oro                |      91 |\n",
      "+------------------------+---------+\n",
      "| loc.phys.geo           |      84 |\n",
      "+------------------------+---------+\n",
      "| loc.phys.hydro         |      73 |\n",
      "+------------------------+---------+\n",
      "| loc.unk                |       3 |\n",
      "+------------------------+---------+\n",
      "| org.adm                |     208 |\n",
      "+------------------------+---------+\n",
      "| org.ent                |     571 |\n",
      "+------------------------+---------+\n",
      "| org.ent.pressagency    |      57 |\n",
      "+------------------------+---------+\n",
      "| pers.coll              |      20 |\n",
      "+------------------------+---------+\n",
      "| pers.ind               |    2407 |\n",
      "+------------------------+---------+\n",
      "| pers.ind.articleauthor |      98 |\n",
      "+------------------------+---------+\n",
      "| prod.doctr             |      48 |\n",
      "+------------------------+---------+\n",
      "| prod.media             |     152 |\n",
      "+------------------------+---------+\n",
      "| time.date.abs          |     276 |\n",
      "+------------------------+---------+\n",
      "fine_meto\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| loc.adm.town |       2 |\n",
      "+--------------+---------+\n",
      "| loc.fac      |       1 |\n",
      "+--------------+---------+\n",
      "| loc.oro      |       4 |\n",
      "+--------------+---------+\n",
      "| org.adm      |     306 |\n",
      "+--------------+---------+\n",
      "| org.ent      |     137 |\n",
      "+--------------+---------+\n",
      "fine_comp\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| comp.demonym   |      22 |\n",
      "+----------------+---------+\n",
      "| comp.function  |     535 |\n",
      "+----------------+---------+\n",
      "| comp.name      |    1476 |\n",
      "+----------------+---------+\n",
      "| comp.qualifier |      24 |\n",
      "+----------------+---------+\n",
      "| comp.title     |     993 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc.adm.nat    |      76 |\n",
      "+----------------+---------+\n",
      "| loc.adm.reg    |      36 |\n",
      "+----------------+---------+\n",
      "| loc.adm.sup    |       5 |\n",
      "+----------------+---------+\n",
      "| loc.adm.town   |     251 |\n",
      "+----------------+---------+\n",
      "| loc.fac        |       2 |\n",
      "+----------------+---------+\n",
      "| loc.oro        |       4 |\n",
      "+----------------+---------+\n",
      "| loc.phys.geo   |       9 |\n",
      "+----------------+---------+\n",
      "| loc.phys.hydro |      12 |\n",
      "+----------------+---------+\n",
      "| org.adm        |      11 |\n",
      "+----------------+---------+\n",
      "| org.ent        |      53 |\n",
      "+----------------+---------+\n",
      "| pers.coll      |       3 |\n",
      "+----------------+---------+\n",
      "| pers.ind       |      11 |\n",
      "+----------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=hipe2020_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0b06c25",
   "metadata": {},
   "source": [
    "## letemps\n",
    "\n",
    "See the [README file](./documentation/README-letemps.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dcea5cf",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "61dec1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "letemps_dev_fr_path = os.path.join(HIPE2022_data_path, f\"letemps/fr/HIPE-2022-{RELEASE_VERSION}-letemps-dev-fr.tsv\")\n",
    "letemps_train_fr_path = os.path.join(HIPE2022_data_path, f\"letemps/fr/HIPE-2022-{RELEASE_VERSION}-letemps-train-fr.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b0537e0",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ceedad6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "letemps_dev_fr_docs = parse_tsv(file_path=letemps_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "cae0932b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/letemps/fr/HIPE-2022-v2.1-letemps-dev-fr.tsv \n",
      "Number of documents: 51 \n",
      "Number of entities: {'coarse_lit': 869, 'fine_lit': 869, 'nested': 12} \n",
      "Number of tokens: 38649 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |     516 |\n",
      "+------+---------+\n",
      "| org  |      41 |\n",
      "+------+---------+\n",
      "| pers |     312 |\n",
      "+------+---------+\n",
      "fine_lit\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc            |      16 |\n",
      "+----------------+---------+\n",
      "| loc.add        |       2 |\n",
      "+----------------+---------+\n",
      "| loc.add.phys   |      21 |\n",
      "+----------------+---------+\n",
      "| loc.adm        |      20 |\n",
      "+----------------+---------+\n",
      "| loc.adm.nat    |     104 |\n",
      "+----------------+---------+\n",
      "| loc.adm.reg    |      45 |\n",
      "+----------------+---------+\n",
      "| loc.adm.town   |     270 |\n",
      "+----------------+---------+\n",
      "| loc.admin.sup  |       1 |\n",
      "+----------------+---------+\n",
      "| loc.phys.geo   |      25 |\n",
      "+----------------+---------+\n",
      "| loc.phys.hydro |      12 |\n",
      "+----------------+---------+\n",
      "| org.adm        |      12 |\n",
      "+----------------+---------+\n",
      "| org.ent        |      29 |\n",
      "+----------------+---------+\n",
      "| pers           |       4 |\n",
      "+----------------+---------+\n",
      "| pers.coll      |       7 |\n",
      "+----------------+---------+\n",
      "| pers.ind       |     301 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+--------------+---------+\n",
      "|              |   count |\n",
      "+==============+=========+\n",
      "| loc.adm.nat  |      10 |\n",
      "+--------------+---------+\n",
      "| loc.adm.town |       1 |\n",
      "+--------------+---------+\n",
      "| pers.ind     |       1 |\n",
      "+--------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=letemps_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64a305b2",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "cc6d3eee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "letemps_train_fr_docs = parse_tsv(file_path=letemps_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "867d6f04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/letemps/fr/HIPE-2022-v2.1-letemps-train-fr.tsv \n",
      "Number of documents: 414 \n",
      "Number of entities: {'coarse_lit': 9159, 'fine_lit': 9159, 'nested': 69} \n",
      "Number of tokens: 379487 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+------+---------+\n",
      "|      |   count |\n",
      "+======+=========+\n",
      "| loc  |    5260 |\n",
      "+------+---------+\n",
      "| org  |     472 |\n",
      "+------+---------+\n",
      "| pers |    3427 |\n",
      "+------+---------+\n",
      "fine_lit\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc            |     262 |\n",
      "+----------------+---------+\n",
      "| loc.add        |       3 |\n",
      "+----------------+---------+\n",
      "| loc.add.phys   |     169 |\n",
      "+----------------+---------+\n",
      "| loc.adm        |     111 |\n",
      "+----------------+---------+\n",
      "| loc.adm.nat    |    1097 |\n",
      "+----------------+---------+\n",
      "| loc.adm.reg    |     477 |\n",
      "+----------------+---------+\n",
      "| loc.adm.town   |    2713 |\n",
      "+----------------+---------+\n",
      "| loc.admin.sup  |      33 |\n",
      "+----------------+---------+\n",
      "| loc.oro        |      13 |\n",
      "+----------------+---------+\n",
      "| loc.phys       |      10 |\n",
      "+----------------+---------+\n",
      "| loc.phys.astro |      10 |\n",
      "+----------------+---------+\n",
      "| loc.phys.geo   |     268 |\n",
      "+----------------+---------+\n",
      "| loc.phys.hydro |      94 |\n",
      "+----------------+---------+\n",
      "| org.adm        |     171 |\n",
      "+----------------+---------+\n",
      "| org.ent        |     301 |\n",
      "+----------------+---------+\n",
      "| pers           |      45 |\n",
      "+----------------+---------+\n",
      "| pers.coll      |     149 |\n",
      "+----------------+---------+\n",
      "| pers.ind       |    3233 |\n",
      "+----------------+---------+\n",
      "nested\n",
      "+----------------+---------+\n",
      "|                |   count |\n",
      "+================+=========+\n",
      "| loc.adm.nat    |       9 |\n",
      "+----------------+---------+\n",
      "| loc.adm.reg    |       5 |\n",
      "+----------------+---------+\n",
      "| loc.adm.town   |      43 |\n",
      "+----------------+---------+\n",
      "| loc.phys.geo   |       5 |\n",
      "+----------------+---------+\n",
      "| loc.phys.hydro |       1 |\n",
      "+----------------+---------+\n",
      "| org.ent        |       1 |\n",
      "+----------------+---------+\n",
      "| pers.ind       |       5 |\n",
      "+----------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=letemps_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed696674",
   "metadata": {},
   "source": [
    "## topRes19th\n",
    "\n",
    "See the [README file](./documentation/README-topres19th.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01e062c6",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "51fa12eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "topRes19th_dev_en_path = os.path.join(HIPE2022_data_path, f\"topres19th/en/HIPE-2022-{RELEASE_VERSION}-topres19th-dev-en.tsv\")\n",
    "topRes19th_train_en_path = os.path.join(HIPE2022_data_path, f\"topres19th/en/HIPE-2022-{RELEASE_VERSION}-topres19th-train-en.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "563e29f2",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2457dbf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "topRes19th_dev_en_docs = parse_tsv(file_path=topRes19th_dev_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e1fa7ece",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/topres19th/en/HIPE-2022-v2.1-topres19th-dev-en.tsv \n",
      "Number of documents: 34 \n",
      "Number of entities: {'coarse_lit': 237} \n",
      "Number of tokens: 11917 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+----------+---------+\n",
      "|          |   count |\n",
      "+==========+=========+\n",
      "| BUILDING |      19 |\n",
      "+----------+---------+\n",
      "| LOC      |     201 |\n",
      "+----------+---------+\n",
      "| STREET   |      17 |\n",
      "+----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=topRes19th_dev_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4784aff3",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "8094673a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "topRes19th_train_en_docs = parse_tsv(file_path=topRes19th_train_en_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "fc46b0f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/topres19th/en/HIPE-2022-v2.1-topres19th-train-en.tsv \n",
      "Number of documents: 309 \n",
      "Number of entities: {'coarse_lit': 3182} \n",
      "Number of tokens: 123983 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+----------+---------+\n",
      "|          |   count |\n",
      "+==========+=========+\n",
      "| BUILDING |     359 |\n",
      "+----------+---------+\n",
      "| LOC      |    2580 |\n",
      "+----------+---------+\n",
      "| STREET   |     243 |\n",
      "+----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=topRes19th_train_en_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9abfd336",
   "metadata": {},
   "source": [
    "## newseye\n",
    "\n",
    "See the [README file](./documentation/README-newseye.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86321d90",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "24c4cf39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FR\n",
    "newseye_dev_fr_path = os.path.join(HIPE2022_data_path, f\"newseye/fr/HIPE-2022-{RELEASE_VERSION}-newseye-dev-fr.tsv\")\n",
    "newseye_train_fr_path = os.path.join(HIPE2022_data_path, f\"newseye/fr/HIPE-2022-{RELEASE_VERSION}-newseye-train-fr.tsv\")\n",
    "\n",
    "# DE\n",
    "newseye_dev_de_path = os.path.join(HIPE2022_data_path, f\"newseye/de/HIPE-2022-{RELEASE_VERSION}-newseye-dev-de.tsv\")\n",
    "newseye_train_de_path = os.path.join(HIPE2022_data_path, f\"newseye/de/HIPE-2022-{RELEASE_VERSION}-newseye-train-de.tsv\")\n",
    "\n",
    "# FI\n",
    "newseye_dev_fi_path = os.path.join(HIPE2022_data_path, f\"newseye/fi/HIPE-2022-{RELEASE_VERSION}-newseye-dev-fi.tsv\")\n",
    "newseye_train_fi_path = os.path.join(HIPE2022_data_path, f\"newseye/fi/HIPE-2022-{RELEASE_VERSION}-newseye-train-fi.tsv\")\n",
    "\n",
    "# SV\n",
    "newseye_dev_sv_path = os.path.join(HIPE2022_data_path, f\"newseye/sv/HIPE-2022-{RELEASE_VERSION}-newseye-dev-sv.tsv\")\n",
    "newseye_train_sv_path = os.path.join(HIPE2022_data_path, f\"newseye/sv/HIPE-2022-{RELEASE_VERSION}-newseye-train-sv.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "289ca720",
   "metadata": {},
   "source": [
    "### newseye FR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acd2b525",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "35e7be47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_fr_docs = parse_tsv(file_path=newseye_dev_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c0093f53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/fr/HIPE-2022-v2.1-newseye-dev-fr.tsv \n",
      "Number of documents: 35 \n",
      "Number of entities: {'coarse_lit': 752, 'fine_lit': 3, 'nested': 32} \n",
      "Number of tokens: 21727 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |      11 |\n",
      "+-----------+---------+\n",
      "| LOC       |     335 |\n",
      "+-----------+---------+\n",
      "| ORG       |     113 |\n",
      "+-----------+---------+\n",
      "| PER       |     293 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |       3 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |       1 |\n",
      "+-----------+---------+\n",
      "| LOC       |      18 |\n",
      "+-----------+---------+\n",
      "| ORG       |       7 |\n",
      "+-----------+---------+\n",
      "| PER       |       6 |\n",
      "+-----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f527b0d3",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "541b4c65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./data/v2.1/newseye/fr/HIPE-2022-v2.1-newseye-train-fr.tsv'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newseye_train_fr_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7b4cd97a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_fr_docs = parse_tsv(file_path=newseye_train_fr_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "4cdc73ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/fr/HIPE-2022-v2.1-newseye-train-fr.tsv \n",
      "Number of documents: 35 \n",
      "Number of entities: {'coarse_lit': 10423, 'fine_lit': 99, 'nested': 522} \n",
      "Number of tokens: 255165 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |     200 |\n",
      "+-----------+---------+\n",
      "| LOC       |    4055 |\n",
      "+-----------+---------+\n",
      "| ORG       |    1285 |\n",
      "+-----------+---------+\n",
      "| PER       |    4883 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |      99 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |       6 |\n",
      "+-----------+---------+\n",
      "| LOC       |     263 |\n",
      "+-----------+---------+\n",
      "| ORG       |     212 |\n",
      "+-----------+---------+\n",
      "| PER       |      41 |\n",
      "+-----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_fr_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47d69e44",
   "metadata": {},
   "source": [
    "### newseye DE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8398af96",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9b6ef18f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_de_docs = parse_tsv(file_path=newseye_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "3da94913",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/de/HIPE-2022-v2.1-newseye-dev-de.tsv \n",
      "Number of documents: 12 \n",
      "Number of entities: {'coarse_lit': 539, 'fine_lit': 5, 'nested': 29} \n",
      "Number of tokens: 40061 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |       4 |\n",
      "+-----------+---------+\n",
      "| LOC       |     263 |\n",
      "+-----------+---------+\n",
      "| ORG       |     123 |\n",
      "+-----------+---------+\n",
      "| PER       |     149 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |       5 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----+---------+\n",
      "|     |   count |\n",
      "+=====+=========+\n",
      "| LOC |      10 |\n",
      "+-----+---------+\n",
      "| ORG |      11 |\n",
      "+-----+---------+\n",
      "| PER |       8 |\n",
      "+-----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c73fbcb",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e0c65881",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_de_docs = parse_tsv(file_path=newseye_train_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "43791e9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/de/HIPE-2022-v2.1-newseye-train-de.tsv \n",
      "Number of documents: 7 \n",
      "Number of entities: {'coarse_lit': 11381, 'fine_lit': 21, 'nested': 1133} \n",
      "Number of tokens: 374332 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |      37 |\n",
      "+-----------+---------+\n",
      "| LOC       |    5135 |\n",
      "+-----------+---------+\n",
      "| ORG       |    3108 |\n",
      "+-----------+---------+\n",
      "| PER       |    3101 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |      21 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |       2 |\n",
      "+-----------+---------+\n",
      "| LOC       |     754 |\n",
      "+-----------+---------+\n",
      "| ORG       |     136 |\n",
      "+-----------+---------+\n",
      "| PER       |     241 |\n",
      "+-----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_de_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "872b6a68",
   "metadata": {},
   "source": [
    "### newseye FI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a10da0f",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "a6550817",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_fi_docs = parse_tsv(file_path=newseye_dev_fi_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2ef3872d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/fi/HIPE-2022-v2.1-newseye-dev-fi.tsv \n",
      "Number of documents: 24 \n",
      "Number of entities: {'coarse_lit': 223, 'fine_lit': 1, 'nested': 26} \n",
      "Number of tokens: 6350 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |      12 |\n",
      "+-----------+---------+\n",
      "| LOC       |      97 |\n",
      "+-----------+---------+\n",
      "| ORG       |      37 |\n",
      "+-----------+---------+\n",
      "| PER       |      77 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |       1 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----+---------+\n",
      "|     |   count |\n",
      "+=====+=========+\n",
      "| LOC |      22 |\n",
      "+-----+---------+\n",
      "| ORG |       2 |\n",
      "+-----+---------+\n",
      "| PER |       2 |\n",
      "+-----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_fi_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51e3615d",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d183a069",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_fi_docs = parse_tsv(file_path=newseye_train_fi_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "06e61ccb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/fi/HIPE-2022-v2.1-newseye-train-fi.tsv \n",
      "Number of documents: 24 \n",
      "Number of entities: {'coarse_lit': 2146, 'fine_lit': 15, 'nested': 225} \n",
      "Number of tokens: 48222 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |     126 |\n",
      "+-----------+---------+\n",
      "| LOC       |     979 |\n",
      "+-----------+---------+\n",
      "| ORG       |     259 |\n",
      "+-----------+---------+\n",
      "| PER       |     782 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |      15 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |       2 |\n",
      "+-----------+---------+\n",
      "| LOC       |     171 |\n",
      "+-----------+---------+\n",
      "| ORG       |      35 |\n",
      "+-----------+---------+\n",
      "| PER       |      17 |\n",
      "+-----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_fi_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74083291",
   "metadata": {},
   "source": [
    "### newseye SV"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b60fcdb9",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "266f8bee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_dev_sv_docs = parse_tsv(file_path=newseye_dev_sv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "42c4a49a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/sv/HIPE-2022-v2.1-newseye-dev-sv.tsv \n",
      "Number of documents: 21 \n",
      "Number of entities: {'coarse_lit': 266, 'fine_lit': 1, 'nested': 9} \n",
      "Number of tokens: 6906 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |      17 |\n",
      "+-----------+---------+\n",
      "| LOC       |     148 |\n",
      "+-----------+---------+\n",
      "| ORG       |      17 |\n",
      "+-----------+---------+\n",
      "| PER       |      84 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |       1 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----+---------+\n",
      "|     |   count |\n",
      "+=====+=========+\n",
      "| LOC |       7 |\n",
      "+-----+---------+\n",
      "| ORG |       1 |\n",
      "+-----+---------+\n",
      "| PER |       1 |\n",
      "+-----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_dev_sv_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92def368",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "cc962b4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "newseye_train_sv_docs = parse_tsv(file_path=newseye_train_sv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b6f6f2e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/newseye/sv/HIPE-2022-v2.1-newseye-train-sv.tsv \n",
      "Number of documents: 21 \n",
      "Number of entities: {'coarse_lit': 2141, 'fine_lit': 16, 'nested': 126} \n",
      "Number of tokens: 56306 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |     164 |\n",
      "+-----------+---------+\n",
      "| LOC       |     985 |\n",
      "+-----------+---------+\n",
      "| ORG       |     153 |\n",
      "+-----------+---------+\n",
      "| PER       |     839 |\n",
      "+-----------+---------+\n",
      "fine_lit\n",
      "+------------+---------+\n",
      "|            |   count |\n",
      "+============+=========+\n",
      "| PER.author |      16 |\n",
      "+------------+---------+\n",
      "nested\n",
      "+-----------+---------+\n",
      "|           |   count |\n",
      "+===========+=========+\n",
      "| HumanProd |       1 |\n",
      "+-----------+---------+\n",
      "| LOC       |     100 |\n",
      "+-----------+---------+\n",
      "| ORG       |      15 |\n",
      "+-----------+---------+\n",
      "| PER       |      10 |\n",
      "+-----------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=newseye_train_sv_docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "170e049d",
   "metadata": {},
   "source": [
    "## sonar\n",
    "\n",
    "See the [README file](./documentation/README-sonar.md) for detailed information about this dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a688da4",
   "metadata": {},
   "source": [
    "### File paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "263a7947",
   "metadata": {},
   "outputs": [],
   "source": [
    "sonar_dev_de_path = os.path.join(HIPE2022_data_path, f\"sonar/de/HIPE-2022-{RELEASE_VERSION}-sonar-dev-de.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65df2a7a",
   "metadata": {},
   "source": [
    "#### Dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "d5ac5fcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse the TSV into a list of `HipeDocument` objects\n",
    "sonar_dev_de_docs = parse_tsv(file_path=sonar_dev_de_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "fba9bf96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Path of the TSV file: ./data/v2.1/sonar/de/HIPE-2022-v2.1-sonar-dev-de.tsv \n",
      "Number of documents: 10 \n",
      "Number of entities: {'coarse_lit': 654} \n",
      "Number of tokens: 17564 \n",
      "Entity breakdown by type: coarse_lit\n",
      "+-----+---------+\n",
      "|     |   count |\n",
      "+=====+=========+\n",
      "| LOC |     300 |\n",
      "+-----+---------+\n",
      "| ORG |     138 |\n",
      "+-----+---------+\n",
      "| PER |     216 |\n",
      "+-----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print some basic stats for the TSV dataset \n",
    "print(describe_dataset(documents=sonar_dev_de_docs))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}