18_XX_CoNLLExperiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import good_organic_hp_params, default_params\n",
    "from data.conll import conll2003_dataset as dsl\n",
    "from misc.run_configuration import conll_params\n",
    "from misc.experimental_environment import Experiment\n",
    "import time\n",
    "import pprint\n",
    "from misc import utils\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "STATUS_FAIL = 'fail'\n",
    "STATUS_OK = 'ok'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "        data_root='./data/data/conll2003',\n",
    "        data_train='eng.train.txt',\n",
    "        data_validation='eng.testa.txt',\n",
    "        data_test='eng.testb.txt',\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        file_format='txt',\n",
    "        language='en'\n",
    "    )\n",
    "main_experiment_name = 'CoNLL2003-FinalExperiment'\n",
    "use_cuda = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{ 'att_d_k': 150,\n",
      "  'att_d_v': 150,\n",
      "  'clip_comments_to': 200,\n",
      "  'dropout_rate': 0.302424,\n",
      "  'embedding_type': 'fasttext',\n",
      "  'learning_rate_scheduler': { 'noam_learning_rate_factor': 0.5,\n",
      "                               'noam_learning_rate_warmup': 3500},\n",
      "  'model_size': 300,\n",
      "  'num_encoder_blocks': 2,\n",
      "  'num_heads': 2,\n",
      "  'optimizer': { 'adam_beta1': 0.9,\n",
      "                 'adam_beta2': 0.98,\n",
      "                 'adam_eps': 1e-08,\n",
      "                 'adam_weight_decay': 1e-06,\n",
      "                 'learning_rate': 7.2e-05},\n",
      "  'optimizer_type': <OptimizerType.Adam: 1>,\n",
      "  'pointwise_layer_size': 287,\n",
      "  'task': 'ner'}\n"
     ]
    }
   ],
   "source": [
    "baseline = conll_params\n",
    "print(pprint.pformat(baseline, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        'name': 'Baseline',\n",
    "        'description': 'Classification of the CoNLL-2003 NER task',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {}\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current commit: b'95a112a'\n"
     ]
    }
   ],
   "source": [
    "utils.get_current_git_commit()\n",
    "print('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Experiment CoNLL2003-FinalExperiment initialized\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\CoNLL2003-FinalExperiment\\20190430\\0\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\CoNLL2003-FinalExperiment\\20190430\\1\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: CoNLL2003-FinalExperiment\n",
      "Description: Classification of the CoNLL-2003 NER task\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 12, 'learning_rate_schedu[...]one} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                        ner                        |\n",
      "|          batch_size          |                         12                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                      7.2e-05                      |\n",
      "|  noam_learning_rate_warmup   |                        3500                       |\n",
      "|  noam_learning_rate_factor   |                        0.5                        |\n",
      "|          adam_beta1          |                        0.9                        |\n",
      "|          adam_beta2          |                        0.98                       |\n",
      "|           adam_eps           |                       1e-08                       |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                      0.302424                     |\n",
      "|     pointwise_layer_size     |                        287                        |\n",
      "|      last_layer_dropout      |                        0.2                        |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        200                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  14041  |\n",
      "|   validation  |   3250  |\n",
      "|      test     |   3453  |\n",
      "+---------------+---------+\n",
      "+---------------------------+\n",
      "|      Vocabulary Stats     |\n",
      "+-------------------+-------+\n",
      "|     Vocabulary    |  Size |\n",
      "+-------------------+-------+\n",
      "|      comments     | 26056 |\n",
      "| aspect_sentiments |   5   |\n",
      "+-------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                        aspect_sentiments                         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     ORG     |  10025  | 4.923362521547384  |  0.9507663747845262 |\n",
      "|      O      |  169578 |  83.2811939829389  | 0.16718806017061105 |\n",
      "|     MISC    |   4593  | 2.255661253014178  |  0.9774433874698583 |\n",
      "|     PER     |  11128  | 5.4650551760378345 |  0.9453494482396216 |\n",
      "|     LOC     |   8297  | 4.074727066461711  |  0.9592527293353829 |\n",
      "|     Sum     |  203621 |                    |         1.0         |\n",
      "| Head Weight |         |                    |         0.0         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 5.347002744674683\n",
      "pre_training - INFO - Classes: ['O', 'PER', 'ORG', 'LOC', 'MISC']\n",
      "pre_training - INFO - TransformerTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.302424)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(26056, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300,), (300,), (26056, 300)), parameters=8883000\n",
      "  (taggingLayer): SoftmaxOutputLayer(\n",
      "    (output_projection): Linear(in_features=300, out_features=5, bias=True)\n",
      "  ), weights=((5, 300), (5,)), parameters=1505\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 8.884.505\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 1171 Iterations per epoch with batch size of 12\n",
      "pre_training - INFO - Total iterations: 40985\n",
      "pre_training - INFO - Total number of samples: 491820\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b874558063f41d699eacdf4b9981a43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t14k\t0.07\t\t0.71\t\t0.835\t\t0.831\t\t2.07m - 2.1m / 0.0m\n",
      "2\t28k\t0.02\t\t0.48\t\t0.898\t\t0.896\t\t2.08m - 4.2m / 72.5m\n",
      "3\t42k\t0.02\t\t0.44\t\t0.911\t\t0.909\t\t2.16m - 6.4m / 72.9m\n",
      "4\t56k\t0.01\t\t0.45\t\t0.904\t\t0.903\t\t2.16m - 8.5m / 75.7m\n",
      "5\t70k\t0.01\t\t0.44\t\t0.935\t\t0.934\t\t2.13m - 10.7m / 75.6m\n",
      "6\t84k\t0.01\t\t0.42\t\t0.928\t\t0.926\t\t2.12m - 12.9m / 74.6m\n",
      "7\t98k\t0.01\t\t0.37\t\t0.936\t\t0.934\t\t2.10m - 15.0m / 74.2m\n",
      "8\t112k\t0.01\t\t0.41\t\t0.932\t\t0.930\t\t2.05m - 17.0m / 73.9m\n",
      "9\t126k\t0.01\t\t0.41\t\t0.944\t\t0.941\t\t2.15m - 19.2m / 72.4m\n",
      "10\t141k\t0.01\t\t0.50\t\t0.941\t\t0.939\t\t2.10m - 21.3m / 75.1m\n",
      "11\t155k\t0.01\t\t0.44\t\t0.922\t\t0.919\t\t2.11m - 23.4m / 73.9m\n",
      "12\t169k\t0.01\t\t0.40\t\t0.940\t\t0.938\t\t2.01m - 25.5m / 74.1m\n",
      "13\t183k\t0.01\t\t0.45\t\t0.934\t\t0.931\t\t2.08m - 27.6m / 71.7m\n",
      "14\t197k\t0.01\t\t0.42\t\t0.939\t\t0.935\t\t2.10m - 29.7m / 73.3m\n",
      "Training duration was 1782.5369415283203\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.9758064516129032\n",
      "VAL f1\t0.9437711029139099 - (0.9437711029139099)\n",
      "(macro) f1\t{'valid': 0.9908256880733946, 'test': 0.9758064516129032}\n",
      "VAL loss\t0.3681941920240732\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [0/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3681941920240732\n",
      " | \\     )|_\tf1: 0.9437711029139099\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\CoNLL2003-FinalExperiment\\20190430\\2\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: CoNLL2003-FinalExperiment\n",
      "Description: Classification of the CoNLL-2003 NER task\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 12, 'learning_rate_schedu[...]one} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                        ner                        |\n",
      "|          batch_size          |                         12                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                      7.2e-05                      |\n",
      "|  noam_learning_rate_warmup   |                        3500                       |\n",
      "|  noam_learning_rate_factor   |                        0.5                        |\n",
      "|          adam_beta1          |                        0.9                        |\n",
      "|          adam_beta2          |                        0.98                       |\n",
      "|           adam_eps           |                       1e-08                       |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                      0.302424                     |\n",
      "|     pointwise_layer_size     |                        287                        |\n",
      "|      last_layer_dropout      |                        0.2                        |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        200                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  14041  |\n",
      "|   validation  |   3250  |\n",
      "|      test     |   3453  |\n",
      "+---------------+---------+\n",
      "+---------------------------+\n",
      "|      Vocabulary Stats     |\n",
      "+-------------------+-------+\n",
      "|     Vocabulary    |  Size |\n",
      "+-------------------+-------+\n",
      "|      comments     | 26056 |\n",
      "| aspect_sentiments |   5   |\n",
      "+-------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                        aspect_sentiments                         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     ORG     |  10025  | 4.923362521547384  |  0.9507663747845262 |\n",
      "|      O      |  169578 |  83.2811939829389  | 0.16718806017061105 |\n",
      "|     MISC    |   4593  | 2.255661253014178  |  0.9774433874698583 |\n",
      "|     PER     |  11128  | 5.4650551760378345 |  0.9453494482396216 |\n",
      "|     LOC     |   8297  | 4.074727066461711  |  0.9592527293353829 |\n",
      "|     Sum     |  203621 |                    |         1.0         |\n",
      "| Head Weight |         |                    |         0.0         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 5.252006530761719\n",
      "pre_training - INFO - Classes: ['O', 'PER', 'ORG', 'LOC', 'MISC']\n",
      "pre_training - INFO - TransformerTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.302424)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(26056, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300,), (300,), (26056, 300)), parameters=8883000\n",
      "  (taggingLayer): SoftmaxOutputLayer(\n",
      "    (output_projection): Linear(in_features=300, out_features=5, bias=True)\n",
      "  ), weights=((5, 300), (5,)), parameters=1505\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 8.884.505\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 1171 Iterations per epoch with batch size of 12\n",
      "pre_training - INFO - Total iterations: 40985\n",
      "pre_training - INFO - Total number of samples: 491820\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cf705e575bb4ecfa1b88ac5ef92b258",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t14k\t0.07\t\t1.11\t\t0.738\t\t0.731\t\t2.13m - 2.1m / 0.0m\n",
      "2\t28k\t0.02\t\t0.71\t\t0.889\t\t0.884\t\t2.17m - 4.4m / 74.5m\n",
      "3\t42k\t0.02\t\t0.50\t\t0.891\t\t0.890\t\t2.03m - 6.5m / 76.2m\n",
      "4\t56k\t0.01\t\t0.42\t\t0.910\t\t0.908\t\t2.04m - 8.5m / 71.4m\n",
      "5\t70k\t0.01\t\t0.39\t\t0.935\t\t0.932\t\t2.12m - 10.7m / 71.9m\n",
      "6\t84k\t0.01\t\t0.48\t\t0.928\t\t0.924\t\t2.15m - 12.8m / 74.2m\n",
      "7\t98k\t0.01\t\t0.39\t\t0.938\t\t0.935\t\t2.09m - 14.9m / 75.3m\n",
      "8\t112k\t0.01\t\t0.42\t\t0.928\t\t0.925\t\t2.01m - 17.0m / 73.4m\n",
      "9\t126k\t0.01\t\t0.46\t\t0.927\t\t0.923\t\t2.03m - 19.0m / 71.4m\n",
      "10\t141k\t0.01\t\t0.45\t\t0.938\t\t0.935\t\t2.02m - 21.0m / 71.7m\n",
      "11\t155k\t0.01\t\t0.42\t\t0.929\t\t0.926\t\t2.14m - 23.2m / 71.6m\n",
      "12\t169k\t0.01\t\t0.47\t\t0.932\t\t0.930\t\t2.11m - 25.3m / 74.5m\n",
      "13\t183k\t0.01\t\t0.46\t\t0.932\t\t0.929\t\t2.08m - 27.4m / 73.9m\n",
      "14\t197k\t0.01\t\t0.41\t\t0.940\t\t0.938\t\t2.05m - 29.5m / 73.2m\n",
      "15\t211k\t0.01\t\t0.43\t\t0.931\t\t0.928\t\t2.03m - 31.5m / 72.6m\n",
      "16\t225k\t0.01\t\t0.38\t\t0.940\t\t0.938\t\t2.03m - 33.6m / 72.0m\n",
      "17\t239k\t0.01\t\t0.49\t\t0.925\t\t0.922\t\t2.02m - 35.6m / 72.1m\n",
      "18\t253k\t0.01\t\t0.43\t\t0.939\t\t0.936\t\t2.02m - 37.7m / 72.1m\n",
      "19\t267k\t0.01\t\t0.46\t\t0.930\t\t0.928\t\t2.03m - 39.7m / 72.1m\n",
      "20\t281k\t0.01\t\t0.50\t\t0.942\t\t0.938\t\t2.04m - 41.7m / 72.1m\n",
      "21\t295k\t0.01\t\t0.47\t\t0.935\t\t0.932\t\t2.01m - 43.8m / 72.3m\n",
      "22\t309k\t0.01\t\t0.46\t\t0.940\t\t0.938\t\t2.05m - 45.8m / 72.0m\n",
      "23\t323k\t0.01\t\t0.44\t\t0.941\t\t0.939\t\t2.02m - 47.9m / 72.5m\n",
      "24\t337k\t0.01\t\t0.47\t\t0.937\t\t0.935\t\t2.03m - 49.9m / 72.1m\n",
      "25\t351k\t0.01\t\t0.44\t\t0.943\t\t0.940\t\t2.02m - 51.9m / 72.2m\n",
      "26\t365k\t0.01\t\t0.50\t\t0.940\t\t0.936\t\t2.04m - 54.0m / 72.2m\n",
      "27\t379k\t0.01\t\t0.52\t\t0.944\t\t0.941\t\t2.10m - 56.1m / 72.4m\n",
      "28\t393k\t0.01\t\t0.45\t\t0.940\t\t0.937\t\t2.04m - 58.2m / 72.9m\n",
      "29\t408k\t0.01\t\t0.54\t\t0.932\t\t0.929\t\t2.04m - 60.2m / 72.5m\n",
      "30\t422k\t0.01\t\t0.48\t\t0.938\t\t0.936\t\t2.03m - 62.3m / 72.5m\n",
      "31\t436k\t0.01\t\t0.52\t\t0.943\t\t0.939\t\t2.03m - 64.3m / 72.4m\n",
      "32\t450k\t0.01\t\t0.49\t\t0.937\t\t0.935\t\t2.04m - 66.4m / 72.4m\n",
      "Training duration was 3984.873498439789\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.9758064516129032\n",
      "VAL f1\t0.9438706780422093 - (0.9438706780422093)\n",
      "(macro) f1\t{'valid': 0.9908256880733946, 'test': 0.9758064516129032}\n",
      "VAL loss\t0.38134266707370124\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [1/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.38134266707370124\n",
      " | \\     )|_\tf1: 0.9438706780422093\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\CoNLL2003-FinalExperiment\\20190430\\3\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: CoNLL2003-FinalExperiment\n",
      "Description: Classification of the CoNLL-2003 NER task\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 12, 'learning_rate_schedu[...]one} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                        ner                        |\n",
      "|          batch_size          |                         12                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                      7.2e-05                      |\n",
      "|  noam_learning_rate_warmup   |                        3500                       |\n",
      "|  noam_learning_rate_factor   |                        0.5                        |\n",
      "|          adam_beta1          |                        0.9                        |\n",
      "|          adam_beta2          |                        0.98                       |\n",
      "|           adam_eps           |                       1e-08                       |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                      0.302424                     |\n",
      "|     pointwise_layer_size     |                        287                        |\n",
      "|      last_layer_dropout      |                        0.2                        |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        200                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  14041  |\n",
      "|   validation  |   3250  |\n",
      "|      test     |   3453  |\n",
      "+---------------+---------+\n",
      "+---------------------------+\n",
      "|      Vocabulary Stats     |\n",
      "+-------------------+-------+\n",
      "|     Vocabulary    |  Size |\n",
      "+-------------------+-------+\n",
      "|      comments     | 26056 |\n",
      "| aspect_sentiments |   5   |\n",
      "+-------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                        aspect_sentiments                         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     ORG     |  10025  | 4.923362521547384  |  0.9507663747845262 |\n",
      "|      O      |  169578 |  83.2811939829389  | 0.16718806017061105 |\n",
      "|     MISC    |   4593  | 2.255661253014178  |  0.9774433874698583 |\n",
      "|     PER     |  11128  | 5.4650551760378345 |  0.9453494482396216 |\n",
      "|     LOC     |   8297  | 4.074727066461711  |  0.9592527293353829 |\n",
      "|     Sum     |  203621 |                    |         1.0         |\n",
      "| Head Weight |         |                    |         0.0         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "dataset loaded. Duration: 4.873996257781982\n",
      "pre_training - INFO - Classes: ['O', 'PER', 'ORG', 'LOC', 'MISC']\n",
      "pre_training - INFO - TransformerTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.302424)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(26056, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300,), (300,), (26056, 300)), parameters=8883000\n",
      "  (taggingLayer): SoftmaxOutputLayer(\n",
      "    (output_projection): Linear(in_features=300, out_features=5, bias=True)\n",
      "  ), weights=((5, 300), (5,)), parameters=1505\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 8.884.505\n",
      "==================================\n",
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 1171 Iterations per epoch with batch size of 12\n",
      "pre_training - INFO - Total iterations: 40985\n",
      "pre_training - INFO - Total number of samples: 491820\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "367ac9c29a154ab0b4ff9274f19f936e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t14k\t0.06\t\t1.26\t\t0.716\t\t0.702\t\t2.01m - 2.0m / 0.0m\n",
      "2\t28k\t0.02\t\t0.66\t\t0.879\t\t0.873\t\t2.05m - 4.1m / 70.2m\n",
      "3\t42k\t0.02\t\t0.63\t\t0.881\t\t0.879\t\t1.97m - 6.1m / 71.6m\n",
      "4\t56k\t0.01\t\t0.40\t\t0.908\t\t0.906\t\t1.97m - 8.0m / 69.1m\n",
      "5\t70k\t0.01\t\t0.48\t\t0.903\t\t0.901\t\t1.98m - 10.0m / 69.2m\n",
      "6\t84k\t0.01\t\t0.51\t\t0.902\t\t0.899\t\t1.97m - 12.0m / 69.4m\n",
      "7\t98k\t0.01\t\t0.50\t\t0.910\t\t0.908\t\t1.98m - 14.0m / 69.0m\n",
      "8\t112k\t0.01\t\t0.51\t\t0.914\t\t0.910\t\t1.97m - 16.0m / 69.4m\n",
      "9\t126k\t0.01\t\t0.42\t\t0.930\t\t0.927\t\t1.97m - 18.0m / 69.3m\n",
      "10\t141k\t0.01\t\t0.54\t\t0.908\t\t0.905\t\t1.99m - 20.0m / 69.3m\n",
      "11\t155k\t0.01\t\t0.51\t\t0.900\t\t0.896\t\t2.01m - 22.0m / 69.8m\n",
      "12\t169k\t0.01\t\t0.46\t\t0.914\t\t0.911\t\t2.00m - 24.0m / 70.2m\n",
      "13\t183k\t0.01\t\t0.46\t\t0.908\t\t0.906\t\t1.98m - 26.0m / 70.0m\n",
      "14\t197k\t0.01\t\t0.49\t\t0.917\t\t0.914\t\t1.98m - 28.0m / 69.7m\n",
      "Training duration was 1683.1723942756653\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.9758064516129032\n",
      "VAL f1\t0.9299016163035839 - (0.9299016163035839)\n",
      "(macro) f1\t{'valid': 0.9908256880733946, 'test': 0.9758064516129032}\n",
      "VAL loss\t0.3974885346663807\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [2/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3974885346663807\n",
      " | \\     )|_\tf1: 0.9299016163035839\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\CoNLL2003-FinalExperiment\\20190430\\4\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: CoNLL2003-FinalExperiment\n",
      "Description: Classification of the CoNLL-2003 NER task\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 12, 'learning_rate_schedu[...]one} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                        ner                        |\n",
      "|          batch_size          |                         12                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                      7.2e-05                      |\n",
      "|  noam_learning_rate_warmup   |                        3500                       |\n",
      "|  noam_learning_rate_factor   |                        0.5                        |\n",
      "|          adam_beta1          |                        0.9                        |\n",
      "|          adam_beta2          |                        0.98                       |\n",
      "|           adam_eps           |                       1e-08                       |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                      0.302424                     |\n",
      "|     pointwise_layer_size     |                        287                        |\n",
      "|      last_layer_dropout      |                        0.2                        |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        200                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  14041  |\n",
      "|   validation  |   3250  |\n",
      "|      test     |   3453  |\n",
      "+---------------+---------+\n",
      "+---------------------------+\n",
      "|      Vocabulary Stats     |\n",
      "+-------------------+-------+\n",
      "|     Vocabulary    |  Size |\n",
      "+-------------------+-------+\n",
      "|      comments     | 26056 |\n",
      "| aspect_sentiments |   5   |\n",
      "+-------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                        aspect_sentiments                         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     ORG     |  10025  | 4.923362521547384  |  0.9507663747845262 |\n",
      "|      O      |  169578 |  83.2811939829389  | 0.16718806017061105 |\n",
      "|     MISC    |   4593  | 2.255661253014178  |  0.9774433874698583 |\n",
      "|     PER     |  11128  | 5.4650551760378345 |  0.9453494482396216 |\n",
      "|     LOC     |   8297  | 4.074727066461711  |  0.9592527293353829 |\n",
      "|     Sum     |  203621 |                    |         1.0         |\n",
      "| Head Weight |         |                    |         0.0         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 4.865987539291382\n",
      "pre_training - INFO - Classes: ['O', 'PER', 'ORG', 'LOC', 'MISC']\n",
      "pre_training - INFO - TransformerTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.302424)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(26056, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300,), (300,), (26056, 300)), parameters=8883000\n",
      "  (taggingLayer): SoftmaxOutputLayer(\n",
      "    (output_projection): Linear(in_features=300, out_features=5, bias=True)\n",
      "  ), weights=((5, 300), (5,)), parameters=1505\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 8.884.505\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 1171 Iterations per epoch with batch size of 12\n",
      "pre_training - INFO - Total iterations: 40985\n",
      "pre_training - INFO - Total number of samples: 491820\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2959e2273c8148d5aaca4861ccfb59cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t14k\t0.06\t\t1.34\t\t0.719\t\t0.702\t\t2.00m - 2.0m / 0.0m\n",
      "2\t28k\t0.02\t\t0.80\t\t0.877\t\t0.869\t\t2.15m - 4.2m / 70.0m\n",
      "3\t42k\t0.02\t\t0.64\t\t0.863\t\t0.860\t\t2.17m - 6.4m / 75.2m\n",
      "4\t56k\t0.01\t\t0.41\t\t0.923\t\t0.921\t\t1.96m - 8.3m / 75.8m\n",
      "5\t70k\t0.01\t\t0.41\t\t0.923\t\t0.921\t\t1.99m - 10.3m / 69.2m\n",
      "6\t84k\t0.01\t\t0.61\t\t0.888\t\t0.885\t\t1.98m - 12.3m / 69.9m\n",
      "7\t98k\t0.01\t\t0.52\t\t0.902\t\t0.899\t\t1.99m - 14.3m / 69.8m\n",
      "8\t112k\t0.01\t\t0.54\t\t0.903\t\t0.898\t\t1.96m - 16.3m / 70.0m\n",
      "9\t126k\t0.01\t\t0.43\t\t0.926\t\t0.923\t\t1.99m - 18.3m / 69.3m\n",
      "10\t141k\t0.01\t\t0.51\t\t0.923\t\t0.919\t\t1.98m - 20.3m / 70.0m\n",
      "11\t155k\t0.01\t\t0.45\t\t0.924\t\t0.920\t\t1.99m - 22.3m / 69.8m\n",
      "12\t169k\t0.01\t\t0.50\t\t0.915\t\t0.913\t\t1.99m - 24.3m / 70.0m\n",
      "13\t183k\t0.01\t\t0.47\t\t0.919\t\t0.916\t\t2.00m - 26.3m / 70.0m\n",
      "14\t197k\t0.01\t\t0.49\t\t0.936\t\t0.934\t\t1.99m - 28.3m / 70.4m\n",
      "15\t211k\t0.01\t\t0.47\t\t0.914\t\t0.911\t\t1.99m - 30.3m / 70.1m\n",
      "16\t225k\t0.01\t\t0.42\t\t0.931\t\t0.929\t\t1.94m - 32.3m / 70.1m\n",
      "17\t239k\t0.01\t\t0.56\t\t0.917\t\t0.913\t\t1.97m - 34.3m / 69.1m\n",
      "18\t253k\t0.01\t\t0.47\t\t0.935\t\t0.932\t\t1.95m - 36.2m / 69.7m\n",
      "19\t267k\t0.01\t\t0.53\t\t0.920\t\t0.916\t\t1.95m - 38.2m / 69.4m\n",
      "Training duration was 2292.5462703704834\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.9758064516129032\n",
      "VAL f1\t0.9360298280203799 - (0.9360298280203799)\n",
      "(macro) f1\t{'valid': 0.9908256880733946, 'test': 0.9758064516129032}\n",
      "VAL loss\t0.40659390031500237\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [3/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.40659390031500237\n",
      " | \\     )|_\tf1: 0.9360298280203799\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\CoNLL2003-FinalExperiment\\20190430\\5\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: CoNLL2003-FinalExperiment\n",
      "Description: Classification of the CoNLL-2003 NER task\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 12, 'learning_rate_schedu[...]one} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                        ner                        |\n",
      "|          batch_size          |                         12                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                      7.2e-05                      |\n",
      "|  noam_learning_rate_warmup   |                        3500                       |\n",
      "|  noam_learning_rate_factor   |                        0.5                        |\n",
      "|          adam_beta1          |                        0.9                        |\n",
      "|          adam_beta2          |                        0.98                       |\n",
      "|           adam_eps           |                       1e-08                       |\n",
      "|      adam_weight_decay       |                       1e-06                       |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                      0.302424                     |\n",
      "|     pointwise_layer_size     |                        287                        |\n",
      "|      last_layer_dropout      |                        0.2                        |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        200                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  14041  |\n",
      "|   validation  |   3250  |\n",
      "|      test     |   3453  |\n",
      "+---------------+---------+\n",
      "+---------------------------+\n",
      "|      Vocabulary Stats     |\n",
      "+-------------------+-------+\n",
      "|     Vocabulary    |  Size |\n",
      "+-------------------+-------+\n",
      "|      comments     | 26056 |\n",
      "| aspect_sentiments |   5   |\n",
      "+-------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                        aspect_sentiments                         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     ORG     |  10025  | 4.923362521547384  |  0.9507663747845262 |\n",
      "|      O      |  169578 |  83.2811939829389  | 0.16718806017061105 |\n",
      "|     MISC    |   4593  | 2.255661253014178  |  0.9774433874698583 |\n",
      "|     PER     |  11128  | 5.4650551760378345 |  0.9453494482396216 |\n",
      "|     LOC     |   8297  | 4.074727066461711  |  0.9592527293353829 |\n",
      "|     Sum     |  203621 |                    |         1.0         |\n",
      "| Head Weight |         |                    |         0.0         |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 5.0009987354278564\n",
      "pre_training - INFO - Classes: ['O', 'PER', 'ORG', 'LOC', 'MISC']\n",
      "pre_training - INFO - TransformerTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.302424)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.302424)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=287, bias=False)\n",
      "          (w_2): Linear(in_features=287, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.302424)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(26056, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (287, 300), (300, 287), (300,), (300,), (300,), (300,), (26056, 300)), parameters=8883000\n",
      "  (taggingLayer): SoftmaxOutputLayer(\n",
      "    (output_projection): Linear(in_features=300, out_features=5, bias=True)\n",
      "  ), weights=((5, 300), (5,)), parameters=1505\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 8.884.505\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 1171 Iterations per epoch with batch size of 12\n",
      "pre_training - INFO - Total iterations: 40985\n",
      "pre_training - INFO - Total number of samples: 491820\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b86b9553ffcc40b4b5ca4d436e3891c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t14k\t0.06\t\t0.86\t\t0.793\t\t0.787\t\t2.00m - 2.0m / 0.0m\n",
      "2\t28k\t0.02\t\t0.61\t\t0.878\t\t0.874\t\t2.02m - 4.0m / 70.1m\n",
      "3\t42k\t0.02\t\t0.44\t\t0.908\t\t0.906\t\t2.22m - 6.3m / 70.9m\n",
      "4\t56k\t0.01\t\t0.40\t\t0.919\t\t0.918\t\t2.12m - 8.4m / 77.3m\n",
      "5\t70k\t0.01\t\t0.41\t\t0.928\t\t0.926\t\t2.23m - 10.7m / 74.0m\n",
      "6\t84k\t0.01\t\t0.48\t\t0.928\t\t0.925\t\t2.13m - 12.8m / 77.4m\n",
      "7\t98k\t0.01\t\t0.41\t\t0.930\t\t0.927\t\t2.16m - 15.0m / 74.6m\n",
      "8\t112k\t0.01\t\t0.36\t\t0.934\t\t0.933\t\t2.15m - 17.2m / 75.5m\n",
      "9\t126k\t0.01\t\t0.41\t\t0.936\t\t0.934\t\t2.15m - 19.3m / 75.1m\n",
      "10\t141k\t0.01\t\t0.44\t\t0.939\t\t0.938\t\t2.11m - 21.4m / 75.2m\n",
      "11\t155k\t0.01\t\t0.39\t\t0.930\t\t0.926\t\t2.12m - 23.6m / 74.2m\n",
      "12\t169k\t0.01\t\t0.43\t\t0.938\t\t0.936\t\t2.11m - 25.7m / 74.4m\n",
      "13\t183k\t0.01\t\t0.42\t\t0.925\t\t0.923\t\t2.13m - 27.8m / 74.2m\n",
      "14\t197k\t0.01\t\t0.42\t\t0.943\t\t0.940\t\t2.13m - 30.0m / 74.7m\n",
      "15\t211k\t0.01\t\t0.39\t\t0.931\t\t0.929\t\t2.05m - 32.1m / 74.8m\n",
      "16\t225k\t0.01\t\t0.43\t\t0.939\t\t0.936\t\t2.03m - 34.1m / 73.1m\n",
      "17\t239k\t0.01\t\t0.54\t\t0.906\t\t0.902\t\t2.05m - 36.2m / 72.7m\n",
      "18\t253k\t0.01\t\t0.50\t\t0.937\t\t0.934\t\t2.08m - 38.3m / 73.1m\n",
      "19\t267k\t0.01\t\t0.46\t\t0.931\t\t0.929\t\t2.11m - 40.4m / 73.6m\n",
      "Training duration was 2426.303972005844\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.9596774193548389\n",
      "VAL f1\t0.9429698638698463 - (0.9429698638698463)\n",
      "(macro) f1\t{'valid': 0.9724770642201835, 'test': 0.9596774193548389}\n",
      "VAL loss\t0.36433127904383356\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [4/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.36433127904383356\n",
      " | \\     )|_\tf1: 0.9429698638698463\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "#################################################################################\n",
      "############################## EXPERIMENT COMPLETE ##############################\n",
      "\n",
      "\n",
      "Run [0/5]: 0.9261606580860808\n",
      "Run [1/5]: 0.9232427488894696\n",
      "Run [2/5]: 0.9070501654763978\n",
      "Run [3/5]: 0.9119561195394293\n",
      "Run [4/5]: 0.9195935065500283\n",
      "------------------------------\n",
      "Mean: 0.9176006397082812\n",
      "TEST MICRO F1 Statistics\n",
      "count    5.000000\n",
      "mean     0.917601\n",
      "std      0.007941\n",
      "min      0.907050\n",
      "25%      0.911956\n",
      "50%      0.919594\n",
      "75%      0.923243\n",
      "max      0.926161\n",
      "Name: test_f1, dtype: float64\n",
      "TEST MACRO F1 Statistics\n",
      "count    5.000000\n",
      "mean     0.972581\n",
      "std      0.007213\n",
      "min      0.959677\n",
      "25%      0.975806\n",
      "50%      0.975806\n",
      "75%      0.975806\n",
      "max      0.975806\n",
      "Name: test_f1_macro, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "for e in experiments:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    test_params = {**baseline, **{'num_epochs': 35, 'language': 'en'}}\n",
    "    test_params = {**test_params, **e['rc']}\n",
    "    e = Experiment(main_experiment_name, e['description'], default_params, test_params, dsl, runs=5)\n",
    "    df, e_path = e.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save dataframe of experiment to C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\CoNLL2003-FinalExperiment\\20190430\\0\\exp_df.pkl\n"
     ]
    }
   ],
   "source": [
    "p = os.path.join(e_path, 'exp_df.pkl')\n",
    "print('Save dataframe of experiment to ' + p)\n",
    "df.to_pickle(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}